virt-subproc/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import re as regexp
  33
  34 debuglevel = None
  35 progname = "<VirtSubproc>"
  36 devnull_read = file('/dev/null','r')
  37 caller = __main__
  38 copy_timeout = 300
  39
  40 class Quit:
  41         def __init__(q,ec,m): q.ec = ec; q.m = m
  42
  43 class Timeout: pass
  44 def alarm_handler(*a): raise Timeout()
  45 def timeout_start(to): signal.alarm(to)
  46 def timeout_stop(): signal.alarm(0)
  47
  48 class FailedCmd:
  49         def __init__(fc,e): fc.e = e
  50
  51 def debug(m):
  52         if not debuglevel: return
  53         print >> sys.stderr, progname+": debug:", m
  54
  55 def bomb(m):
  56         raise Quit(12, progname+": failure: %s" % m)
  57
  58 def ok(): print 'ok'
  59
  60 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  61         if len(c) < 1+nargs:
  62                 bomb("too few arguments to command `%s'" % ce[0])
  63         if noptargs is not None and len(c) > 1+nargs+noptargs:
  64                 bomb("too many arguments to command `%s'" % ce[0])
  65
  66 def cmd_capabilities(c, ce):
  67         cmdnumargs(c, ce)
  68         return caller.hook_capabilities() + ['execute-debug']
  69
  70 def cmd_quit(c, ce):
  71         cmdnumargs(c, ce)
  72         raise Quit(0, '')
  73
  74 def cmd_close(c, ce):
  75         cmdnumargs(c, ce)
  76         if not downtmp: bomb("`close' when not open")
  77         cleanup()
  78
  79 def preexecfn():
  80         caller.hook_forked_inchild()
  81
  82 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  83         debug(" ++ %s" % string.join(popenargs[0]))
  84         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  85         if instr is None: popenargsk['stdin'] = devnull_read
  86         timeout_start(timeout)
  87         (out, err) = sp.communicate(instr)
  88         timeout_stop()
  89         if err: bomb("%s unexpectedly produced stderr output `%s'" %
  90                         (what, err))
  91         status = sp.wait()
  92         return (status, out)
  93
  94 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
  95         cmdl = cmd_string.split()
  96
  97         if downp: perhaps_down = down
  98         else: downp = []
  99
 100         if outp: stdout = subprocess.PIPE
 101         else: stdout = None
 102
 103         cmd = cmdl + cmd_list
 104         if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)]
 105
 106         (status, out) = execute_raw(cmdl[0], None, timeout,
 107                                 cmd, stdout=stdout)
 108
 109         if status: bomb("%s%s failed (exit status %d)" %
 110                         ((downp and "(down) " or ""), cmdl[0], status))
 111
 112         if outp and out and out[-1]=='\n': out = out[:-1]
 113         return out
 114
 115 def cmd_open(c, ce):
 116         global downtmp
 117         cmdnumargs(c, ce)
 118         if downtmp: bomb("`open' when already open")
 119         downtmp = caller.hook_open()
 120         return [downtmp]
 121
 122 def cmd_revert(c, ce):
 123         cmdnumargs(c, ce)
 124         if not downtmp: bomb("`revert' when not open")
 125         if not 'revert' in caller.hook_capabilities():
 126                 bomb("`revert' when `revert' not advertised")
 127         caller.hook_revert()
 128
 129 def down_python_script(gobody, functions=''):
 130         # Many things are made much harder by the inability of
 131         # dchroot, ssh, et al, to cope without mangling the arguments.
 132         # So we run a sub-python on the testbed and feed it a script
 133         # on stdin.  The sub-python decodes the arguments.
 134
 135         script = (      "import urllib\n"
 136                         "import os\n"
 137                         "def setfd(fd,fnamee,write,mode=0666):\n"
 138                         "       fname = urllib.unquote(fnamee)\n"
 139                         "       if write: rw = os.O_WRONLY|os.O_CREAT|os.O_TRUNC\n"
 140                         "       else: rw = os.O_RDONLY\n"
 141                         "       nfd = os.open(fname, rw, mode)\n"
 142                         "       if fd >= 0: os.dup2(nfd,fd)\n"
 143                         + functions +
 144                         "def go():\n" )
 145         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 146                                 urllib.quote(downtmp)   )
 147         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 148         script += (     gobody +
 149                         "go()\n" )
 150
 151         debug("+P ...\n"+script)
 152
 153         scripte = urllib.quote(script)
 154         cmdl = down + ["exec python -c 'import urllib; s = urllib.unquote(%s);"
 155                        " exec s'" % ('"%s"' % scripte)]
 156         return cmdl
 157
 158 def cmd_execute(c, ce):
 159         cmdnumargs(c, ce, 5, None)
 160         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 161         debug_g = None
 162         timeout = 0
 163         envs = []
 164         for kw in ce[6:]:
 165                 if kw.startswith('debug='):
 166                         if debug_g: bomb("multiple debug= in execute")
 167                         m = debug_re.match(kw)
 168                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 169                         debug_g = m.groups()
 170                 elif kw.startswith('timeout='):
 171                         try: timeout = int(kw[8:],0)
 172                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 173                 elif kw.startswith('env='):
 174                         es = kw[4:]; eq = es.find('=')
 175                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 176                         envs.append((es[:eq], es[eq+1:]))
 177                 else: bomb("invalid execute kw arg `%s'" % kw)
 178
 179         gobody = "      import sys\n"
 180         stdout = None
 181         tfd = None
 182         if debug_g:
 183                 (tfd,hfd) = m.groups()
 184                 tfd = int(tfd)
 185                 gobody += "     os.dup2(1,%d)\n" % tfd
 186                 stdout = int(hfd)
 187         for ioe in range(3):
 188                 ioe_tfd = ioe
 189                 if ioe == tfd: ioe_tfd = -1
 190                 gobody += "     setfd(%d,'%s',%d)\n" % (
 191                         ioe_tfd, ce[ioe+2], ioe>0 )
 192         for e in envs:
 193                 gobody += ("    os.environ[urllib.unquote('%s')]"
 194                            " = urllib.unquote('%s')\n"
 195                                 % tuple(map(urllib.quote, e)))
 196         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 197         gobody += "     cmd = '%s'\n" % ce[1]
 198         gobody += ("    cmd = cmd.split(',')\n"
 199                 "       cmd = map(urllib.unquote, cmd)\n"
 200                 "       c0 = cmd[0]\n"
 201                 "       if '/' in c0:\n"
 202                 "               if not os.access(c0, os.X_OK):\n"
 203                 "                       status = os.stat(c0)\n"
 204                 "                       mode = status.st_mode | 0111\n"
 205                 "                       os.chmod(c0, mode)\n"
 206                 "       try: os.execvp(c0, cmd)\n"
 207                 "       except (IOError,OSError), e:\n"
 208                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 209                 "                       (c0, os.strerror(e.errno)))\n"
 210                 "               os._exit(127)\n")
 211         cmdl = down_python_script(gobody)
 212
 213         stdout_copy = None
 214         try:
 215                 if type(stdout) == type(2): stdout_copy = os.dup(stdout)
 216                 try:
 217                         (status, out) = execute_raw('sub-python', None,
 218                                 timeout, cmdl, stdout=stdout_copy,
 219                                 stdin=devnull_read, stderr=subprocess.PIPE)
 220                 except Timeout:
 221                         raise FailedCmd(['timeout'])
 222         finally:
 223                 if stdout_copy is not None: os.close(stdout_copy)
 224
 225         if out: bomb("sub-python unexpected produced stdout"
 226                         " visible to us `%s'" % out)
 227         return [`status`]
 228
 229 def copyupdown(c, ce, upp):
 230         cmdnumargs(c, ce, 2)
 231         isrc = 0
 232         idst = 1
 233         ilocal = 0 + upp
 234         iremote = 1 - upp
 235         wh = ce[0]
 236         sd = c[1:]
 237         sde = ce[1:]
 238         if not sd[0] or not sd[1]:
 239                 bomb("%s paths must be nonempty" % wh)
 240         dirsp = sd[0][-1]=='/'
 241         functions = "import errno\n"
 242         if dirsp != (sd[1][-1]=='/'):
 243                 bomb("% paths must agree about directoryness"
 244                         " (presence or absence of trailing /)" % wh)
 245         localfd = None
 246         deststdout = devnull_read
 247         srcstdin = devnull_read
 248         if not dirsp:
 249                 modestr = ''
 250                 if upp:
 251                         deststdout = file(sd[idst], 'w')
 252                 else:
 253                         srcstdin = file(sd[isrc], 'r')
 254                         status = os.fstat(srcstdin.fileno())
 255                         if status.st_mode & 0111: modestr = ',0777'
 256                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 257                                         1-upp, sde[iremote], not upp, modestr)
 258                 gobody += "     os.execvp('cat', ['cat'])\n"
 259                 localcmdl = ['cat']
 260         else:
 261                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 262                 if upp:
 263                         try: os.mkdir(sd[ilocal])
 264                         except (IOError,OSError), oe:
 265                                 if oe.errno != errno.EEXIST: raise
 266                 else:
 267                         gobody += ("    try: os.mkdir(dir)\n"
 268                                 "       except (IOError,OSError), oe:\n"
 269                                 "               if oe.errno != errno.EEXIST: raise\n")
 270                 gobody +=( "    os.chdir(dir)\n"
 271                         "       tarcmd = 'tar -f -'.split()\n")
 272                 localcmdl = 'tar -f -'.split()
 273                 taropts = [None, None]
 274                 taropts[isrc] = '-c .'
 275                 taropts[idst] = '-p -x --no-same-owner'
 276                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 277                 localcmdl += ['-C',sd[ilocal]]
 278                 localcmdl += taropts[ilocal].split()
 279                 gobody += "     os.execvp('tar', tarcmd)\n";
 280
 281         downcmdl = down_python_script(gobody, functions)
 282
 283         if upp: cmdls = (downcmdl, localcmdl)
 284         else: cmdls = (localcmdl, downcmdl)
 285
 286         debug(`["cmdls", `cmdls`]`)
 287         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 288
 289         subprocs = [None,None]
 290         debug(" +< %s" % string.join(cmdls[0]))
 291         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 292                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 293         debug(" +> %s" % string.join(cmdls[1]))
 294         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 295                         stdout=deststdout, preexec_fn=preexecfn)
 296         subprocs[0].stdout.close()
 297         timeout_start(copy_timeout)
 298         for sdn in [1,0]:
 299                 debug(" +"+"<>"[sdn]+"?");
 300                 status = subprocs[sdn].wait()
 301                 if not (status==0 or (sdn==0 and status==-13)):
 302                         timeout_stop()
 303                         bomb("%s %s failed, status %d" %
 304                                 (wh, ['source','destination'][sdn], status))
 305         timeout_stop()
 306
 307 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 308 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 309
 310 def command():
 311         sys.stdout.flush()
 312         ce = sys.stdin.readline()
 313         if not ce: bomb('end of file - caller quit?')
 314         ce = ce.rstrip().split()
 315         c = map(urllib.unquote, ce)
 316         if not c: bomb('empty commands are not permitted')
 317         debug('executing '+string.join(ce))
 318         try: f = globals()['cmd_'+c[0]]
 319         except KeyError: bomb("unknown command `%s'" % ce[0])
 320         try:
 321                 r = f(c, ce)
 322                 if not r: r = []
 323                 r.insert(0, 'ok')
 324         except FailedCmd, fc:
 325                 r = fc.e
 326         print string.join(r)
 327
 328 signal_list = [ signal.SIGHUP, signal.SIGTERM,
 329                 signal.SIGINT, signal.SIGPIPE ]
 330
 331 def sethandlers(f):
 332         for signum in signal_list: signal.signal(signum, f)
 333
 334 def cleanup():
 335         global downtmp, cleaning
 336         debug("cleanup...");
 337         sethandlers(signal.SIG_DFL)
 338         cleaning = True
 339         if downtmp: caller.hook_cleanup()
 340         cleaning = False
 341         downtmp = False
 342
 343 def error_cleanup():
 344         try:
 345                 ok = False
 346                 try:
 347                         cleanup()
 348                         ok = True
 349                 except Quit, q:
 350                         print >> sys.stderr, q.m
 351                 except:
 352                         print >> sys.stderr, "Unexpected cleanup error:"
 353                         traceback.print_exc()
 354                         print >> sys.stderr, ''
 355                 if not ok:
 356                         print >> sys.stderr, ("while cleaning up"
 357                                 " because of another error:")
 358         except:
 359                 pass
 360
 361 def prepare():
 362         global downtmp, cleaning
 363         downtmp = None
 364         def handler(sig, *any):
 365                 cleanup()
 366                 os.kill(os.getpid(), sig)
 367         sethandlers(handler)
 368
 369 def mainloop():
 370         try:
 371                 while True: command()
 372         except Quit, q:
 373                 error_cleanup()
 374                 if q.m: print >> sys.stderr, q.m
 375                 sys.exit(q.ec)
 376         except:
 377                 error_cleanup()
 378                 print >> sys.stderr, "Unexpected error:"
 379                 traceback.print_exc()
 380                 sys.exit(16)
 381
 382 def main():
 383         signal.signal(signal.SIGALRM, alarm_handler)
 384         debug("down = %s" % string.join(down))
 385         ok()
 386         prepare()
 387         mainloop()