virt-subproc/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import re as regexp
  33
  34 debuglevel = None
  35 progname = "<VirtSubproc>"
  36 devnull_read = file('/dev/null','r')
  37 caller = __main__
  38 copy_timeout = 300
  39
  40 class Quit:
  41         def __init__(q,ec,m): q.ec = ec; q.m = m
  42
  43 class Timeout: pass
  44 def alarm_handler(*a): raise Timeout()
  45 def timeout_start(to): signal.alarm(to)
  46 def timeout_stop(): signal.alarm(0)
  47
  48 class FailedCmd:
  49         def __init__(fc,e): fc.e = e
  50
  51 def debug(m):
  52         if not debuglevel: return
  53         print >> sys.stderr, progname+": debug:", m
  54
  55 def bomb(m):
  56         raise Quit(12, progname+": failure: %s" % m)
  57
  58 def ok(): print 'ok'
  59
  60 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  61         if len(c) < 1+nargs:
  62                 bomb("too few arguments to command `%s'" % ce[0])
  63         if noptargs is not None and len(c) > 1+nargs+noptargs:
  64                 bomb("too many arguments to command `%s'" % ce[0])
  65
  66 def cmd_capabilities(c, ce):
  67         cmdnumargs(c, ce)
  68         return caller.hook_capabilities() + ['execute-debug',
  69                 'print-execute-command']
  70
  71 def cmd_quit(c, ce):
  72         cmdnumargs(c, ce)
  73         raise Quit(0, '')
  74
  75 def cmd_close(c, ce):
  76         cmdnumargs(c, ce)
  77         if not downtmp: bomb("`close' when not open")
  78         cleanup()
  79
  80 def cmd_print_execute_command(c, ce):
  81         cmdnumargs(c, ce)
  82         if not downtmp: bomb("`print-execute-command' when not open")
  83         if hasattr(caller,'hook_callerexeccmd'):
  84                 (cl,kvl) = caller.hook_callerexeccmd()
  85         else:
  86                 cl = down
  87                 kvl = ['shstring']
  88         return [','.join(map(urllib.quote, cl))] + kvl
  89
  90 def preexecfn():
  91         caller.hook_forked_inchild()
  92
  93 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  94         debug(" ++ %s" % string.join(popenargs[0]))
  95         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  96         if instr is None: popenargsk['stdin'] = devnull_read
  97         timeout_start(timeout)
  98         (out, err) = sp.communicate(instr)
  99         timeout_stop()
 100         if err: bomb("%s unexpectedly produced stderr output `%s'" %
 101                         (what, err))
 102         status = sp.wait()
 103         return (status, out)
 104
 105 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
 106         cmdl = cmd_string.split()
 107
 108         if downp: perhaps_down = down
 109         else: perhaps_down = []
 110
 111         if outp: stdout = subprocess.PIPE
 112         else: stdout = None
 113
 114         cmd = cmdl + cmd_list
 115         if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)]
 116
 117         (status, out) = execute_raw(cmdl[0], None, timeout,
 118                                 cmd, stdout=stdout)
 119
 120         if status: bomb("%s%s failed (exit status %d)" %
 121                         ((downp and "(down) " or ""), cmdl[0], status))
 122
 123         if outp and out and out[-1]=='\n': out = out[:-1]
 124         return out
 125
 126 def cmd_open(c, ce):
 127         global downtmp
 128         cmdnumargs(c, ce)
 129         if downtmp: bomb("`open' when already open")
 130         downtmp = caller.hook_open()
 131         if downtmp is None:
 132                 downtmp = execute('mktemp -t -d', downp=True, outp=True)
 133         debug("down = %s, downtmp = %s" % (string.join(down), downtmp))
 134         return [downtmp]
 135
 136 def cmd_revert(c, ce):
 137         cmdnumargs(c, ce)
 138         if not downtmp: bomb("`revert' when not open")
 139         if not 'revert' in caller.hook_capabilities():
 140                 bomb("`revert' when `revert' not advertised")
 141         caller.hook_revert()
 142
 143 def down_python_script(gobody, functions=''):
 144         # Many things are made much harder by the inability of
 145         # dchroot, ssh, et al, to cope without mangling the arguments.
 146         # So we run a sub-python on the testbed and feed it a script
 147         # on stdin.  The sub-python decodes the arguments.
 148
 149         script = (      "import urllib\n"
 150                         "import os\n"
 151                         "def setfd(fd,fnamee,write,mode=0666):\n"
 152                         "       fname = urllib.unquote(fnamee)\n"
 153                         "       if write: rw = os.O_WRONLY|os.O_CREAT|os.O_TRUNC\n"
 154                         "       else: rw = os.O_RDONLY\n"
 155                         "       nfd = os.open(fname, rw, mode)\n"
 156                         "       if fd >= 0: os.dup2(nfd,fd)\n"
 157                         + functions +
 158                         "def go():\n" )
 159         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 160                                 urllib.quote(downtmp)   )
 161         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 162         script += (     gobody +
 163                         "go()\n" )
 164
 165         debug("+P ...\n"+script)
 166
 167         scripte = urllib.quote(script)
 168         cmdl = down + ["exec python -c 'import urllib; s = urllib.unquote(%s);"
 169                        " exec s'" % ('"%s"' % scripte)]
 170         return cmdl
 171
 172 def cmd_execute(c, ce):
 173         cmdnumargs(c, ce, 5, None)
 174         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 175         debug_g = None
 176         timeout = 0
 177         envs = []
 178         for kw in ce[6:]:
 179                 if kw.startswith('debug='):
 180                         if debug_g: bomb("multiple debug= in execute")
 181                         m = debug_re.match(kw)
 182                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 183                         debug_g = m.groups()
 184                 elif kw.startswith('timeout='):
 185                         try: timeout = int(kw[8:],0)
 186                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 187                 elif kw.startswith('env='):
 188                         es = kw[4:]; eq = es.find('=')
 189                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 190                         envs.append((es[:eq], es[eq+1:]))
 191                 else: bomb("invalid execute kw arg `%s'" % kw)
 192
 193         gobody = "      import sys\n"
 194         stdout = None
 195         tfd = None
 196         if debug_g:
 197                 (tfd,hfd) = m.groups()
 198                 tfd = int(tfd)
 199                 gobody += "     os.dup2(1,%d)\n" % tfd
 200                 stdout = int(hfd)
 201         for ioe in range(3):
 202                 ioe_tfd = ioe
 203                 if ioe == tfd: ioe_tfd = -1
 204                 gobody += "     setfd(%d,'%s',%d)\n" % (
 205                         ioe_tfd, ce[ioe+2], ioe>0 )
 206         for e in envs:
 207                 gobody += ("    os.environ[urllib.unquote('%s')]"
 208                            " = urllib.unquote('%s')\n"
 209                                 % tuple(map(urllib.quote, e)))
 210         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 211         gobody += "     cmd = '%s'\n" % ce[1]
 212         gobody += ("    cmd = cmd.split(',')\n"
 213                 "       cmd = map(urllib.unquote, cmd)\n"
 214                 "       c0 = cmd[0]\n"
 215                 "       if '/' in c0:\n"
 216                 "               if not os.access(c0, os.X_OK):\n"
 217                 "                       status = os.stat(c0)\n"
 218                 "                       mode = status.st_mode | 0111\n"
 219                 "                       os.chmod(c0, mode)\n"
 220                 "       try: os.execvp(c0, cmd)\n"
 221                 "       except (IOError,OSError), e:\n"
 222                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 223                 "                       (c0, os.strerror(e.errno)))\n"
 224                 "               os._exit(127)\n")
 225         cmdl = down_python_script(gobody)
 226
 227         stdout_copy = None
 228         try:
 229                 if type(stdout) == type(2): stdout_copy = os.dup(stdout)
 230                 try:
 231                         (status, out) = execute_raw('sub-python', None,
 232                                 timeout, cmdl, stdout=stdout_copy,
 233                                 stdin=devnull_read, stderr=subprocess.PIPE)
 234                 except Timeout:
 235                         raise FailedCmd(['timeout'])
 236         finally:
 237                 if stdout_copy is not None: os.close(stdout_copy)
 238
 239         if out: bomb("sub-python unexpected produced stdout"
 240                         " visible to us `%s'" % out)
 241         return [`status`]
 242
 243 def copyupdown(c, ce, upp):
 244         cmdnumargs(c, ce, 2)
 245         isrc = 0
 246         idst = 1
 247         ilocal = 0 + upp
 248         iremote = 1 - upp
 249         wh = ce[0]
 250         sd = c[1:]
 251         sde = ce[1:]
 252         if not sd[0] or not sd[1]:
 253                 bomb("%s paths must be nonempty" % wh)
 254         dirsp = sd[0][-1]=='/'
 255         functions = "import errno\n"
 256         if dirsp != (sd[1][-1]=='/'):
 257                 bomb("% paths must agree about directoryness"
 258                         " (presence or absence of trailing /)" % wh)
 259         localfd = None
 260         deststdout = devnull_read
 261         srcstdin = devnull_read
 262         if not dirsp:
 263                 modestr = ''
 264                 if upp:
 265                         deststdout = file(sd[idst], 'w')
 266                 else:
 267                         srcstdin = file(sd[isrc], 'r')
 268                         status = os.fstat(srcstdin.fileno())
 269                         if status.st_mode & 0111: modestr = ',0777'
 270                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 271                                         1-upp, sde[iremote], not upp, modestr)
 272                 gobody += "     os.execvp('cat', ['cat'])\n"
 273                 localcmdl = ['cat']
 274         else:
 275                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 276                 if upp:
 277                         try: os.mkdir(sd[ilocal])
 278                         except (IOError,OSError), oe:
 279                                 if oe.errno != errno.EEXIST: raise
 280                 else:
 281                         gobody += ("    try: os.mkdir(dir)\n"
 282                                 "       except (IOError,OSError), oe:\n"
 283                                 "               if oe.errno != errno.EEXIST: raise\n")
 284                 gobody +=( "    os.chdir(dir)\n"
 285                         "       tarcmd = 'tar -f -'.split()\n")
 286                 localcmdl = 'tar -f -'.split()
 287                 taropts = [None, None]
 288                 taropts[isrc] = '-c .'
 289                 taropts[idst] = '-p -x --no-same-owner'
 290                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 291                 localcmdl += ['-C',sd[ilocal]]
 292                 localcmdl += taropts[ilocal].split()
 293                 gobody += "     os.execvp('tar', tarcmd)\n";
 294
 295         downcmdl = down_python_script(gobody, functions)
 296
 297         if upp: cmdls = (downcmdl, localcmdl)
 298         else: cmdls = (localcmdl, downcmdl)
 299
 300         debug(`["cmdls", `cmdls`]`)
 301         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 302
 303         subprocs = [None,None]
 304         debug(" +< %s" % string.join(cmdls[0]))
 305         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 306                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 307         debug(" +> %s" % string.join(cmdls[1]))
 308         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 309                         stdout=deststdout, preexec_fn=preexecfn)
 310         subprocs[0].stdout.close()
 311         timeout_start(copy_timeout)
 312         for sdn in [1,0]:
 313                 debug(" +"+"<>"[sdn]+"?");
 314                 status = subprocs[sdn].wait()
 315                 if not (status==0 or (sdn==0 and status==-13)):
 316                         timeout_stop()
 317                         bomb("%s %s failed, status %d" %
 318                                 (wh, ['source','destination'][sdn], status))
 319         timeout_stop()
 320
 321 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 322 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 323
 324 def command():
 325         sys.stdout.flush()
 326         ce = sys.stdin.readline()
 327         if not ce: bomb('end of file - caller quit?')
 328         ce = ce.rstrip().split()
 329         c = map(urllib.unquote, ce)
 330         if not c: bomb('empty commands are not permitted')
 331         debug('executing '+string.join(ce))
 332         c_lookup = c[0].replace('-','_')
 333         try: f = globals()['cmd_'+c_lookup]
 334         except KeyError: bomb("unknown command `%s'" % ce[0])
 335         try:
 336                 r = f(c, ce)
 337                 if not r: r = []
 338                 r.insert(0, 'ok')
 339         except FailedCmd, fc:
 340                 r = fc.e
 341         print string.join(r)
 342
 343 signal_list = [ signal.SIGHUP, signal.SIGTERM,
 344                 signal.SIGINT, signal.SIGPIPE ]
 345
 346 def sethandlers(f):
 347         for signum in signal_list: signal.signal(signum, f)
 348
 349 def cleanup():
 350         global downtmp, cleaning
 351         debug("cleanup...");
 352         sethandlers(signal.SIG_DFL)
 353         cleaning = True
 354         if downtmp:
 355                 if not 'revert' in caller.hook_capabilities():
 356                         execute('rm -rf --', [downtmp], downp=True)
 357                 caller.hook_cleanup()
 358         cleaning = False
 359         downtmp = False
 360
 361 def error_cleanup():
 362         try:
 363                 ok = False
 364                 try:
 365                         cleanup()
 366                         ok = True
 367                 except Quit, q:
 368                         print >> sys.stderr, q.m
 369                 except:
 370                         print >> sys.stderr, "Unexpected cleanup error:"
 371                         traceback.print_exc()
 372                         print >> sys.stderr, ''
 373                 if not ok:
 374                         print >> sys.stderr, ("while cleaning up"
 375                                 " because of another error:")
 376         except:
 377                 pass
 378
 379 def prepare():
 380         global downtmp, cleaning
 381         downtmp = None
 382         def handler(sig, *any):
 383                 cleanup()
 384                 os.kill(os.getpid(), sig)
 385         sethandlers(handler)
 386
 387 def mainloop():
 388         try:
 389                 while True: command()
 390         except Quit, q:
 391                 error_cleanup()
 392                 if q.m: print >> sys.stderr, q.m
 393                 sys.exit(q.ec)
 394         except:
 395                 error_cleanup()
 396                 print >> sys.stderr, "Unexpected error:"
 397                 traceback.print_exc()
 398                 sys.exit(16)
 399
 400 def main():
 401         signal.signal(signal.SIGALRM, alarm_handler)
 402         ok()
 403         prepare()
 404         mainloop()