lib/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import errno
  33 import re as regexp
  34
  35 from Autopkgtest import *
  36
  37 debuglevel = None
  38 progname = "<VirtSubproc>"
  39 devnull_read = file('/dev/null','r')
  40 caller = __main__
  41 copy_timeout = 300
  42
  43 class Quit:
  44         def __init__(q,ec,m): q.ec = ec; q.m = m
  45
  46 class Timeout: pass
  47 def alarm_handler(*a): raise Timeout()
  48 def timeout_start(to): signal.alarm(to)
  49 def timeout_stop(): signal.alarm(0)
  50
  51 class FailedCmd:
  52         def __init__(fc,e): fc.e = e
  53
  54 def debug(m):
  55         if not debuglevel: return
  56         print >> sys.stderr, progname+": debug:", m
  57
  58 def bomb(m):
  59         raise Quit(12, progname+": failure: %s" % m)
  60
  61 def ok(): print 'ok'
  62
  63 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  64         if len(c) < 1+nargs:
  65                 bomb("too few arguments to command `%s'" % ce[0])
  66         if noptargs is not None and len(c) > 1+nargs+noptargs:
  67                 bomb("too many arguments to command `%s'" % ce[0])
  68
  69 def cmd_capabilities(c, ce):
  70         cmdnumargs(c, ce)
  71         return caller.hook_capabilities() + ['execute-debug']
  72
  73 def cmd_quit(c, ce):
  74         cmdnumargs(c, ce)
  75         raise Quit(0, '')
  76
  77 def cmd_close(c, ce):
  78         cmdnumargs(c, ce)
  79         if not downtmp: bomb("`close' when not open")
  80         cleanup()
  81
  82 def cmd_print_auxverb_command(c, ce): return print_command('auxverb', c, ce)
  83 def cmd_print_shstring_command(c, ce): return print_command('shstring', c, ce)
  84
  85 def print_command(which, c, ce):
  86         global downs
  87         cmdnumargs(c, ce)
  88         if not downtmp: bomb("`print-%s-command' when not open" % which)
  89         cl = downs[which]
  90         if not len(cl):
  91                 cl = ['sh','-c','exec "$@"','x'] + cl
  92         return [','.join(map(urllib.quote, cl))]
  93
  94 def preexecfn():
  95         caller.hook_forked_inchild()
  96
  97 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  98         debug(" ++ %s" % string.join(popenargs[0]))
  99         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
 100         if instr is None: popenargsk['stdin'] = devnull_read
 101         timeout_start(timeout)
 102         (out, err) = sp.communicate(instr)
 103         timeout_stop()
 104         if err: bomb("%s unexpectedly produced stderr output `%s'" %
 105                         (what, err))
 106         status = sp.wait()
 107         return (status, out)
 108
 109 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
 110         cmdl = cmd_string.split()
 111
 112         if downp: perhaps_down = downs['auxverb']
 113         else: perhaps_down = []
 114
 115         if outp: stdout = subprocess.PIPE
 116         else: stdout = None
 117
 118         cmd = cmdl + cmd_list
 119         if len(perhaps_down): cmd = perhaps_down + cmd
 120
 121         (status, out) = execute_raw(cmdl[0], None, timeout,
 122                                 cmd, stdout=stdout)
 123
 124         if status: bomb("%s%s failed (exit status %d)" %
 125                         ((downp and "(down) " or ""), cmdl[0], status))
 126
 127         if outp and out and out[-1]=='\n': out = out[:-1]
 128         return out
 129
 130 def cmd_open(c, ce):
 131         global downtmp
 132         cmdnumargs(c, ce)
 133         if downtmp: bomb("`open' when already open")
 134         caller.hook_open()
 135         opened1()
 136         downtmp = caller.hook_downtmp()
 137         return opened2()
 138
 139 def downtmp_mktemp():
 140         global downtmp
 141         return execute('mktemp -t -d', downp=True, outp=True)
 142
 143 def downtmp_remove():
 144         global downtmp
 145         execute('rm -rf --', [downtmp], downp=True)
 146
 147 perl_quote_re = regexp.compile('[^-+=_.,;:() 0-9a-zA-Z]')
 148 def perl_quote_1chargroup(m): return '\\x%02x' % ord(m.group(0))
 149 def perl_quote(s): return '"'+perl_quote_re.sub(perl_quote_1chargroup, s)+'"'
 150
 151 def opened1():
 152         global down, downkind, downs
 153         debug("downkind = %s, down = %s" % (downkind, `down`))
 154         if downkind == 'auxverb':
 155                 downs = { 'auxverb': down,
 156                           'shstring': down + ['sh','-c'] }
 157         elif downkind == 'shstring':
 158                 downs = { 'shstring': down,
 159                           'auxverb': ['perl','-e','''
 160                         @cmd=('''+(','.join(map(perl_quote,down)))+''');
 161                         my $shstring = pop @ARGV;
 162                         s/'/'\\\\''/g foreach @ARGV;
 163                         push @cmd, "'$_'" foreach @ARGV;
 164                         my $argv0=$cmd[0];
 165                         exec $argv0 @cmd;
 166                         die "$argv0: $!";
 167                 '''] }
 168         debug("downs = %s" % `downs`)
 169
 170 def opened2():
 171         global downtmp, downs
 172         debug("downtmp = %s" % (downtmp))
 173         return [downtmp]
 174
 175 def cmd_revert(c, ce):
 176         global downtmp
 177         cmdnumargs(c, ce)
 178         if not downtmp: bomb("`revert' when not open")
 179         if not 'revert' in caller.hook_capabilities():
 180                 bomb("`revert' when `revert' not advertised")
 181         caller.hook_revert()
 182         opened1()
 183         downtmp = caller.hook_downtmp()
 184         return opened2()
 185
 186 def down_python_script(gobody, functions=''):
 187         # Many things are made much harder by the inability of
 188         # dchroot, ssh, et al, to cope without mangling the arguments.
 189         # So we run a sub-python on the testbed and feed it a script
 190         # on stdin.  The sub-python decodes the arguments.
 191
 192         script = (      "import urllib\n"
 193                         "import os\n"
 194                         "import errno\n"
 195                         "def setfd(fd,fnamee,write,mode=0666):\n"
 196                         "       fname = urllib.unquote(fnamee)\n"
 197                         "       if write: rw = os.O_WRONLY|os.O_CREAT|os.O_TRUNC\n"
 198                         "       else: rw = os.O_RDONLY\n"
 199                         "       nfd = os.open(fname, rw, mode)\n"
 200                         "       if fd >= 0: os.dup2(nfd,fd)\n"
 201                         + functions +
 202                         "def go():\n" )
 203         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 204                                 urllib.quote(downtmp)   )
 205         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 206         script += (     gobody +
 207                         "go()\n" )
 208
 209         debug("+P ...\n"+script)
 210
 211         scripte = urllib.quote(script)
 212         cmdl = (downs['shstring'] +
 213                 ["exec python -c 'import urllib; s = urllib.unquote(%s);"
 214                  " exec s'" % ('"%s"' % scripte)])
 215         return cmdl
 216
 217 def cmd_execute(c, ce):
 218         cmdnumargs(c, ce, 5, None)
 219         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 220         debug_g = None
 221         timeout = 0
 222         envs = []
 223         for kw in ce[6:]:
 224                 if kw.startswith('debug='):
 225                         if debug_g: bomb("multiple debug= in execute")
 226                         m = debug_re.match(kw)
 227                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 228                         debug_g = m.groups()
 229                 elif kw.startswith('timeout='):
 230                         try: timeout = int(kw[8:],0)
 231                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 232                 elif kw.startswith('env='):
 233                         es = kw[4:]; eq = es.find('=')
 234                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 235                         envs.append((es[:eq], es[eq+1:]))
 236                 else: bomb("invalid execute kw arg `%s'" % kw)
 237
 238         gobody = "      import sys\n"
 239         stdout = None
 240         tfd = None
 241         if debug_g:
 242                 (tfd,hfd) = m.groups()
 243                 tfd = int(tfd)
 244                 gobody += "     os.dup2(1,%d)\n" % tfd
 245                 stdout = int(hfd)
 246         for ioe in range(3):
 247                 ioe_tfd = ioe
 248                 if ioe == tfd: ioe_tfd = -1
 249                 gobody += "     setfd(%d,'%s',%d)\n" % (
 250                         ioe_tfd, ce[ioe+2], ioe>0 )
 251         for e in envs:
 252                 gobody += ("    os.environ[urllib.unquote('%s')]"
 253                            " = urllib.unquote('%s')\n"
 254                                 % tuple(map(urllib.quote, e)))
 255         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 256         gobody += "     cmd = '%s'\n" % ce[1]
 257         gobody += ("    cmd = cmd.split(',')\n"
 258                 "       cmd = map(urllib.unquote, cmd)\n"
 259                 "       c0 = cmd[0]\n"
 260                 "       if '/' in c0:\n"
 261                 "               if not os.access(c0, os.X_OK):\n"
 262                 "                       status = os.stat(c0)\n"
 263                 "                       mode = status.st_mode | 0111\n"
 264                 "                       os.chmod(c0, mode)\n"
 265                 "       try: os.execvp(c0, cmd)\n"
 266                 "       except (IOError,OSError), e:\n"
 267                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 268                 "                       (c0, os.strerror(e.errno)))\n"
 269                 "               os._exit(127)\n")
 270         cmdl = down_python_script(gobody)
 271
 272         stdout_copy = None
 273         try:
 274                 if type(stdout) == type(2): stdout_copy = os.dup(stdout)
 275                 try:
 276                         (status, out) = execute_raw('sub-python', None,
 277                                 timeout, cmdl, stdout=stdout_copy,
 278                                 stdin=devnull_read, stderr=subprocess.PIPE)
 279                 except Timeout:
 280                         raise FailedCmd(['timeout'])
 281         finally:
 282                 if stdout_copy is not None: os.close(stdout_copy)
 283
 284         if out: bomb("sub-python unexpected produced stdout"
 285                         " visible to us `%s'" % out)
 286         return [`status`]
 287
 288 def copyupdown(c, ce, upp):
 289         cmdnumargs(c, ce, 2)
 290         isrc = 0
 291         idst = 1
 292         ilocal = 0 + upp
 293         iremote = 1 - upp
 294         wh = ce[0]
 295         sd = c[1:]
 296         sde = ce[1:]
 297         if not sd[0] or not sd[1]:
 298                 bomb("%s paths must be nonempty" % wh)
 299         dirsp = sd[0][-1]=='/'
 300         functions = "import errno\n"
 301         if dirsp != (sd[1][-1]=='/'):
 302                 bomb("% paths must agree about directoryness"
 303                         " (presence or absence of trailing /)" % wh)
 304         localfd = None
 305         deststdout = devnull_read
 306         srcstdin = devnull_read
 307         if not dirsp:
 308                 modestr = ''
 309                 if upp:
 310                         deststdout = file(sd[idst], 'w')
 311                 else:
 312                         srcstdin = file(sd[isrc], 'r')
 313                         status = os.fstat(srcstdin.fileno())
 314                         if status.st_mode & 0111: modestr = ',0777'
 315                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 316                                         1-upp, sde[iremote], not upp, modestr)
 317                 gobody += "     os.execvp('cat', ['cat'])\n"
 318                 localcmdl = ['cat']
 319         else:
 320                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 321                 if upp:
 322                         try: os.mkdir(sd[ilocal])
 323                         except (IOError,OSError), oe:
 324                                 if oe.errno != errno.EEXIST: raise
 325                 else:
 326                         gobody += ("    try: os.mkdir(dir)\n"
 327                                 "       except (IOError,OSError), oe:\n"
 328                                 "               if oe.errno != errno.EEXIST: raise\n")
 329                 gobody +=( "    os.chdir(dir)\n"
 330                         "       tarcmd = 'tar -f -'.split()\n")
 331                 localcmdl = 'tar -f -'.split()
 332                 taropts = [None, None]
 333                 taropts[isrc] = '-c .'
 334                 taropts[idst] = '-p -x --no-same-owner'
 335                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 336                 localcmdl += ['-C',sd[ilocal]]
 337                 localcmdl += taropts[ilocal].split()
 338                 gobody += "     os.execvp('tar', tarcmd)\n";
 339
 340         downcmdl = down_python_script(gobody, functions)
 341
 342         if upp: cmdls = (downcmdl, localcmdl)
 343         else: cmdls = (localcmdl, downcmdl)
 344
 345         debug(`["cmdls", `cmdls`]`)
 346         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 347
 348         subprocs = [None,None]
 349         debug(" +< %s" % string.join(cmdls[0]))
 350         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 351                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 352         debug(" +> %s" % string.join(cmdls[1]))
 353         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 354                         stdout=deststdout, preexec_fn=preexecfn)
 355         subprocs[0].stdout.close()
 356         timeout_start(copy_timeout)
 357         for sdn in [1,0]:
 358                 debug(" +"+"<>"[sdn]+"?");
 359                 status = subprocs[sdn].wait()
 360                 if not (status==0 or (sdn==0 and status==-13)):
 361                         timeout_stop()
 362                         bomb("%s %s failed, status %d" %
 363                                 (wh, ['source','destination'][sdn], status))
 364         timeout_stop()
 365
 366 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 367 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 368
 369 def command():
 370         sys.stdout.flush()
 371         ce = sys.stdin.readline()
 372         if not ce: bomb('end of file - caller quit?')
 373         ce = ce.rstrip().split()
 374         c = map(urllib.unquote, ce)
 375         if not c: bomb('empty commands are not permitted')
 376         debug('executing '+string.join(ce))
 377         c_lookup = c[0].replace('-','_')
 378         try: f = globals()['cmd_'+c_lookup]
 379         except KeyError: bomb("unknown command `%s'" % ce[0])
 380         try:
 381                 r = f(c, ce)
 382                 if not r: r = []
 383                 r.insert(0, 'ok')
 384         except FailedCmd, fc:
 385                 r = fc.e
 386         print string.join(r)
 387
 388 signal_list = [ signal.SIGHUP, signal.SIGTERM,
 389                 signal.SIGINT, signal.SIGPIPE ]
 390
 391 def sethandlers(f):
 392         for signum in signal_list: signal.signal(signum, f)
 393
 394 def cleanup():
 395         global downtmp, cleaning
 396         debug("cleanup...");
 397         sethandlers(signal.SIG_DFL)
 398         cleaning = True
 399         if downtmp:
 400                 caller.hook_cleanup()
 401         cleaning = False
 402         downtmp = False
 403
 404 def error_cleanup():
 405         try:
 406                 ok = False
 407                 try:
 408                         cleanup()
 409                         ok = True
 410                 except Quit, q:
 411                         print >> sys.stderr, q.m
 412                 except:
 413                         print >> sys.stderr, "Unexpected cleanup error:"
 414                         traceback.print_exc()
 415                         print >> sys.stderr, ''
 416                 if not ok:
 417                         print >> sys.stderr, ("while cleaning up"
 418                                 " because of another error:")
 419         except:
 420                 pass
 421
 422 def prepare():
 423         global downtmp, cleaning
 424         downtmp = None
 425         def handler(sig, *any):
 426                 cleanup()
 427                 os.kill(os.getpid(), sig)
 428         sethandlers(handler)
 429
 430 def mainloop():
 431         try:
 432                 while True: command()
 433         except Quit, q:
 434                 error_cleanup()
 435                 if q.m: print >> sys.stderr, q.m
 436                 sys.exit(q.ec)
 437         except:
 438                 error_cleanup()
 439                 print >> sys.stderr, "Unexpected error:"
 440                 traceback.print_exc()
 441                 sys.exit(16)
 442
 443 def main():
 444         signal.signal(signal.SIGALRM, alarm_handler)
 445         ok()
 446         prepare()
 447         mainloop()