lib/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import errno
  33 import re as regexp
  34
  35 debuglevel = None
  36 progname = "<VirtSubproc>"
  37 devnull_read = file('/dev/null','r')
  38 caller = __main__
  39 copy_timeout = 300
  40
  41 class Quit:
  42         def __init__(q,ec,m): q.ec = ec; q.m = m
  43
  44 class Timeout: pass
  45 def alarm_handler(*a): raise Timeout()
  46 def timeout_start(to): signal.alarm(to)
  47 def timeout_stop(): signal.alarm(0)
  48
  49 class FailedCmd:
  50         def __init__(fc,e): fc.e = e
  51
  52 def debug(m):
  53         if not debuglevel: return
  54         print >> sys.stderr, progname+": debug:", m
  55
  56 def bomb(m):
  57         raise Quit(12, progname+": failure: %s" % m)
  58
  59 def ok(): print 'ok'
  60
  61 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  62         if len(c) < 1+nargs:
  63                 bomb("too few arguments to command `%s'" % ce[0])
  64         if noptargs is not None and len(c) > 1+nargs+noptargs:
  65                 bomb("too many arguments to command `%s'" % ce[0])
  66
  67 def cmd_capabilities(c, ce):
  68         cmdnumargs(c, ce)
  69         return caller.hook_capabilities() + ['execute-debug']
  70
  71 def cmd_quit(c, ce):
  72         cmdnumargs(c, ce)
  73         raise Quit(0, '')
  74
  75 def cmd_close(c, ce):
  76         cmdnumargs(c, ce)
  77         if not downtmp: bomb("`close' when not open")
  78         cleanup()
  79
  80 def cmd_print_auxverb_command(c, ce): return print_command('auxverb', c, ce)
  81 def cmd_print_shstring_command(c, ce): return print_command('shstring', c, ce)
  82
  83 def print_command(which, c, ce):
  84         global downs
  85         cmdnumargs(c, ce)
  86         if not downtmp: bomb("`print-%s-command' when not open" % which)
  87         cl = downs[which]
  88         if not len(cl):
  89                 cl = ['sh','-c','exec "$@"','x'] + cl
  90         return [','.join(map(urllib.quote, cl))]
  91
  92 def preexecfn():
  93         caller.hook_forked_inchild()
  94
  95 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  96         debug(" ++ %s" % string.join(popenargs[0]))
  97         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  98         if instr is None: popenargsk['stdin'] = devnull_read
  99         timeout_start(timeout)
 100         (out, err) = sp.communicate(instr)
 101         timeout_stop()
 102         if err: bomb("%s unexpectedly produced stderr output `%s'" %
 103                         (what, err))
 104         status = sp.wait()
 105         return (status, out)
 106
 107 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
 108         cmdl = cmd_string.split()
 109
 110         if downp: perhaps_down = downs['auxverb']
 111         else: perhaps_down = []
 112
 113         if outp: stdout = subprocess.PIPE
 114         else: stdout = None
 115
 116         cmd = cmdl + cmd_list
 117         if len(perhaps_down): cmd = perhaps_down + cmd
 118
 119         (status, out) = execute_raw(cmdl[0], None, timeout,
 120                                 cmd, stdout=stdout)
 121
 122         if status: bomb("%s%s failed (exit status %d)" %
 123                         ((downp and "(down) " or ""), cmdl[0], status))
 124
 125         if outp and out and out[-1]=='\n': out = out[:-1]
 126         return out
 127
 128 def cmd_open(c, ce):
 129         global downtmp
 130         cmdnumargs(c, ce)
 131         if downtmp: bomb("`open' when already open")
 132         caller.hook_open()
 133         opened1()
 134         downtmp = caller.hook_downtmp()
 135         return opened2()
 136
 137 def downtmp_mktemp():
 138         global downtmp
 139         return execute('mktemp -t -d', downp=True, outp=True)
 140
 141 def downtmp_remove():
 142         global downtmp
 143         execute('rm -rf --', [downtmp], downp=True)
 144
 145 perl_quote_re = regexp.compile('[^-+=_.,;:() 0-9a-zA-Z]')
 146 def perl_quote_1chargroup(m): return '\\x%02x' % ord(m.group(0))
 147 def perl_quote(s): return '"'+perl_quote_re.sub(perl_quote_1chargroup, s)+'"'
 148
 149 def opened1():
 150         global down, downkind, downs
 151         debug("downkind = %s, down = %s" % (downkind, `down`))
 152         if downkind == 'auxverb':
 153                 downs = { 'auxverb': down,
 154                           'shstring': down + ['sh','-c'] }
 155         elif downkind == 'shstring':
 156                 downs = { 'shstring': down,
 157                           'auxverb': ['perl','-e','''
 158                         @cmd=('''+(','.join(map(perl_quote,down)))+''');
 159                         my $shstring = pop @ARGV;
 160                         s/'/'\\\\''/g foreach @ARGV;
 161                         push @cmd, "'$_'" foreach @ARGV;
 162                         my $argv0=$cmd[0];
 163                         exec $argv0 @cmd;
 164                         die "$argv0: $!";
 165                 '''] }
 166         debug("downs = %s" % `downs`)
 167
 168 def opened2():
 169         global downtmp, downs
 170         debug("downtmp = %s" % (downtmp))
 171         return [downtmp]
 172
 173 def cmd_revert(c, ce):
 174         global downtmp
 175         cmdnumargs(c, ce)
 176         if not downtmp: bomb("`revert' when not open")
 177         if not 'revert' in caller.hook_capabilities():
 178                 bomb("`revert' when `revert' not advertised")
 179         caller.hook_revert()
 180         opened1()
 181         downtmp = caller.hook_downtmp()
 182         return opened2()
 183
 184 def down_python_script(gobody, functions=''):
 185         # Many things are made much harder by the inability of
 186         # dchroot, ssh, et al, to cope without mangling the arguments.
 187         # So we run a sub-python on the testbed and feed it a script
 188         # on stdin.  The sub-python decodes the arguments.
 189
 190         script = (      "import urllib\n"
 191                         "import os\n"
 192                         "import errno\n"
 193                         "def setfd(fd,fnamee,write,mode=0666):\n"
 194                         "       fname = urllib.unquote(fnamee)\n"
 195                         "       if write: rw = os.O_WRONLY|os.O_CREAT|os.O_TRUNC\n"
 196                         "       else: rw = os.O_RDONLY\n"
 197                         "       nfd = os.open(fname, rw, mode)\n"
 198                         "       if fd >= 0: os.dup2(nfd,fd)\n"
 199                         + functions +
 200                         "def go():\n" )
 201         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 202                                 urllib.quote(downtmp)   )
 203         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 204         script += (     gobody +
 205                         "go()\n" )
 206
 207         debug("+P ...\n"+script)
 208
 209         scripte = urllib.quote(script)
 210         cmdl = (downs['shstring'] +
 211                 ["exec python -c 'import urllib; s = urllib.unquote(%s);"
 212                  " exec s'" % ('"%s"' % scripte)])
 213         return cmdl
 214
 215 def cmd_execute(c, ce):
 216         cmdnumargs(c, ce, 5, None)
 217         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 218         debug_g = None
 219         timeout = 0
 220         envs = []
 221         for kw in ce[6:]:
 222                 if kw.startswith('debug='):
 223                         if debug_g: bomb("multiple debug= in execute")
 224                         m = debug_re.match(kw)
 225                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 226                         debug_g = m.groups()
 227                 elif kw.startswith('timeout='):
 228                         try: timeout = int(kw[8:],0)
 229                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 230                 elif kw.startswith('env='):
 231                         es = kw[4:]; eq = es.find('=')
 232                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 233                         envs.append((es[:eq], es[eq+1:]))
 234                 else: bomb("invalid execute kw arg `%s'" % kw)
 235
 236         gobody = "      import sys\n"
 237         stdout = None
 238         tfd = None
 239         if debug_g:
 240                 (tfd,hfd) = m.groups()
 241                 tfd = int(tfd)
 242                 gobody += "     os.dup2(1,%d)\n" % tfd
 243                 stdout = int(hfd)
 244         for ioe in range(3):
 245                 ioe_tfd = ioe
 246                 if ioe == tfd: ioe_tfd = -1
 247                 gobody += "     setfd(%d,'%s',%d)\n" % (
 248                         ioe_tfd, ce[ioe+2], ioe>0 )
 249         for e in envs:
 250                 gobody += ("    os.environ[urllib.unquote('%s')]"
 251                            " = urllib.unquote('%s')\n"
 252                                 % tuple(map(urllib.quote, e)))
 253         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 254         gobody += "     cmd = '%s'\n" % ce[1]
 255         gobody += ("    cmd = cmd.split(',')\n"
 256                 "       cmd = map(urllib.unquote, cmd)\n"
 257                 "       c0 = cmd[0]\n"
 258                 "       if '/' in c0:\n"
 259                 "               if not os.access(c0, os.X_OK):\n"
 260                 "                       status = os.stat(c0)\n"
 261                 "                       mode = status.st_mode | 0111\n"
 262                 "                       os.chmod(c0, mode)\n"
 263                 "       try: os.execvp(c0, cmd)\n"
 264                 "       except (IOError,OSError), e:\n"
 265                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 266                 "                       (c0, os.strerror(e.errno)))\n"
 267                 "               os._exit(127)\n")
 268         cmdl = down_python_script(gobody)
 269
 270         stdout_copy = None
 271         try:
 272                 if type(stdout) == type(2): stdout_copy = os.dup(stdout)
 273                 try:
 274                         (status, out) = execute_raw('sub-python', None,
 275                                 timeout, cmdl, stdout=stdout_copy,
 276                                 stdin=devnull_read, stderr=subprocess.PIPE)
 277                 except Timeout:
 278                         raise FailedCmd(['timeout'])
 279         finally:
 280                 if stdout_copy is not None: os.close(stdout_copy)
 281
 282         if out: bomb("sub-python unexpected produced stdout"
 283                         " visible to us `%s'" % out)
 284         return [`status`]
 285
 286 def copyupdown(c, ce, upp):
 287         cmdnumargs(c, ce, 2)
 288         isrc = 0
 289         idst = 1
 290         ilocal = 0 + upp
 291         iremote = 1 - upp
 292         wh = ce[0]
 293         sd = c[1:]
 294         sde = ce[1:]
 295         if not sd[0] or not sd[1]:
 296                 bomb("%s paths must be nonempty" % wh)
 297         dirsp = sd[0][-1]=='/'
 298         functions = "import errno\n"
 299         if dirsp != (sd[1][-1]=='/'):
 300                 bomb("% paths must agree about directoryness"
 301                         " (presence or absence of trailing /)" % wh)
 302         localfd = None
 303         deststdout = devnull_read
 304         srcstdin = devnull_read
 305         if not dirsp:
 306                 modestr = ''
 307                 if upp:
 308                         deststdout = file(sd[idst], 'w')
 309                 else:
 310                         srcstdin = file(sd[isrc], 'r')
 311                         status = os.fstat(srcstdin.fileno())
 312                         if status.st_mode & 0111: modestr = ',0777'
 313                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 314                                         1-upp, sde[iremote], not upp, modestr)
 315                 gobody += "     os.execvp('cat', ['cat'])\n"
 316                 localcmdl = ['cat']
 317         else:
 318                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 319                 if upp:
 320                         try: os.mkdir(sd[ilocal])
 321                         except (IOError,OSError), oe:
 322                                 if oe.errno != errno.EEXIST: raise
 323                 else:
 324                         gobody += ("    try: os.mkdir(dir)\n"
 325                                 "       except (IOError,OSError), oe:\n"
 326                                 "               if oe.errno != errno.EEXIST: raise\n")
 327                 gobody +=( "    os.chdir(dir)\n"
 328                         "       tarcmd = 'tar -f -'.split()\n")
 329                 localcmdl = 'tar -f -'.split()
 330                 taropts = [None, None]
 331                 taropts[isrc] = '-c .'
 332                 taropts[idst] = '-p -x --no-same-owner'
 333                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 334                 localcmdl += ['-C',sd[ilocal]]
 335                 localcmdl += taropts[ilocal].split()
 336                 gobody += "     os.execvp('tar', tarcmd)\n";
 337
 338         downcmdl = down_python_script(gobody, functions)
 339
 340         if upp: cmdls = (downcmdl, localcmdl)
 341         else: cmdls = (localcmdl, downcmdl)
 342
 343         debug(`["cmdls", `cmdls`]`)
 344         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 345
 346         subprocs = [None,None]
 347         debug(" +< %s" % string.join(cmdls[0]))
 348         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 349                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 350         debug(" +> %s" % string.join(cmdls[1]))
 351         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 352                         stdout=deststdout, preexec_fn=preexecfn)
 353         subprocs[0].stdout.close()
 354         timeout_start(copy_timeout)
 355         for sdn in [1,0]:
 356                 debug(" +"+"<>"[sdn]+"?");
 357                 status = subprocs[sdn].wait()
 358                 if not (status==0 or (sdn==0 and status==-13)):
 359                         timeout_stop()
 360                         bomb("%s %s failed, status %d" %
 361                                 (wh, ['source','destination'][sdn], status))
 362         timeout_stop()
 363
 364 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 365 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 366
 367 def command():
 368         sys.stdout.flush()
 369         ce = sys.stdin.readline()
 370         if not ce: bomb('end of file - caller quit?')
 371         ce = ce.rstrip().split()
 372         c = map(urllib.unquote, ce)
 373         if not c: bomb('empty commands are not permitted')
 374         debug('executing '+string.join(ce))
 375         c_lookup = c[0].replace('-','_')
 376         try: f = globals()['cmd_'+c_lookup]
 377         except KeyError: bomb("unknown command `%s'" % ce[0])
 378         try:
 379                 r = f(c, ce)
 380                 if not r: r = []
 381                 r.insert(0, 'ok')
 382         except FailedCmd, fc:
 383                 r = fc.e
 384         print string.join(r)
 385
 386 signal_list = [ signal.SIGHUP, signal.SIGTERM,
 387                 signal.SIGINT, signal.SIGPIPE ]
 388
 389 def sethandlers(f):
 390         for signum in signal_list: signal.signal(signum, f)
 391
 392 def cleanup():
 393         global downtmp, cleaning
 394         debug("cleanup...");
 395         sethandlers(signal.SIG_DFL)
 396         cleaning = True
 397         if downtmp:
 398                 caller.hook_cleanup()
 399         cleaning = False
 400         downtmp = False
 401
 402 def error_cleanup():
 403         try:
 404                 ok = False
 405                 try:
 406                         cleanup()
 407                         ok = True
 408                 except Quit, q:
 409                         print >> sys.stderr, q.m
 410                 except:
 411                         print >> sys.stderr, "Unexpected cleanup error:"
 412                         traceback.print_exc()
 413                         print >> sys.stderr, ''
 414                 if not ok:
 415                         print >> sys.stderr, ("while cleaning up"
 416                                 " because of another error:")
 417         except:
 418                 pass
 419
 420 def prepare():
 421         global downtmp, cleaning
 422         downtmp = None
 423         def handler(sig, *any):
 424                 cleanup()
 425                 os.kill(os.getpid(), sig)
 426         sethandlers(handler)
 427
 428 def mainloop():
 429         try:
 430                 while True: command()
 431         except Quit, q:
 432                 error_cleanup()
 433                 if q.m: print >> sys.stderr, q.m
 434                 sys.exit(q.ec)
 435         except:
 436                 error_cleanup()
 437                 print >> sys.stderr, "Unexpected error:"
 438                 traceback.print_exc()
 439                 sys.exit(16)
 440
 441 def main():
 442         signal.signal(signal.SIGALRM, alarm_handler)
 443         ok()
 444         prepare()
 445         mainloop()