virt-subproc/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import re as regexp
  33
  34 debuglevel = None
  35 progname = "<VirtSubproc>"
  36 devnull_read = file('/dev/null','r')
  37 caller = __main__
  38 copy_timeout = 300
  39
  40 class Quit:
  41         def __init__(q,ec,m): q.ec = ec; q.m = m
  42
  43 class Timeout: pass
  44 def alarm_handler(*a): raise Timeout()
  45 def timeout_start(to): signal.alarm(to)
  46 def timeout_stop(): signal.alarm(0)
  47
  48 class FailedCmd:
  49         def __init__(fc,e): fc.e = e
  50
  51 def debug(m):
  52         if not debuglevel: return
  53         print >> sys.stderr, progname+": debug:", m
  54
  55 def bomb(m):
  56         raise Quit(12, progname+": failure: %s" % m)
  57
  58 def ok(): print 'ok'
  59
  60 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  61         if len(c) < 1+nargs:
  62                 bomb("too few arguments to command `%s'" % ce[0])
  63         if noptargs is not None and len(c) > 1+nargs+noptargs:
  64                 bomb("too many arguments to command `%s'" % ce[0])
  65
  66 def cmd_capabilities(c, ce):
  67         cmdnumargs(c, ce)
  68         return caller.hook_capabilities() + ['execute-debug',
  69                 'print-execute-command']
  70
  71 def cmd_quit(c, ce):
  72         cmdnumargs(c, ce)
  73         raise Quit(0, '')
  74
  75 def cmd_close(c, ce):
  76         cmdnumargs(c, ce)
  77         if not downtmp: bomb("`close' when not open")
  78         cleanup()
  79
  80 def cmd_print_execute_command(c, ce):
  81         cmdnumargs(c, ce)
  82         if not downtmp: bomb("`print-execute-command' when not open")
  83         if hasattr(caller,'hook_callerexeccmd'):
  84                 (cl,kvl) = caller.hook_callerexeccmd()
  85         else:
  86                 cl = down
  87                 kvl = ['shstring']
  88         return [','.join(map(urllib.quote, cl))] + kvl
  89
  90 def preexecfn():
  91         caller.hook_forked_inchild()
  92
  93 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  94         debug(" ++ %s" % string.join(popenargs[0]))
  95         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  96         if instr is None: popenargsk['stdin'] = devnull_read
  97         timeout_start(timeout)
  98         (out, err) = sp.communicate(instr)
  99         timeout_stop()
 100         if err: bomb("%s unexpectedly produced stderr output `%s'" %
 101                         (what, err))
 102         status = sp.wait()
 103         return (status, out)
 104
 105 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
 106         cmdl = cmd_string.split()
 107
 108         if downp: perhaps_down = down
 109         else: downp = []
 110
 111         if outp: stdout = subprocess.PIPE
 112         else: stdout = None
 113
 114         cmd = cmdl + cmd_list
 115         if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)]
 116
 117         (status, out) = execute_raw(cmdl[0], None, timeout,
 118                                 cmd, stdout=stdout)
 119
 120         if status: bomb("%s%s failed (exit status %d)" %
 121                         ((downp and "(down) " or ""), cmdl[0], status))
 122
 123         if outp and out and out[-1]=='\n': out = out[:-1]
 124         return out
 125
 126 def cmd_open(c, ce):
 127         global downtmp
 128         cmdnumargs(c, ce)
 129         if downtmp: bomb("`open' when already open")
 130         downtmp = caller.hook_open()
 131         return [downtmp]
 132
 133 def cmd_revert(c, ce):
 134         cmdnumargs(c, ce)
 135         if not downtmp: bomb("`revert' when not open")
 136         if not 'revert' in caller.hook_capabilities():
 137                 bomb("`revert' when `revert' not advertised")
 138         caller.hook_revert()
 139
 140 def down_python_script(gobody, functions=''):
 141         # Many things are made much harder by the inability of
 142         # dchroot, ssh, et al, to cope without mangling the arguments.
 143         # So we run a sub-python on the testbed and feed it a script
 144         # on stdin.  The sub-python decodes the arguments.
 145
 146         script = (      "import urllib\n"
 147                         "import os\n"
 148                         "def setfd(fd,fnamee,write,mode=0666):\n"
 149                         "       fname = urllib.unquote(fnamee)\n"
 150                         "       if write: rw = os.O_WRONLY|os.O_CREAT|os.O_TRUNC\n"
 151                         "       else: rw = os.O_RDONLY\n"
 152                         "       nfd = os.open(fname, rw, mode)\n"
 153                         "       if fd >= 0: os.dup2(nfd,fd)\n"
 154                         + functions +
 155                         "def go():\n" )
 156         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 157                                 urllib.quote(downtmp)   )
 158         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 159         script += (     gobody +
 160                         "go()\n" )
 161
 162         debug("+P ...\n"+script)
 163
 164         scripte = urllib.quote(script)
 165         cmdl = down + ["exec python -c 'import urllib; s = urllib.unquote(%s);"
 166                        " exec s'" % ('"%s"' % scripte)]
 167         return cmdl
 168
 169 def cmd_execute(c, ce):
 170         cmdnumargs(c, ce, 5, None)
 171         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 172         debug_g = None
 173         timeout = 0
 174         envs = []
 175         for kw in ce[6:]:
 176                 if kw.startswith('debug='):
 177                         if debug_g: bomb("multiple debug= in execute")
 178                         m = debug_re.match(kw)
 179                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 180                         debug_g = m.groups()
 181                 elif kw.startswith('timeout='):
 182                         try: timeout = int(kw[8:],0)
 183                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 184                 elif kw.startswith('env='):
 185                         es = kw[4:]; eq = es.find('=')
 186                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 187                         envs.append((es[:eq], es[eq+1:]))
 188                 else: bomb("invalid execute kw arg `%s'" % kw)
 189
 190         gobody = "      import sys\n"
 191         stdout = None
 192         tfd = None
 193         if debug_g:
 194                 (tfd,hfd) = m.groups()
 195                 tfd = int(tfd)
 196                 gobody += "     os.dup2(1,%d)\n" % tfd
 197                 stdout = int(hfd)
 198         for ioe in range(3):
 199                 ioe_tfd = ioe
 200                 if ioe == tfd: ioe_tfd = -1
 201                 gobody += "     setfd(%d,'%s',%d)\n" % (
 202                         ioe_tfd, ce[ioe+2], ioe>0 )
 203         for e in envs:
 204                 gobody += ("    os.environ[urllib.unquote('%s')]"
 205                            " = urllib.unquote('%s')\n"
 206                                 % tuple(map(urllib.quote, e)))
 207         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 208         gobody += "     cmd = '%s'\n" % ce[1]
 209         gobody += ("    cmd = cmd.split(',')\n"
 210                 "       cmd = map(urllib.unquote, cmd)\n"
 211                 "       c0 = cmd[0]\n"
 212                 "       if '/' in c0:\n"
 213                 "               if not os.access(c0, os.X_OK):\n"
 214                 "                       status = os.stat(c0)\n"
 215                 "                       mode = status.st_mode | 0111\n"
 216                 "                       os.chmod(c0, mode)\n"
 217                 "       try: os.execvp(c0, cmd)\n"
 218                 "       except (IOError,OSError), e:\n"
 219                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 220                 "                       (c0, os.strerror(e.errno)))\n"
 221                 "               os._exit(127)\n")
 222         cmdl = down_python_script(gobody)
 223
 224         stdout_copy = None
 225         try:
 226                 if type(stdout) == type(2): stdout_copy = os.dup(stdout)
 227                 try:
 228                         (status, out) = execute_raw('sub-python', None,
 229                                 timeout, cmdl, stdout=stdout_copy,
 230                                 stdin=devnull_read, stderr=subprocess.PIPE)
 231                 except Timeout:
 232                         raise FailedCmd(['timeout'])
 233         finally:
 234                 if stdout_copy is not None: os.close(stdout_copy)
 235
 236         if out: bomb("sub-python unexpected produced stdout"
 237                         " visible to us `%s'" % out)
 238         return [`status`]
 239
 240 def copyupdown(c, ce, upp):
 241         cmdnumargs(c, ce, 2)
 242         isrc = 0
 243         idst = 1
 244         ilocal = 0 + upp
 245         iremote = 1 - upp
 246         wh = ce[0]
 247         sd = c[1:]
 248         sde = ce[1:]
 249         if not sd[0] or not sd[1]:
 250                 bomb("%s paths must be nonempty" % wh)
 251         dirsp = sd[0][-1]=='/'
 252         functions = "import errno\n"
 253         if dirsp != (sd[1][-1]=='/'):
 254                 bomb("% paths must agree about directoryness"
 255                         " (presence or absence of trailing /)" % wh)
 256         localfd = None
 257         deststdout = devnull_read
 258         srcstdin = devnull_read
 259         if not dirsp:
 260                 modestr = ''
 261                 if upp:
 262                         deststdout = file(sd[idst], 'w')
 263                 else:
 264                         srcstdin = file(sd[isrc], 'r')
 265                         status = os.fstat(srcstdin.fileno())
 266                         if status.st_mode & 0111: modestr = ',0777'
 267                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 268                                         1-upp, sde[iremote], not upp, modestr)
 269                 gobody += "     os.execvp('cat', ['cat'])\n"
 270                 localcmdl = ['cat']
 271         else:
 272                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 273                 if upp:
 274                         try: os.mkdir(sd[ilocal])
 275                         except (IOError,OSError), oe:
 276                                 if oe.errno != errno.EEXIST: raise
 277                 else:
 278                         gobody += ("    try: os.mkdir(dir)\n"
 279                                 "       except (IOError,OSError), oe:\n"
 280                                 "               if oe.errno != errno.EEXIST: raise\n")
 281                 gobody +=( "    os.chdir(dir)\n"
 282                         "       tarcmd = 'tar -f -'.split()\n")
 283                 localcmdl = 'tar -f -'.split()
 284                 taropts = [None, None]
 285                 taropts[isrc] = '-c .'
 286                 taropts[idst] = '-p -x --no-same-owner'
 287                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 288                 localcmdl += ['-C',sd[ilocal]]
 289                 localcmdl += taropts[ilocal].split()
 290                 gobody += "     os.execvp('tar', tarcmd)\n";
 291
 292         downcmdl = down_python_script(gobody, functions)
 293
 294         if upp: cmdls = (downcmdl, localcmdl)
 295         else: cmdls = (localcmdl, downcmdl)
 296
 297         debug(`["cmdls", `cmdls`]`)
 298         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 299
 300         subprocs = [None,None]
 301         debug(" +< %s" % string.join(cmdls[0]))
 302         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 303                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 304         debug(" +> %s" % string.join(cmdls[1]))
 305         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 306                         stdout=deststdout, preexec_fn=preexecfn)
 307         subprocs[0].stdout.close()
 308         timeout_start(copy_timeout)
 309         for sdn in [1,0]:
 310                 debug(" +"+"<>"[sdn]+"?");
 311                 status = subprocs[sdn].wait()
 312                 if not (status==0 or (sdn==0 and status==-13)):
 313                         timeout_stop()
 314                         bomb("%s %s failed, status %d" %
 315                                 (wh, ['source','destination'][sdn], status))
 316         timeout_stop()
 317
 318 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 319 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 320
 321 def command():
 322         sys.stdout.flush()
 323         ce = sys.stdin.readline()
 324         if not ce: bomb('end of file - caller quit?')
 325         ce = ce.rstrip().split()
 326         c = map(urllib.unquote, ce)
 327         if not c: bomb('empty commands are not permitted')
 328         debug('executing '+string.join(ce))
 329         c_lookup = c[0].replace('-','_')
 330         try: f = globals()['cmd_'+c_lookup]
 331         except KeyError: bomb("unknown command `%s'" % ce[0])
 332         try:
 333                 r = f(c, ce)
 334                 if not r: r = []
 335                 r.insert(0, 'ok')
 336         except FailedCmd, fc:
 337                 r = fc.e
 338         print string.join(r)
 339
 340 signal_list = [ signal.SIGHUP, signal.SIGTERM,
 341                 signal.SIGINT, signal.SIGPIPE ]
 342
 343 def sethandlers(f):
 344         for signum in signal_list: signal.signal(signum, f)
 345
 346 def cleanup():
 347         global downtmp, cleaning
 348         debug("cleanup...");
 349         sethandlers(signal.SIG_DFL)
 350         cleaning = True
 351         if downtmp: caller.hook_cleanup()
 352         cleaning = False
 353         downtmp = False
 354
 355 def error_cleanup():
 356         try:
 357                 ok = False
 358                 try:
 359                         cleanup()
 360                         ok = True
 361                 except Quit, q:
 362                         print >> sys.stderr, q.m
 363                 except:
 364                         print >> sys.stderr, "Unexpected cleanup error:"
 365                         traceback.print_exc()
 366                         print >> sys.stderr, ''
 367                 if not ok:
 368                         print >> sys.stderr, ("while cleaning up"
 369                                 " because of another error:")
 370         except:
 371                 pass
 372
 373 def prepare():
 374         global downtmp, cleaning
 375         downtmp = None
 376         def handler(sig, *any):
 377                 cleanup()
 378                 os.kill(os.getpid(), sig)
 379         sethandlers(handler)
 380
 381 def mainloop():
 382         try:
 383                 while True: command()
 384         except Quit, q:
 385                 error_cleanup()
 386                 if q.m: print >> sys.stderr, q.m
 387                 sys.exit(q.ec)
 388         except:
 389                 error_cleanup()
 390                 print >> sys.stderr, "Unexpected error:"
 391                 traceback.print_exc()
 392                 sys.exit(16)
 393
 394 def main():
 395         signal.signal(signal.SIGALRM, alarm_handler)
 396         debug("down = %s" % string.join(down))
 397         ok()
 398         prepare()
 399         mainloop()