virt-subproc/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import re as regexp
  33
  34 debuglevel = None
  35 progname = "<VirtSubproc>"
  36 devnull_read = file('/dev/null','r')
  37 caller = __main__
  38
  39 class Quit:
  40         def __init__(q,ec,m): q.ec = ec; q.m = m
  41
  42 def debug(m):
  43         if not debuglevel: return
  44         print >> sys.stderr, progname+": debug:", m
  45
  46 def bomb(m):
  47         raise Quit(12, progname+": failure: %s" % m)
  48
  49 def ok(): print 'ok'
  50
  51 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  52         if len(c) < 1+nargs:
  53                 bomb("too few arguments to command `%s'" % ce[0])
  54         if noptargs is not None and len(c) > 1+nargs+noptargs:
  55                 bomb("too many arguments to command `%s'" % ce[0])
  56
  57 def cmd_capabilities(c, ce):
  58         cmdnumargs(c, ce)
  59         return caller.hook_capabilities()
  60
  61 def cmd_quit(c, ce):
  62         cmdnumargs(c, ce)
  63         raise Quit(0, '')
  64
  65 def cmd_close(c, ce):
  66         cmdnumargs(c, ce)
  67         if not downtmp: bomb("`close' when not open")
  68         cleanup()
  69
  70 def preexecfn():
  71         caller.hook_forked_inchild()
  72
  73 def execute_raw(what, instr, *popenargs, **popenargsk):
  74         debug(" ++ %s" % string.join(popenargs[0]))
  75         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  76         if instr is None: popenargsk['stdin'] = devnull_read
  77         (out, err) = sp.communicate(instr)
  78         if err: bomb("%s unexpectedly produced stderr output `%s'" %
  79                         (what, err))
  80         status = sp.wait()
  81         return (status, out)
  82
  83 def execute(cmd_string, cmd_list=[], downp=False, outp=False):
  84         cmdl = cmd_string.split()
  85
  86         if downp: perhaps_down = down
  87         else: downp = []
  88
  89         if outp: stdout = subprocess.PIPE
  90         else: stdout = None
  91
  92         cmd = cmdl + cmd_list
  93         if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)]
  94
  95         (status, out) = execute_raw(cmdl[0], None, cmd, stdout=stdout)
  96
  97         if status: bomb("%s%s failed (exit status %d)" %
  98                         ((downp and "(down) " or ""), cmdl[0], status))
  99
 100         if outp and out and out[-1]=='\n': out = out[:-1]
 101         return out
 102
 103 def cmd_open(c, ce):
 104         global downtmp
 105         cmdnumargs(c, ce)
 106         if downtmp: bomb("`open' when already open")
 107         downtmp = caller.hook_open()
 108         return [downtmp]
 109
 110 def cmd_revert(c, ce):
 111         cmdnumargs(c, ce)
 112         if not downtmp: bomb("`revert' when not open")
 113         if not 'revert' in caller.hook_capabilities():
 114                 bomb("`revert' when `revert' not advertised")
 115         caller.hook_revert()
 116
 117 def down_python_script(gobody, functions=''):
 118         # Many things are made much harder by the inability of
 119         # dchroot, ssh, et al, to cope without mangling the arguments.
 120         # So we run a sub-python on the testbed and feed it a script
 121         # on stdin.  The sub-python decodes the arguments.
 122
 123         script = (      "import urllib\n"
 124                         "import os\n"
 125                         "def setfd(fd,fnamee,write,mode=0666):\n"
 126                         "       fname = urllib.unquote(fnamee)\n"
 127                         "       if write: rw = os.O_WRONLY|os.O_CREAT\n"
 128                         "       else: rw = os.O_RDONLY\n"
 129                         "       nfd = os.open(fname, rw, mode)\n"
 130                         "       os.dup2(nfd,fd)\n"
 131                         + functions +
 132                         "def go():\n" )
 133         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 134                                 urllib.quote(downtmp)   )
 135         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 136         script += (     gobody +
 137                         "go()\n" )
 138
 139         debug("+P ...\n"+script)
 140
 141         scripte = urllib.quote(script)
 142         cmdl = down + ['python','-c',
 143                 "'import urllib; s = urllib.unquote(%s); exec s'" %
 144                         ('"%s"' % scripte)]
 145         return cmdl
 146
 147 def cmd_execute(c, ce):
 148         cmdnumargs(c, ce, 5, None)
 149         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 150         debug_g = None
 151         for kw in ce[6:]:
 152                 if kw.startswith('debug='):
 153                         if debug_g: bomb("multiple debug= in execute")
 154                         m = debug_re.match(kw)
 155                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 156                         debug_g = m.groups()
 157                 else: bomb("invalid execute kw arg `%s'" % kw)
 158
 159         gobody = "      import sys\n"
 160         stdout = None
 161         tfd = None
 162         if debug_g:
 163                 (tfd,hfd) = m.groups()
 164                 gobody += "     os.dup2(1,%d)\n" % int(tfd)
 165                 stdout = int(hfd)
 166         for ioe in range(3):
 167                 if ioe == tfd: continue
 168                 gobody += "     setfd(%d,'%s',%d)\n" % (
 169                         ioe, ce[ioe+2], ioe>0 )
 170         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 171         gobody += "     cmd = '%s'\n" % ce[1]
 172         gobody += ("    cmd = cmd.split(',')\n"
 173                 "       cmd = map(urllib.unquote, cmd)\n"
 174                 "       c0 = cmd[0]\n"
 175                 "       if '/' in c0:\n"
 176                 "               if not os.access(c0, os.X_OK):\n"
 177                 "                       status = os.stat(c0)\n"
 178                 "                       mode = status.st_mode | 0111\n"
 179                 "                       os.chmod(c0, mode)\n"
 180                 "       try: os.execvp(c0, cmd)\n"
 181                 "       except OSError, e:\n"
 182                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 183                 "                       (c0, os.strerror(e.errno)))\n"
 184                 "               os._exit(127)\n")
 185         cmdl = down_python_script(gobody)
 186
 187         (status, out) = execute_raw('sub-python', None, cmdl, stdout=stdout,
 188                                 stdin=devnull_read, stderr=subprocess.PIPE)
 189         if out: bomb("sub-python unexpected produced stdout"
 190                         " visible to us `%s'" % out)
 191         return [`status`]
 192
 193 def copyupdown(c, ce, upp):
 194         cmdnumargs(c, ce, 2)
 195         isrc = 0
 196         idst = 1
 197         ilocal = 0 + upp
 198         iremote = 1 - upp
 199         wh = ce[0]
 200         sd = c[1:]
 201         sde = ce[1:]
 202         if not sd[0] or not sd[1]:
 203                 bomb("%s paths must be nonempty" % wh)
 204         dirsp = sd[0][-1]=='/'
 205         functions = "import errno\n"
 206         if dirsp != (sd[1][-1]=='/'):
 207                 bomb("% paths must agree about directoryness"
 208                         " (presence or absence of trailing /)" % wh)
 209         localfd = None
 210         deststdout = devnull_read
 211         srcstdin = devnull_read
 212         if not dirsp:
 213                 modestr = ''
 214                 if upp:
 215                         deststdout = file(sd[idst], 'w')
 216                 else:
 217                         srcstdin = file(sd[isrc], 'r')
 218                         status = os.fstat(srcstdin.fileno())
 219                         if status.st_mode & 0111: modestr = ',0777'
 220                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 221                                         1-upp, sde[iremote], not upp, modestr)
 222                 gobody += "     os.execvp('cat', ['cat'])\n"
 223                 localcmdl = ['cat']
 224         else:
 225                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 226                 if upp:
 227                         try: os.mkdir(sd[ilocal])
 228                         except OSError, oe:
 229                                 if oe.errno != errno.EEXIST: raise
 230                 else:
 231                         gobody += ("    try: os.mkdir(dir)\n"
 232                                 "       except OSError, oe:\n"
 233                                 "               if oe.errno != errno.EEXIST: raise\n")
 234                 gobody +=( "    os.chdir(dir)\n"
 235                         "       tarcmd = 'tar -f -'.split()\n")
 236                 localcmdl = 'tar -f -'.split()
 237                 taropts = [None, None]
 238                 taropts[isrc] = '-c .'
 239                 taropts[idst] = '-p -x --no-same-owner'
 240                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 241                 localcmdl += ['-C',sd[ilocal]]
 242                 localcmdl += taropts[ilocal].split()
 243                 gobody += "     os.execvp('tar', tarcmd)\n";
 244
 245         downcmdl = down_python_script(gobody, functions)
 246
 247         if upp: cmdls = (downcmdl, localcmdl)
 248         else: cmdls = (localcmdl, downcmdl)
 249
 250         debug(`["cmdls", `cmdls`]`)
 251         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 252
 253         subprocs = [None,None]
 254         debug(" +< %s" % string.join(cmdls[0]))
 255         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 256                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 257         debug(" +> %s" % string.join(cmdls[1]))
 258         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 259                         stdout=deststdout, preexec_fn=preexecfn)
 260         for sdn in [1,0]:
 261                 debug(" +"+"<>"[sdn]+"?");
 262                 status = subprocs[sdn].wait()
 263                 if status: bomb("%s %s failed, status %d" %
 264                         (wh, ['source','destination'][sdn], status))
 265
 266 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 267 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 268
 269 def command():
 270         sys.stdout.flush()
 271         ce = sys.stdin.readline()
 272         if not ce: bomb('end of file - caller quit?')
 273         ce = ce.rstrip().split()
 274         c = map(urllib.unquote, ce)
 275         if not c: bomb('empty commands are not permitted')
 276         debug('executing '+string.join(ce))
 277         try: f = globals()['cmd_'+c[0]]
 278         except KeyError: bomb("unknown command `%s'" % ce[0])
 279         r = f(c, ce)
 280         if not r: r = []
 281         r.insert(0, 'ok')
 282         print string.join(r)
 283
 284 def cleanup():
 285         global downtmp, cleaning
 286         debug("cleanup...");
 287         cleaning = True
 288         if downtmp: caller.hook_cleanup()
 289         cleaning = False
 290         downtmp = False
 291
 292 def error_cleanup():
 293         try:
 294                 ok = False
 295                 try:
 296                         cleanup()
 297                         ok = True
 298                 except Quit, q:
 299                         print >> sys.stderr, q.m
 300                 except:
 301                         print >> sys.stderr, "Unexpected cleanup error:"
 302                         traceback.print_exc()
 303                         print >> sys.stderr, ''
 304                 if not ok:
 305                         print >> sys.stderr, ("while cleaning up"
 306                                 " because of another error:")
 307         except:
 308                 pass
 309
 310 def prepare():
 311         global downtmp, cleaning
 312         downtmp = None
 313         signal_list = [ signal.SIGHUP, signal.SIGTERM,
 314                         signal.SIGINT, signal.SIGPIPE ]
 315         def sethandlers(f):
 316                 for signum in signal_list: signal.signal(signum, f)
 317         def handler(sig, *any):
 318                 sethandlers(signal.SIG_DFL)
 319                 cleanup()
 320                 os.kill(os.getpid(), sig)
 321         sethandlers(handler)
 322
 323 def mainloop():
 324         try:
 325                 while True: command()
 326         except Quit, q:
 327                 error_cleanup()
 328                 if q.m: print >> sys.stderr, q.m
 329                 sys.exit(q.ec)
 330         except:
 331                 error_cleanup()
 332                 print >> sys.stderr, "Unexpected error:"
 333                 traceback.print_exc()
 334                 sys.exit(16)
 335
 336 def main():
 337         debug("down = %s" % string.join(down))
 338         ok()
 339         prepare()
 340         mainloop()