virt-subproc/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import re as regexp
  33
  34 debuglevel = None
  35 progname = "<VirtSubproc>"
  36 devnull_read = file('/dev/null','r')
  37 caller = __main__
  38
  39 class Quit:
  40         def __init__(q,ec,m): q.ec = ec; q.m = m
  41
  42 def debug(m):
  43         if not debuglevel: return
  44         print >> sys.stderr, progname+": debug:", m
  45
  46 def bomb(m):
  47         raise Quit(12, progname+": failure: %s" % m)
  48
  49 def ok(): print 'ok'
  50
  51 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  52         if len(c) < 1+nargs:
  53                 bomb("too few arguments to command `%s'" % ce[0])
  54         if noptargs is not None and len(c) > 1+nargs+noptargs:
  55                 bomb("too many arguments to command `%s'" % ce[0])
  56
  57 def cmd_capabilities(c, ce):
  58         cmdnumargs(c, ce)
  59         return caller.hook_capabilities()
  60
  61 def cmd_quit(c, ce):
  62         cmdnumargs(c, ce)
  63         raise Quit(0, '')
  64
  65 def cmd_close(c, ce):
  66         cmdnumargs(c, ce)
  67         if not downtmp: bomb("`close' when not open")
  68         cleanup()
  69
  70 def preexecfn():
  71         caller.hook_forked_inchild()
  72
  73 def execute_raw(what, instr, *popenargs, **popenargsk):
  74         debug(" ++ %s" % string.join(popenargs[0]))
  75         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  76         if instr is None: popenargsk['stdin'] = devnull_read
  77         (out, err) = sp.communicate(instr)
  78         if err: bomb("%s unexpectedly produced stderr output `%s'" %
  79                         (what, err))
  80         status = sp.wait()
  81         return (status, out)
  82
  83 def execute(cmd_string, cmd_list=[], downp=False, outp=False):
  84         cmdl = cmd_string.split()
  85
  86         if downp: perhaps_down = down
  87         else: downp = []
  88
  89         if outp: stdout = subprocess.PIPE
  90         else: stdout = None
  91
  92         cmd = cmdl + cmd_list
  93         if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)]
  94
  95         (status, out) = execute_raw(cmdl[0], None, cmd, stdout=stdout)
  96
  97         if status: bomb("%s%s failed (exit status %d)" %
  98                         ((downp and "(down) " or ""), cmdl[0], status))
  99
 100         if outp and out and out[-1]=='\n': out = out[:-1]
 101         return out
 102
 103 def cmd_open(c, ce):
 104         global downtmp
 105         cmdnumargs(c, ce)
 106         if downtmp: bomb("`open' when already open")
 107         downtmp = caller.hook_open()
 108         return [downtmp]
 109
 110 def cmd_revert(c, ce):
 111         cmdnumargs(c, ce)
 112         if not downtmp: bomb("`revert' when not open")
 113         if not 'revert' in caller.hook_capabilities():
 114                 bomb("`revert' when `revert' not advertised")
 115         caller.hook_revert()
 116
 117 def down_python_script(gobody, functions=''):
 118         # Many things are made much harder by the inability of
 119         # dchroot, ssh, et al, to cope without mangling the arguments.
 120         # So we run a sub-python on the testbed and feed it a script
 121         # on stdin.  The sub-python decodes the arguments.
 122
 123         script = (      "import urllib\n"
 124                         "import os\n"
 125                         "def setfd(fd,fnamee,write,mode=0666):\n"
 126                         "       fname = urllib.unquote(fnamee)\n"
 127                         "       if write: rw = os.O_WRONLY|os.O_CREAT\n"
 128                         "       else: rw = os.O_RDONLY\n"
 129                         "       nfd = os.open(fname, rw, mode)\n"
 130                         "       os.dup2(nfd,fd)\n"
 131                         + functions +
 132                         "def go():\n" )
 133         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 134                                 urllib.quote(downtmp)   )
 135         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 136         script += (     gobody +
 137                         "go()\n" )
 138
 139         debug("+P ...\n"+script)
 140
 141         scripte = urllib.quote(script)
 142         cmdl = down + ['python','-c',
 143                 "'import urllib; s = urllib.unquote(%s); exec s'" %
 144                         ('"%s"' % scripte)]
 145         return cmdl
 146
 147 def cmd_execute(c, ce):
 148         cmdnumargs(c, ce, 5, None)
 149         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 150         debug_g = None
 151         for kw in ce[6:]:
 152                 if kw.startswith('debug='):
 153                         if debug_g: bomb("multiple debug= in execute")
 154                         m = debug_re.match(kw)
 155                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 156                         debug_g = m.groups()
 157                 else: bomb("invalid execute kw arg `%s'" % kw)
 158
 159         gobody = "      import sys\n"
 160         stdout = None
 161         if debug_g:
 162                 (tfd,hfd) = m.groups()
 163                 gobody += "     os.dup2(1,%d)\n" % int(tfd)
 164                 stdout = int(hfd)
 165         for ioe in range(3):
 166                 gobody += "     setfd(%d,'%s',%d)\n" % (
 167                         ioe, ce[ioe+2], ioe>0 )
 168         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 169         gobody += "     cmd = '%s'\n" % ce[1]
 170         gobody += ("    cmd = cmd.split(',')\n"
 171                 "       cmd = map(urllib.unquote, cmd)\n"
 172                 "       c0 = cmd[0]\n"
 173                 "       if '/' in c0:\n"
 174                 "               if not os.access(c0, os.X_OK):\n"
 175                 "                       status = os.stat(c0)\n"
 176                 "                       mode = status.st_mode | 0111\n"
 177                 "                       os.chmod(c0, mode)\n"
 178                 "       try: os.execvp(c0, cmd)\n"
 179                 "       except OSError, e:\n"
 180                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 181                 "                       (c0, os.strerror(e.errno)))\n"
 182                 "               os._exit(127)\n")
 183         cmdl = down_python_script(gobody)
 184
 185         (status, out) = execute_raw('sub-python', None, cmdl, stdout=stdout,
 186                                 stdin=devnull_read, stderr=subprocess.PIPE)
 187         if out: bomb("sub-python unexpected produced stdout"
 188                         " visible to us `%s'" % out)
 189         return [`status`]
 190
 191 def copyupdown(c, ce, upp):
 192         cmdnumargs(c, ce, 2)
 193         isrc = 0
 194         idst = 1
 195         ilocal = 0 + upp
 196         iremote = 1 - upp
 197         wh = ce[0]
 198         sd = c[1:]
 199         sde = ce[1:]
 200         if not sd[0] or not sd[1]:
 201                 bomb("%s paths must be nonempty" % wh)
 202         dirsp = sd[0][-1]=='/'
 203         functions = "import errno\n"
 204         if dirsp != (sd[1][-1]=='/'):
 205                 bomb("% paths must agree about directoryness"
 206                         " (presence or absence of trailing /)" % wh)
 207         localfd = None
 208         deststdout = devnull_read
 209         srcstdin = devnull_read
 210         if not dirsp:
 211                 modestr = ''
 212                 if upp:
 213                         deststdout = file(sd[idst], 'w')
 214                 else:
 215                         srcstdin = file(sd[isrc], 'r')
 216                         status = os.fstat(srcstdin.fileno())
 217                         if status.st_mode & 0111: modestr = ',0777'
 218                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 219                                         1-upp, sde[iremote], not upp, modestr)
 220                 gobody += "     os.execvp('cat', ['cat'])\n"
 221                 localcmdl = ['cat']
 222         else:
 223                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 224                 if upp:
 225                         try: os.mkdir(sd[ilocal])
 226                         except OSError, oe:
 227                                 if oe.errno != errno.EEXIST: raise
 228                 else:
 229                         gobody += ("    try: os.mkdir(dir)\n"
 230                                 "       except OSError, oe:\n"
 231                                 "               if oe.errno != errno.EEXIST: raise\n")
 232                 gobody +=( "    os.chdir(dir)\n"
 233                         "       tarcmd = 'tar -f -'.split()\n")
 234                 localcmdl = 'tar -f -'.split()
 235                 taropts = [None, None]
 236                 taropts[isrc] = '-c .'
 237                 taropts[idst] = '-p -x --no-same-owner'
 238                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 239                 localcmdl += ['-C',sd[ilocal]]
 240                 localcmdl += taropts[ilocal].split()
 241                 gobody += "     os.execvp('tar', tarcmd)\n";
 242
 243         downcmdl = down_python_script(gobody, functions)
 244
 245         if upp: cmdls = (downcmdl, localcmdl)
 246         else: cmdls = (localcmdl, downcmdl)
 247
 248         debug(`["cmdls", `cmdls`]`)
 249         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 250
 251         subprocs = [None,None]
 252         debug(" +< %s" % string.join(cmdls[0]))
 253         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 254                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 255         debug(" +> %s" % string.join(cmdls[1]))
 256         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 257                         stdout=deststdout, preexec_fn=preexecfn)
 258         for sdn in [1,0]:
 259                 debug(" +"+"<>"[sdn]+"?");
 260                 status = subprocs[sdn].wait()
 261                 if status: bomb("%s %s failed, status %d" %
 262                         (wh, ['source','destination'][sdn], status))
 263
 264 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 265 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 266
 267 def command():
 268         sys.stdout.flush()
 269         ce = sys.stdin.readline()
 270         if not ce: bomb('end of file - caller quit?')
 271         ce = ce.rstrip().split()
 272         c = map(urllib.unquote, ce)
 273         if not c: bomb('empty commands are not permitted')
 274         debug('executing '+string.join(ce))
 275         try: f = globals()['cmd_'+c[0]]
 276         except KeyError: bomb("unknown command `%s'" % ce[0])
 277         r = f(c, ce)
 278         if not r: r = []
 279         r.insert(0, 'ok')
 280         print string.join(r)
 281
 282 def cleanup():
 283         global downtmp, cleaning
 284         debug("cleanup...");
 285         cleaning = True
 286         if downtmp: caller.hook_cleanup()
 287         cleaning = False
 288         downtmp = False
 289
 290 def error_cleanup():
 291         try:
 292                 ok = False
 293                 try:
 294                         cleanup()
 295                         ok = True
 296                 except Quit, q:
 297                         print >> sys.stderr, q.m
 298                 except:
 299                         print >> sys.stderr, "Unexpected cleanup error:"
 300                         traceback.print_exc()
 301                         print >> sys.stderr, ''
 302                 if not ok:
 303                         print >> sys.stderr, ("while cleaning up"
 304                                 " because of another error:")
 305         except:
 306                 pass
 307
 308 def prepare():
 309         global downtmp, cleaning
 310         downtmp = None
 311         signal_list = [ signal.SIGHUP, signal.SIGTERM,
 312                         signal.SIGINT, signal.SIGPIPE ]
 313         def sethandlers(f):
 314                 for signum in signal_list: signal.signal(signum, f)
 315         def handler(sig, *any):
 316                 sethandlers(signal.SIG_DFL)
 317                 cleanup()
 318                 os.kill(os.getpid(), sig)
 319         sethandlers(handler)
 320
 321 def mainloop():
 322         try:
 323                 while True: command()
 324         except Quit, q:
 325                 error_cleanup()
 326                 if q.m: print >> sys.stderr, q.m
 327                 sys.exit(q.ec)
 328         except:
 329                 error_cleanup()
 330                 print >> sys.stderr, "Unexpected error:"
 331                 traceback.print_exc()
 332                 sys.exit(16)
 333
 334 def main():
 335         debug("down = %s" % string.join(down))
 336         ok()
 337         prepare()
 338         mainloop()