virt-subproc/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import re as regexp
  33
  34 debuglevel = None
  35 progname = "<VirtSubproc>"
  36 devnull_read = file('/dev/null','r')
  37 caller = __main__
  38
  39 class Quit:
  40         def __init__(q,ec,m): q.ec = ec; q.m = m
  41
  42 def debug(m):
  43         if not debuglevel: return
  44         print >> sys.stderr, progname+": debug:", m
  45
  46 def bomb(m):
  47         raise Quit(12, progname+": failure: %s" % m)
  48
  49 def ok(): print 'ok'
  50
  51 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  52         if len(c) < 1+nargs:
  53                 bomb("too few arguments to command `%s'" % ce[0])
  54         if noptargs is not None and len(c) > 1+nargs+noptargs:
  55                 bomb("too many arguments to command `%s'" % ce[0])
  56
  57 def cmd_capabilities(c, ce):
  58         cmdnumargs(c, ce)
  59         return caller.hook_capabilities() + ['execute-debug']
  60
  61 def cmd_quit(c, ce):
  62         cmdnumargs(c, ce)
  63         raise Quit(0, '')
  64
  65 def cmd_close(c, ce):
  66         cmdnumargs(c, ce)
  67         if not downtmp: bomb("`close' when not open")
  68         cleanup()
  69
  70 def preexecfn():
  71         caller.hook_forked_inchild()
  72
  73 def execute_raw(what, instr, *popenargs, **popenargsk):
  74         debug(" ++ %s" % string.join(popenargs[0]))
  75         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  76         if instr is None: popenargsk['stdin'] = devnull_read
  77         (out, err) = sp.communicate(instr)
  78         if err: bomb("%s unexpectedly produced stderr output `%s'" %
  79                         (what, err))
  80         status = sp.wait()
  81         return (status, out)
  82
  83 def execute(cmd_string, cmd_list=[], downp=False, outp=False):
  84         cmdl = cmd_string.split()
  85
  86         if downp: perhaps_down = down
  87         else: downp = []
  88
  89         if outp: stdout = subprocess.PIPE
  90         else: stdout = None
  91
  92         cmd = cmdl + cmd_list
  93         if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)]
  94
  95         (status, out) = execute_raw(cmdl[0], None, cmd, stdout=stdout)
  96
  97         if status: bomb("%s%s failed (exit status %d)" %
  98                         ((downp and "(down) " or ""), cmdl[0], status))
  99
 100         if outp and out and out[-1]=='\n': out = out[:-1]
 101         return out
 102
 103 def cmd_open(c, ce):
 104         global downtmp
 105         cmdnumargs(c, ce)
 106         if downtmp: bomb("`open' when already open")
 107         downtmp = caller.hook_open()
 108         return [downtmp]
 109
 110 def cmd_revert(c, ce):
 111         cmdnumargs(c, ce)
 112         if not downtmp: bomb("`revert' when not open")
 113         if not 'revert' in caller.hook_capabilities():
 114                 bomb("`revert' when `revert' not advertised")
 115         caller.hook_revert()
 116
 117 def down_python_script(gobody, functions=''):
 118         # Many things are made much harder by the inability of
 119         # dchroot, ssh, et al, to cope without mangling the arguments.
 120         # So we run a sub-python on the testbed and feed it a script
 121         # on stdin.  The sub-python decodes the arguments.
 122
 123         script = (      "import urllib\n"
 124                         "import os\n"
 125                         "def setfd(fd,fnamee,write,mode=0666):\n"
 126                         "       fname = urllib.unquote(fnamee)\n"
 127                         "       if write: rw = os.O_WRONLY|os.O_CREAT\n"
 128                         "       else: rw = os.O_RDONLY\n"
 129                         "       nfd = os.open(fname, rw, mode)\n"
 130                         "       if fd >= 0: os.dup2(nfd,fd)\n"
 131                         + functions +
 132                         "def go():\n" )
 133         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 134                                 urllib.quote(downtmp)   )
 135         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 136         script += (     gobody +
 137                         "go()\n" )
 138
 139         debug("+P ...\n"+script)
 140
 141         scripte = urllib.quote(script)
 142         cmdl = down + ['python','-c',
 143                 "'import urllib; s = urllib.unquote(%s); exec s'" %
 144                         ('"%s"' % scripte)]
 145         return cmdl
 146
 147 def cmd_execute(c, ce):
 148         cmdnumargs(c, ce, 5, None)
 149         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 150         debug_g = None
 151         for kw in ce[6:]:
 152                 if kw.startswith('debug='):
 153                         if debug_g: bomb("multiple debug= in execute")
 154                         m = debug_re.match(kw)
 155                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 156                         debug_g = m.groups()
 157                 else: bomb("invalid execute kw arg `%s'" % kw)
 158
 159         gobody = "      import sys\n"
 160         stdout = None
 161         tfd = None
 162         if debug_g:
 163                 (tfd,hfd) = m.groups()
 164                 tfd = int(tfd)
 165                 gobody += "     os.dup2(1,%d)\n" % tfd
 166                 stdout = int(hfd)
 167         for ioe in range(3):
 168                 ioe_tfd = ioe
 169                 if ioe == tfd: ioe_tfd = -1
 170                 gobody += "     setfd(%d,'%s',%d)\n" % (
 171                         ioe_tfd, ce[ioe+2], ioe>0 )
 172         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 173         gobody += "     cmd = '%s'\n" % ce[1]
 174         gobody += ("    cmd = cmd.split(',')\n"
 175                 "       cmd = map(urllib.unquote, cmd)\n"
 176                 "       c0 = cmd[0]\n"
 177                 "       if '/' in c0:\n"
 178                 "               if not os.access(c0, os.X_OK):\n"
 179                 "                       status = os.stat(c0)\n"
 180                 "                       mode = status.st_mode | 0111\n"
 181                 "                       os.chmod(c0, mode)\n"
 182                 "       try: os.execvp(c0, cmd)\n"
 183                 "       except OSError, e:\n"
 184                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 185                 "                       (c0, os.strerror(e.errno)))\n"
 186                 "               os._exit(127)\n")
 187         cmdl = down_python_script(gobody)
 188
 189         (status, out) = execute_raw('sub-python', None, cmdl, stdout=stdout,
 190                                 stdin=devnull_read, stderr=subprocess.PIPE)
 191         if out: bomb("sub-python unexpected produced stdout"
 192                         " visible to us `%s'" % out)
 193         return [`status`]
 194
 195 def copyupdown(c, ce, upp):
 196         cmdnumargs(c, ce, 2)
 197         isrc = 0
 198         idst = 1
 199         ilocal = 0 + upp
 200         iremote = 1 - upp
 201         wh = ce[0]
 202         sd = c[1:]
 203         sde = ce[1:]
 204         if not sd[0] or not sd[1]:
 205                 bomb("%s paths must be nonempty" % wh)
 206         dirsp = sd[0][-1]=='/'
 207         functions = "import errno\n"
 208         if dirsp != (sd[1][-1]=='/'):
 209                 bomb("% paths must agree about directoryness"
 210                         " (presence or absence of trailing /)" % wh)
 211         localfd = None
 212         deststdout = devnull_read
 213         srcstdin = devnull_read
 214         if not dirsp:
 215                 modestr = ''
 216                 if upp:
 217                         deststdout = file(sd[idst], 'w')
 218                 else:
 219                         srcstdin = file(sd[isrc], 'r')
 220                         status = os.fstat(srcstdin.fileno())
 221                         if status.st_mode & 0111: modestr = ',0777'
 222                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 223                                         1-upp, sde[iremote], not upp, modestr)
 224                 gobody += "     os.execvp('cat', ['cat'])\n"
 225                 localcmdl = ['cat']
 226         else:
 227                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 228                 if upp:
 229                         try: os.mkdir(sd[ilocal])
 230                         except OSError, oe:
 231                                 if oe.errno != errno.EEXIST: raise
 232                 else:
 233                         gobody += ("    try: os.mkdir(dir)\n"
 234                                 "       except OSError, oe:\n"
 235                                 "               if oe.errno != errno.EEXIST: raise\n")
 236                 gobody +=( "    os.chdir(dir)\n"
 237                         "       tarcmd = 'tar -f -'.split()\n")
 238                 localcmdl = 'tar -f -'.split()
 239                 taropts = [None, None]
 240                 taropts[isrc] = '-c .'
 241                 taropts[idst] = '-p -x --no-same-owner'
 242                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 243                 localcmdl += ['-C',sd[ilocal]]
 244                 localcmdl += taropts[ilocal].split()
 245                 gobody += "     os.execvp('tar', tarcmd)\n";
 246
 247         downcmdl = down_python_script(gobody, functions)
 248
 249         if upp: cmdls = (downcmdl, localcmdl)
 250         else: cmdls = (localcmdl, downcmdl)
 251
 252         debug(`["cmdls", `cmdls`]`)
 253         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 254
 255         subprocs = [None,None]
 256         debug(" +< %s" % string.join(cmdls[0]))
 257         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 258                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 259         debug(" +> %s" % string.join(cmdls[1]))
 260         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 261                         stdout=deststdout, preexec_fn=preexecfn)
 262         for sdn in [1,0]:
 263                 debug(" +"+"<>"[sdn]+"?");
 264                 status = subprocs[sdn].wait()
 265                 if status: bomb("%s %s failed, status %d" %
 266                         (wh, ['source','destination'][sdn], status))
 267
 268 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 269 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 270
 271 def command():
 272         sys.stdout.flush()
 273         ce = sys.stdin.readline()
 274         if not ce: bomb('end of file - caller quit?')
 275         ce = ce.rstrip().split()
 276         c = map(urllib.unquote, ce)
 277         if not c: bomb('empty commands are not permitted')
 278         debug('executing '+string.join(ce))
 279         try: f = globals()['cmd_'+c[0]]
 280         except KeyError: bomb("unknown command `%s'" % ce[0])
 281         r = f(c, ce)
 282         if not r: r = []
 283         r.insert(0, 'ok')
 284         print string.join(r)
 285
 286 def cleanup():
 287         global downtmp, cleaning
 288         debug("cleanup...");
 289         cleaning = True
 290         if downtmp: caller.hook_cleanup()
 291         cleaning = False
 292         downtmp = False
 293
 294 def error_cleanup():
 295         try:
 296                 ok = False
 297                 try:
 298                         cleanup()
 299                         ok = True
 300                 except Quit, q:
 301                         print >> sys.stderr, q.m
 302                 except:
 303                         print >> sys.stderr, "Unexpected cleanup error:"
 304                         traceback.print_exc()
 305                         print >> sys.stderr, ''
 306                 if not ok:
 307                         print >> sys.stderr, ("while cleaning up"
 308                                 " because of another error:")
 309         except:
 310                 pass
 311
 312 def prepare():
 313         global downtmp, cleaning
 314         downtmp = None
 315         signal_list = [ signal.SIGHUP, signal.SIGTERM,
 316                         signal.SIGINT, signal.SIGPIPE ]
 317         def sethandlers(f):
 318                 for signum in signal_list: signal.signal(signum, f)
 319         def handler(sig, *any):
 320                 sethandlers(signal.SIG_DFL)
 321                 cleanup()
 322                 os.kill(os.getpid(), sig)
 323         sethandlers(handler)
 324
 325 def mainloop():
 326         try:
 327                 while True: command()
 328         except Quit, q:
 329                 error_cleanup()
 330                 if q.m: print >> sys.stderr, q.m
 331                 sys.exit(q.ec)
 332         except:
 333                 error_cleanup()
 334                 print >> sys.stderr, "Unexpected error:"
 335                 traceback.print_exc()
 336                 sys.exit(16)
 337
 338 def main():
 339         debug("down = %s" % string.join(down))
 340         ok()
 341         prepare()
 342         mainloop()