virt-subproc/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import re as regexp
  33
  34 debuglevel = None
  35 progname = "<VirtSubproc>"
  36 devnull_read = file('/dev/null','r')
  37 caller = __main__
  38
  39 class Quit:
  40         def __init__(q,ec,m): q.ec = ec; q.m = m
  41
  42 def debug(m):
  43         if not debuglevel: return
  44         print >> sys.stderr, progname+": debug:", m
  45
  46 def bomb(m):
  47         raise Quit(12, progname+": failure: %s" % m)
  48
  49 def ok(): print 'ok'
  50
  51 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  52         if len(c) < 1+nargs:
  53                 bomb("too few arguments to command `%s'" % ce[0])
  54         if noptargs is not None and len(c) > 1+nargs+noptargs:
  55                 bomb("too many arguments to command `%s'" % ce[0])
  56
  57 def cmd_capabilities(c, ce):
  58         cmdnumargs(c, ce)
  59         return caller.hook_capabilities() + ['execute-debug']
  60
  61 def cmd_quit(c, ce):
  62         cmdnumargs(c, ce)
  63         raise Quit(0, '')
  64
  65 def cmd_close(c, ce):
  66         cmdnumargs(c, ce)
  67         if not downtmp: bomb("`close' when not open")
  68         cleanup()
  69
  70 def preexecfn():
  71         caller.hook_forked_inchild()
  72
  73 def execute_raw(what, instr, *popenargs, **popenargsk):
  74         debug(" ++ %s" % string.join(popenargs[0]))
  75         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  76         if instr is None: popenargsk['stdin'] = devnull_read
  77         (out, err) = sp.communicate(instr)
  78         if err: bomb("%s unexpectedly produced stderr output `%s'" %
  79                         (what, err))
  80         status = sp.wait()
  81         return (status, out)
  82
  83 def execute(cmd_string, cmd_list=[], downp=False, outp=False):
  84         cmdl = cmd_string.split()
  85
  86         if downp: perhaps_down = down
  87         else: downp = []
  88
  89         if outp: stdout = subprocess.PIPE
  90         else: stdout = None
  91
  92         cmd = cmdl + cmd_list
  93         if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)]
  94
  95         (status, out) = execute_raw(cmdl[0], None, cmd, stdout=stdout)
  96
  97         if status: bomb("%s%s failed (exit status %d)" %
  98                         ((downp and "(down) " or ""), cmdl[0], status))
  99
 100         if outp and out and out[-1]=='\n': out = out[:-1]
 101         return out
 102
 103 def cmd_open(c, ce):
 104         global downtmp
 105         cmdnumargs(c, ce)
 106         if downtmp: bomb("`open' when already open")
 107         downtmp = caller.hook_open()
 108         return [downtmp]
 109
 110 def cmd_revert(c, ce):
 111         cmdnumargs(c, ce)
 112         if not downtmp: bomb("`revert' when not open")
 113         if not 'revert' in caller.hook_capabilities():
 114                 bomb("`revert' when `revert' not advertised")
 115         caller.hook_revert()
 116
 117 def down_python_script(gobody, functions=''):
 118         # Many things are made much harder by the inability of
 119         # dchroot, ssh, et al, to cope without mangling the arguments.
 120         # So we run a sub-python on the testbed and feed it a script
 121         # on stdin.  The sub-python decodes the arguments.
 122
 123         script = (      "import urllib\n"
 124                         "import os\n"
 125                         "def setfd(fd,fnamee,write,mode=0666):\n"
 126                         "       fname = urllib.unquote(fnamee)\n"
 127                         "       if write: rw = os.O_WRONLY|os.O_CREAT\n"
 128                         "       else: rw = os.O_RDONLY\n"
 129                         "       nfd = os.open(fname, rw, mode)\n"
 130                         "       if fd >= 0: os.dup2(nfd,fd)\n"
 131                         + functions +
 132                         "def go():\n" )
 133         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 134                                 urllib.quote(downtmp)   )
 135         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 136         script += (     gobody +
 137                         "go()\n" )
 138
 139         debug("+P ...\n"+script)
 140
 141         scripte = urllib.quote(script)
 142         cmdl = down + ['python','-c',
 143                 "'import urllib; s = urllib.unquote(%s); exec s'" %
 144                         ('"%s"' % scripte)]
 145         return cmdl
 146
 147 def cmd_execute(c, ce):
 148         cmdnumargs(c, ce, 5, None)
 149         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 150         debug_g = None
 151         envs = []
 152         for kw in ce[6:]:
 153                 if kw.startswith('debug='):
 154                         if debug_g: bomb("multiple debug= in execute")
 155                         m = debug_re.match(kw)
 156                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 157                         debug_g = m.groups()
 158                 elif kw.startswith('env='):
 159                         es = kw[4:]; eq = es.find('=')
 160                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 161                         envs.append((es[:eq], es[eq+1:]))
 162                 else: bomb("invalid execute kw arg `%s'" % kw)
 163
 164         gobody = "      import sys\n"
 165         stdout = None
 166         tfd = None
 167         if debug_g:
 168                 (tfd,hfd) = m.groups()
 169                 tfd = int(tfd)
 170                 gobody += "     os.dup2(1,%d)\n" % tfd
 171                 stdout = int(hfd)
 172         for ioe in range(3):
 173                 ioe_tfd = ioe
 174                 if ioe == tfd: ioe_tfd = -1
 175                 gobody += "     setfd(%d,'%s',%d)\n" % (
 176                         ioe_tfd, ce[ioe+2], ioe>0 )
 177         for e in envs:
 178                 gobody += ("    os.environ[urllib.unquote('%s')]"
 179                            " = urllib.unquote('%s')\n"
 180                                 % tuple(map(urllib.quote, e)))
 181         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 182         gobody += "     cmd = '%s'\n" % ce[1]
 183         gobody += ("    cmd = cmd.split(',')\n"
 184                 "       cmd = map(urllib.unquote, cmd)\n"
 185                 "       c0 = cmd[0]\n"
 186                 "       if '/' in c0:\n"
 187                 "               if not os.access(c0, os.X_OK):\n"
 188                 "                       status = os.stat(c0)\n"
 189                 "                       mode = status.st_mode | 0111\n"
 190                 "                       os.chmod(c0, mode)\n"
 191                 "       try: os.execvp(c0, cmd)\n"
 192                 "       except OSError, e:\n"
 193                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 194                 "                       (c0, os.strerror(e.errno)))\n"
 195                 "               os._exit(127)\n")
 196         cmdl = down_python_script(gobody)
 197
 198         (status, out) = execute_raw('sub-python', None, cmdl, stdout=stdout,
 199                                 stdin=devnull_read, stderr=subprocess.PIPE)
 200         if out: bomb("sub-python unexpected produced stdout"
 201                         " visible to us `%s'" % out)
 202         return [`status`]
 203
 204 def copyupdown(c, ce, upp):
 205         cmdnumargs(c, ce, 2)
 206         isrc = 0
 207         idst = 1
 208         ilocal = 0 + upp
 209         iremote = 1 - upp
 210         wh = ce[0]
 211         sd = c[1:]
 212         sde = ce[1:]
 213         if not sd[0] or not sd[1]:
 214                 bomb("%s paths must be nonempty" % wh)
 215         dirsp = sd[0][-1]=='/'
 216         functions = "import errno\n"
 217         if dirsp != (sd[1][-1]=='/'):
 218                 bomb("% paths must agree about directoryness"
 219                         " (presence or absence of trailing /)" % wh)
 220         localfd = None
 221         deststdout = devnull_read
 222         srcstdin = devnull_read
 223         if not dirsp:
 224                 modestr = ''
 225                 if upp:
 226                         deststdout = file(sd[idst], 'w')
 227                 else:
 228                         srcstdin = file(sd[isrc], 'r')
 229                         status = os.fstat(srcstdin.fileno())
 230                         if status.st_mode & 0111: modestr = ',0777'
 231                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 232                                         1-upp, sde[iremote], not upp, modestr)
 233                 gobody += "     os.execvp('cat', ['cat'])\n"
 234                 localcmdl = ['cat']
 235         else:
 236                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 237                 if upp:
 238                         try: os.mkdir(sd[ilocal])
 239                         except OSError, oe:
 240                                 if oe.errno != errno.EEXIST: raise
 241                 else:
 242                         gobody += ("    try: os.mkdir(dir)\n"
 243                                 "       except OSError, oe:\n"
 244                                 "               if oe.errno != errno.EEXIST: raise\n")
 245                 gobody +=( "    os.chdir(dir)\n"
 246                         "       tarcmd = 'tar -f -'.split()\n")
 247                 localcmdl = 'tar -f -'.split()
 248                 taropts = [None, None]
 249                 taropts[isrc] = '-c .'
 250                 taropts[idst] = '-p -x --no-same-owner'
 251                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 252                 localcmdl += ['-C',sd[ilocal]]
 253                 localcmdl += taropts[ilocal].split()
 254                 gobody += "     os.execvp('tar', tarcmd)\n";
 255
 256         downcmdl = down_python_script(gobody, functions)
 257
 258         if upp: cmdls = (downcmdl, localcmdl)
 259         else: cmdls = (localcmdl, downcmdl)
 260
 261         debug(`["cmdls", `cmdls`]`)
 262         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 263
 264         subprocs = [None,None]
 265         debug(" +< %s" % string.join(cmdls[0]))
 266         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 267                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 268         debug(" +> %s" % string.join(cmdls[1]))
 269         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 270                         stdout=deststdout, preexec_fn=preexecfn)
 271         for sdn in [1,0]:
 272                 debug(" +"+"<>"[sdn]+"?");
 273                 status = subprocs[sdn].wait()
 274                 if status: bomb("%s %s failed, status %d" %
 275                         (wh, ['source','destination'][sdn], status))
 276
 277 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 278 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 279
 280 def command():
 281         sys.stdout.flush()
 282         ce = sys.stdin.readline()
 283         if not ce: bomb('end of file - caller quit?')
 284         ce = ce.rstrip().split()
 285         c = map(urllib.unquote, ce)
 286         if not c: bomb('empty commands are not permitted')
 287         debug('executing '+string.join(ce))
 288         try: f = globals()['cmd_'+c[0]]
 289         except KeyError: bomb("unknown command `%s'" % ce[0])
 290         r = f(c, ce)
 291         if not r: r = []
 292         r.insert(0, 'ok')
 293         print string.join(r)
 294
 295 def cleanup():
 296         global downtmp, cleaning
 297         debug("cleanup...");
 298         cleaning = True
 299         if downtmp: caller.hook_cleanup()
 300         cleaning = False
 301         downtmp = False
 302
 303 def error_cleanup():
 304         try:
 305                 ok = False
 306                 try:
 307                         cleanup()
 308                         ok = True
 309                 except Quit, q:
 310                         print >> sys.stderr, q.m
 311                 except:
 312                         print >> sys.stderr, "Unexpected cleanup error:"
 313                         traceback.print_exc()
 314                         print >> sys.stderr, ''
 315                 if not ok:
 316                         print >> sys.stderr, ("while cleaning up"
 317                                 " because of another error:")
 318         except:
 319                 pass
 320
 321 def prepare():
 322         global downtmp, cleaning
 323         downtmp = None
 324         signal_list = [ signal.SIGHUP, signal.SIGTERM,
 325                         signal.SIGINT, signal.SIGPIPE ]
 326         def sethandlers(f):
 327                 for signum in signal_list: signal.signal(signum, f)
 328         def handler(sig, *any):
 329                 sethandlers(signal.SIG_DFL)
 330                 cleanup()
 331                 os.kill(os.getpid(), sig)
 332         sethandlers(handler)
 333
 334 def mainloop():
 335         try:
 336                 while True: command()
 337         except Quit, q:
 338                 error_cleanup()
 339                 if q.m: print >> sys.stderr, q.m
 340                 sys.exit(q.ec)
 341         except:
 342                 error_cleanup()
 343                 print >> sys.stderr, "Unexpected error:"
 344                 traceback.print_exc()
 345                 sys.exit(16)
 346
 347 def main():
 348         debug("down = %s" % string.join(down))
 349         ok()
 350         prepare()
 351         mainloop()