virt-subproc/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import re as regexp
  33
  34 debuglevel = None
  35 progname = "<VirtSubproc>"
  36 devnull_read = file('/dev/null','r')
  37 caller = __main__
  38 copy_timeout = 300
  39
  40 class Quit:
  41         def __init__(q,ec,m): q.ec = ec; q.m = m
  42
  43 class Timeout: pass
  44 def alarm_handler(*a): raise Timeout()
  45 def timeout_start(to): signal.alarm(to)
  46 def timeout_stop(): signal.alarm(0)
  47
  48 class FailedCmd:
  49         def __init__(fc,e): fc.e = e
  50
  51 def debug(m):
  52         if not debuglevel: return
  53         print >> sys.stderr, progname+": debug:", m
  54
  55 def bomb(m):
  56         raise Quit(12, progname+": failure: %s" % m)
  57
  58 def ok(): print 'ok'
  59
  60 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  61         if len(c) < 1+nargs:
  62                 bomb("too few arguments to command `%s'" % ce[0])
  63         if noptargs is not None and len(c) > 1+nargs+noptargs:
  64                 bomb("too many arguments to command `%s'" % ce[0])
  65
  66 def cmd_capabilities(c, ce):
  67         cmdnumargs(c, ce)
  68         return caller.hook_capabilities() + ['execute-debug']
  69
  70 def cmd_quit(c, ce):
  71         cmdnumargs(c, ce)
  72         raise Quit(0, '')
  73
  74 def cmd_close(c, ce):
  75         cmdnumargs(c, ce)
  76         if not downtmp: bomb("`close' when not open")
  77         cleanup()
  78
  79 def preexecfn():
  80         caller.hook_forked_inchild()
  81
  82 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  83         debug(" ++ %s" % string.join(popenargs[0]))
  84         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
  85         if instr is None: popenargsk['stdin'] = devnull_read
  86         timeout_start(timeout)
  87         (out, err) = sp.communicate(instr)
  88         timeout_stop()
  89         if err: bomb("%s unexpectedly produced stderr output `%s'" %
  90                         (what, err))
  91         status = sp.wait()
  92         return (status, out)
  93
  94 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
  95         cmdl = cmd_string.split()
  96
  97         if downp: perhaps_down = down
  98         else: downp = []
  99
 100         if outp: stdout = subprocess.PIPE
 101         else: stdout = None
 102
 103         cmd = cmdl + cmd_list
 104         if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)]
 105
 106         (status, out) = execute_raw(cmdl[0], None, timeout,
 107                                 cmd, stdout=stdout)
 108
 109         if status: bomb("%s%s failed (exit status %d)" %
 110                         ((downp and "(down) " or ""), cmdl[0], status))
 111
 112         if outp and out and out[-1]=='\n': out = out[:-1]
 113         return out
 114
 115 def cmd_open(c, ce):
 116         global downtmp
 117         cmdnumargs(c, ce)
 118         if downtmp: bomb("`open' when already open")
 119         downtmp = caller.hook_open()
 120         return [downtmp]
 121
 122 def cmd_revert(c, ce):
 123         cmdnumargs(c, ce)
 124         if not downtmp: bomb("`revert' when not open")
 125         if not 'revert' in caller.hook_capabilities():
 126                 bomb("`revert' when `revert' not advertised")
 127         caller.hook_revert()
 128
 129 def down_python_script(gobody, functions=''):
 130         # Many things are made much harder by the inability of
 131         # dchroot, ssh, et al, to cope without mangling the arguments.
 132         # So we run a sub-python on the testbed and feed it a script
 133         # on stdin.  The sub-python decodes the arguments.
 134
 135         script = (      "import urllib\n"
 136                         "import os\n"
 137                         "def setfd(fd,fnamee,write,mode=0666):\n"
 138                         "       fname = urllib.unquote(fnamee)\n"
 139                         "       if write: rw = os.O_WRONLY|os.O_CREAT\n"
 140                         "       else: rw = os.O_RDONLY\n"
 141                         "       nfd = os.open(fname, rw, mode)\n"
 142                         "       if fd >= 0: os.dup2(nfd,fd)\n"
 143                         + functions +
 144                         "def go():\n" )
 145         script += (     "       os.environ['TMPDIR']= urllib.unquote('%s')\n" %
 146                                 urllib.quote(downtmp)   )
 147         script += (     "       os.chdir(os.environ['TMPDIR'])\n" )
 148         script += (     gobody +
 149                         "go()\n" )
 150
 151         debug("+P ...\n"+script)
 152
 153         scripte = urllib.quote(script)
 154         cmdl = down + ['python','-c',
 155                 "'import urllib; s = urllib.unquote(%s); exec s'" %
 156                         ('"%s"' % scripte)]
 157         return cmdl
 158
 159 def cmd_execute(c, ce):
 160         cmdnumargs(c, ce, 5, None)
 161         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 162         debug_g = None
 163         timeout = 0
 164         envs = []
 165         for kw in ce[6:]:
 166                 if kw.startswith('debug='):
 167                         if debug_g: bomb("multiple debug= in execute")
 168                         m = debug_re.match(kw)
 169                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 170                         debug_g = m.groups()
 171                 elif kw.startswith('timeout='):
 172                         try: timeout = int(kw[8:],0)
 173                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 174                 elif kw.startswith('env='):
 175                         es = kw[4:]; eq = es.find('=')
 176                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 177                         envs.append((es[:eq], es[eq+1:]))
 178                 else: bomb("invalid execute kw arg `%s'" % kw)
 179
 180         gobody = "      import sys\n"
 181         stdout = None
 182         tfd = None
 183         if debug_g:
 184                 (tfd,hfd) = m.groups()
 185                 tfd = int(tfd)
 186                 gobody += "     os.dup2(1,%d)\n" % tfd
 187                 stdout = int(hfd)
 188         for ioe in range(3):
 189                 ioe_tfd = ioe
 190                 if ioe == tfd: ioe_tfd = -1
 191                 gobody += "     setfd(%d,'%s',%d)\n" % (
 192                         ioe_tfd, ce[ioe+2], ioe>0 )
 193         for e in envs:
 194                 gobody += ("    os.environ[urllib.unquote('%s')]"
 195                            " = urllib.unquote('%s')\n"
 196                                 % tuple(map(urllib.quote, e)))
 197         gobody += "     os.chdir(urllib.unquote('" + ce[5] +"'))\n"
 198         gobody += "     cmd = '%s'\n" % ce[1]
 199         gobody += ("    cmd = cmd.split(',')\n"
 200                 "       cmd = map(urllib.unquote, cmd)\n"
 201                 "       c0 = cmd[0]\n"
 202                 "       if '/' in c0:\n"
 203                 "               if not os.access(c0, os.X_OK):\n"
 204                 "                       status = os.stat(c0)\n"
 205                 "                       mode = status.st_mode | 0111\n"
 206                 "                       os.chmod(c0, mode)\n"
 207                 "       try: os.execvp(c0, cmd)\n"
 208                 "       except OSError, e:\n"
 209                 "               print >>sys.stderr, \"%s: %s\" % (\n"
 210                 "                       (c0, os.strerror(e.errno)))\n"
 211                 "               os._exit(127)\n")
 212         cmdl = down_python_script(gobody)
 213
 214         try:
 215                 (status, out) = execute_raw('sub-python', None, timeout,
 216                                 cmdl, stdout=stdout, stdin=devnull_read,
 217                                 stderr=subprocess.PIPE)
 218         except Timeout:
 219                 raise FailedCmd(['timeout'])
 220
 221         if out: bomb("sub-python unexpected produced stdout"
 222                         " visible to us `%s'" % out)
 223         return [`status`]
 224
 225 def copyupdown(c, ce, upp):
 226         cmdnumargs(c, ce, 2)
 227         isrc = 0
 228         idst = 1
 229         ilocal = 0 + upp
 230         iremote = 1 - upp
 231         wh = ce[0]
 232         sd = c[1:]
 233         sde = ce[1:]
 234         if not sd[0] or not sd[1]:
 235                 bomb("%s paths must be nonempty" % wh)
 236         dirsp = sd[0][-1]=='/'
 237         functions = "import errno\n"
 238         if dirsp != (sd[1][-1]=='/'):
 239                 bomb("% paths must agree about directoryness"
 240                         " (presence or absence of trailing /)" % wh)
 241         localfd = None
 242         deststdout = devnull_read
 243         srcstdin = devnull_read
 244         if not dirsp:
 245                 modestr = ''
 246                 if upp:
 247                         deststdout = file(sd[idst], 'w')
 248                 else:
 249                         srcstdin = file(sd[isrc], 'r')
 250                         status = os.fstat(srcstdin.fileno())
 251                         if status.st_mode & 0111: modestr = ',0777'
 252                 gobody = "      setfd(%s,'%s',%s%s)\n" % (
 253                                         1-upp, sde[iremote], not upp, modestr)
 254                 gobody += "     os.execvp('cat', ['cat'])\n"
 255                 localcmdl = ['cat']
 256         else:
 257                 gobody = "      dir = urllib.unquote('%s')\n" % sde[iremote]
 258                 if upp:
 259                         try: os.mkdir(sd[ilocal])
 260                         except OSError, oe:
 261                                 if oe.errno != errno.EEXIST: raise
 262                 else:
 263                         gobody += ("    try: os.mkdir(dir)\n"
 264                                 "       except OSError, oe:\n"
 265                                 "               if oe.errno != errno.EEXIST: raise\n")
 266                 gobody +=( "    os.chdir(dir)\n"
 267                         "       tarcmd = 'tar -f -'.split()\n")
 268                 localcmdl = 'tar -f -'.split()
 269                 taropts = [None, None]
 270                 taropts[isrc] = '-c .'
 271                 taropts[idst] = '-p -x --no-same-owner'
 272                 gobody += "     tarcmd += '%s'.split()\n" % taropts[iremote]
 273                 localcmdl += ['-C',sd[ilocal]]
 274                 localcmdl += taropts[ilocal].split()
 275                 gobody += "     os.execvp('tar', tarcmd)\n";
 276
 277         downcmdl = down_python_script(gobody, functions)
 278
 279         if upp: cmdls = (downcmdl, localcmdl)
 280         else: cmdls = (localcmdl, downcmdl)
 281
 282         debug(`["cmdls", `cmdls`]`)
 283         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 284
 285         subprocs = [None,None]
 286         debug(" +< %s" % string.join(cmdls[0]))
 287         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 288                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 289         debug(" +> %s" % string.join(cmdls[1]))
 290         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 291                         stdout=deststdout, preexec_fn=preexecfn)
 292         timeout_start(copy_timeout)
 293         for sdn in [1,0]:
 294                 debug(" +"+"<>"[sdn]+"?");
 295                 status = subprocs[sdn].wait()
 296                 if status:
 297                         timeout_stop()
 298                         bomb("%s %s failed, status %d" %
 299                                 (wh, ['source','destination'][sdn], status))
 300         timeout_stop()
 301
 302 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 303 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 304
 305 def command():
 306         sys.stdout.flush()
 307         ce = sys.stdin.readline()
 308         if not ce: bomb('end of file - caller quit?')
 309         ce = ce.rstrip().split()
 310         c = map(urllib.unquote, ce)
 311         if not c: bomb('empty commands are not permitted')
 312         debug('executing '+string.join(ce))
 313         try: f = globals()['cmd_'+c[0]]
 314         except KeyError: bomb("unknown command `%s'" % ce[0])
 315         try:
 316                 r = f(c, ce)
 317                 if not r: r = []
 318                 r.insert(0, 'ok')
 319         except FailedCmd, fc:
 320                 r = fc.e
 321         print string.join(r)
 322
 323 def cleanup():
 324         global downtmp, cleaning
 325         debug("cleanup...");
 326         cleaning = True
 327         if downtmp: caller.hook_cleanup()
 328         cleaning = False
 329         downtmp = False
 330
 331 def error_cleanup():
 332         try:
 333                 ok = False
 334                 try:
 335                         cleanup()
 336                         ok = True
 337                 except Quit, q:
 338                         print >> sys.stderr, q.m
 339                 except:
 340                         print >> sys.stderr, "Unexpected cleanup error:"
 341                         traceback.print_exc()
 342                         print >> sys.stderr, ''
 343                 if not ok:
 344                         print >> sys.stderr, ("while cleaning up"
 345                                 " because of another error:")
 346         except:
 347                 pass
 348
 349 def prepare():
 350         global downtmp, cleaning
 351         downtmp = None
 352         signal_list = [ signal.SIGHUP, signal.SIGTERM,
 353                         signal.SIGINT, signal.SIGPIPE ]
 354         def sethandlers(f):
 355                 for signum in signal_list: signal.signal(signum, f)
 356         def handler(sig, *any):
 357                 sethandlers(signal.SIG_DFL)
 358                 cleanup()
 359                 os.kill(os.getpid(), sig)
 360         sethandlers(handler)
 361
 362 def mainloop():
 363         try:
 364                 while True: command()
 365         except Quit, q:
 366                 error_cleanup()
 367                 if q.m: print >> sys.stderr, q.m
 368                 sys.exit(q.ec)
 369         except:
 370                 error_cleanup()
 371                 print >> sys.stderr, "Unexpected error:"
 372                 traceback.print_exc()
 373                 sys.exit(16)
 374
 375 def main():
 376         signal.signal(signal.SIGALRM, alarm_handler)
 377         debug("down = %s" % string.join(down))
 378         ok()
 379         prepare()
 380         mainloop()