lib/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import errno
  33 import re as regexp
  34
  35 from Autopkgtest import *
  36
  37 debuglevel = None
  38 progname = "<VirtSubproc>"
  39 devnull_read = file('/dev/null','r')
  40 caller = __main__
  41 copy_timeout = 300
  42
  43 class Quit:
  44         def __init__(q,ec,m): q.ec = ec; q.m = m
  45
  46 class Timeout: pass
  47 def alarm_handler(*a): raise Timeout()
  48 def timeout_start(to): signal.alarm(to)
  49 def timeout_stop(): signal.alarm(0)
  50
  51 class FailedCmd:
  52         def __init__(fc,e): fc.e = e
  53
  54 def debug(m):
  55         if not debuglevel: return
  56         print >> sys.stderr, progname+": debug:", m
  57
  58 def bomb(m):
  59         raise Quit(12, progname+": failure: %s" % m)
  60
  61 def ok(): print 'ok'
  62
  63 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  64         if len(c) < 1+nargs:
  65                 bomb("too few arguments to command `%s'" % ce[0])
  66         if noptargs is not None and len(c) > 1+nargs+noptargs:
  67                 bomb("too many arguments to command `%s'" % ce[0])
  68
  69 def cmd_capabilities(c, ce):
  70         cmdnumargs(c, ce)
  71         return caller.hook_capabilities() + ['execute-debug']
  72
  73 def cmd_quit(c, ce):
  74         cmdnumargs(c, ce)
  75         raise Quit(0, '')
  76
  77 def cmd_close(c, ce):
  78         cmdnumargs(c, ce)
  79         if not downtmp: bomb("`close' when not open")
  80         cleanup()
  81
  82 def cmd_print_auxverb_command(c, ce): return print_command('auxverb', c, ce)
  83 def cmd_print_shstring_command(c, ce): return print_command('shstring', c, ce)
  84
  85 def print_command(which, c, ce):
  86         global downs
  87         cmdnumargs(c, ce)
  88         if not downtmp: bomb("`print-%s-command' when not open" % which)
  89         cl = downs[which]
  90         if not len(cl):
  91                 cl = ['sh','-c','exec "$@"','x'] + cl
  92         return [','.join(map(urllib.quote, cl))]
  93
  94 def preexecfn():
  95         caller.hook_forked_inchild()
  96
  97 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  98         debug(" ++ %s" % string.join(popenargs[0]))
  99         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
 100         if instr is None: popenargsk['stdin'] = devnull_read
 101         timeout_start(timeout)
 102         (out, err) = sp.communicate(instr)
 103         timeout_stop()
 104         if err: bomb("%s unexpectedly produced stderr output `%s'" %
 105                         (what, err))
 106         status = sp.wait()
 107         return (status, out)
 108
 109 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
 110         cmdl = cmd_string.split()
 111
 112         if downp: perhaps_down = downs['auxverb']
 113         else: perhaps_down = []
 114
 115         if outp: stdout = subprocess.PIPE
 116         else: stdout = None
 117
 118         cmd = cmdl + cmd_list
 119         if len(perhaps_down): cmd = perhaps_down + cmd
 120
 121         (status, out) = execute_raw(cmdl[0], None, timeout,
 122                                 cmd, stdout=stdout)
 123
 124         if status: bomb("%s%s failed (exit status %d)" %
 125                         ((downp and "(down) " or ""), cmdl[0], status))
 126
 127         if outp and out and out[-1]=='\n': out = out[:-1]
 128         return out
 129
 130 def cmd_open(c, ce):
 131         global downtmp
 132         cmdnumargs(c, ce)
 133         if downtmp: bomb("`open' when already open")
 134         caller.hook_open()
 135         opened1()
 136         downtmp = caller.hook_downtmp()
 137         return opened2()
 138
 139 def downtmp_mktemp():
 140         global downtmp
 141         return execute('mktemp -t -d', downp=True, outp=True)
 142
 143 def downtmp_remove():
 144         global downtmp
 145         execute('rm -rf --', [downtmp], downp=True)
 146
 147 perl_quote_re = regexp.compile('[^-+=_.,;:() 0-9a-zA-Z]')
 148 def perl_quote_1chargroup(m): return '\\x%02x' % ord(m.group(0))
 149 def perl_quote(s): return '"'+perl_quote_re.sub(perl_quote_1chargroup, s)+'"'
 150
 151 def opened1():
 152         global down, downkind, downs
 153         debug("downkind = %s, down = %s" % (downkind, `down`))
 154         if downkind == 'auxverb':
 155                 downs = { 'auxverb': down,
 156                           'shstring': down + ['sh','-c'] }
 157         elif downkind == 'shstring':
 158                 downs = { 'shstring': down,
 159                           'auxverb': ['perl','-e','''
 160                         @cmd=('''+(','.join(map(perl_quote,down)))+''');
 161                         my $shstring = pop @ARGV;
 162                         s/'/'\\\\''/g foreach @ARGV;
 163                         push @cmd, "'$_'" foreach @ARGV;
 164                         my $argv0=$cmd[0];
 165                         exec $argv0 @cmd;
 166                         die "$argv0: $!";
 167                 '''] }
 168         debug("downs = %s" % `downs`)
 169
 170 def opened2():
 171         global downtmp, downs
 172         debug("downtmp = %s" % (downtmp))
 173         return [downtmp]
 174
 175 def cmd_revert(c, ce):
 176         global downtmp
 177         cmdnumargs(c, ce)
 178         if not downtmp: bomb("`revert' when not open")
 179         if not 'revert' in caller.hook_capabilities():
 180                 bomb("`revert' when `revert' not advertised")
 181         caller.hook_revert()
 182         opened1()
 183         downtmp = caller.hook_downtmp()
 184         return opened2()
 185
 186 def cmd_execute(c, ce):
 187         cmdnumargs(c, ce, 5, None)
 188         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 189         debug_g = None
 190         timeout = 0
 191         envs = []
 192         for kw in ce[6:]:
 193                 if kw.startswith('debug='):
 194                         if debug_g: bomb("multiple debug= in execute")
 195                         m = debug_re.match(kw)
 196                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 197                         debug_g = m.groups()
 198                 elif kw.startswith('timeout='):
 199                         try: timeout = int(kw[8:],0)
 200                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 201                 elif kw.startswith('env='):
 202                         es = kw[4:]; eq = es.find('=')
 203                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 204                         envs.append((es[:eq], es[eq+1:]))
 205                 else: bomb("invalid execute kw arg `%s'" % kw)
 206
 207         rune = 'set -e; exec '
 208
 209         stdout = None
 210         tfd = None
 211         if debug_g:
 212                 rune += " 3>&1"
 213
 214         for ioe in range(3):
 215                 rune += " %d%s%s" % (ioe, '<>'[ioe>0],
 216                                         shellquote_arg(ce[ioe+2]))
 217         if debug_g:
 218                 (tfd,hfd) = m.groups()
 219                 tfd = int(tfd)
 220                 rune += " %d>&3 3>&-" % tfd
 221                 stdout = int(hfd)
 222
 223         rune += '; '
 224
 225         rune += 'cd %s; ' % shellquote_arg(ce[5])
 226
 227         for e in envs:
 228                 (en, ev) = map(urllib.unquote,e)
 229                 rune += "%s=%s " % (en, shellquote_arg(ev))
 230
 231         rune += 'exec ' + shellquote_cmdl(map(urllib.unquote, ce[1].split(',')))
 232
 233         cmdl = downs['shstring'] + [rune]
 234
 235         stdout_copy = None
 236         try:
 237                 if type(stdout) == type(2):
 238                         stdout_copy = os.dup(stdout)
 239                 try:
 240                         (status, out) = execute_raw('target-cmd', None,
 241                                 timeout, cmdl, stdout=stdout_copy,
 242                                 stdin=devnull_read, stderr=subprocess.PIPE)
 243                 except Timeout:
 244                         raise FailedCmd(['timeout'])
 245         finally:
 246                 if stdout_copy is not None: os.close(stdout_copy)
 247
 248         if out: bomb("target command unexpected produced stdout"
 249                         " visible to us `%s'" % out)
 250         return [`status`]
 251
 252 def copyupdown(c, ce, upp):
 253         cmdnumargs(c, ce, 2)
 254         isrc = 0
 255         idst = 1
 256         ilocal = 0 + upp
 257         iremote = 1 - upp
 258         wh = ce[0]
 259         sd = c[1:]
 260         sde = ce[1:]
 261         if not sd[0] or not sd[1]:
 262                 bomb("%s paths must be nonempty" % wh)
 263         dirsp = sd[0][-1]=='/'
 264         functions = "import errno\n"
 265         if dirsp != (sd[1][-1]=='/'):
 266                 bomb("% paths must agree about directoryness"
 267                         " (presence or absence of trailing /)" % wh)
 268         localfd = None
 269         deststdout = devnull_read
 270         srcstdin = devnull_read
 271         remfileq = shellquote_arg(sde[iremote])
 272         if not dirsp:
 273                 modestr = ''
 274                 rune = 'cat %s%s' % ('><'[upp], remfileq)
 275                 if upp:
 276                         deststdout = file(sd[idst], 'w')
 277                 else:
 278                         srcstdin = file(sd[isrc], 'r')
 279                         status = os.fstat(srcstdin.fileno())
 280                         if status.st_mode & 0111:
 281                                 rune += '; chmod +x -- %s' % (remfileq)
 282                 localcmdl = ['cat']
 283         else:
 284                 taropts = [None, None]
 285                 taropts[isrc] = '-c .'
 286                 taropts[idst] = '-p -x --no-same-owner'
 287
 288                 rune = 'cd %s; tar %s -f -' % (remfileq, taropts[iremote])
 289                 if upp:
 290                         try: os.mkdir(sd[ilocal])
 291                         except (IOError,OSError), oe:
 292                                 if oe.errno != errno.EEXIST: raise
 293                 else:
 294                         rune = ('if ! test -d %s; then mkdir -- %s; fi; ' % (
 295                                                         remfileq,remfileq)
 296                                 ) + rune
 297
 298                 localcmdl = ['tar','-C',sd[ilocal]] + (
 299                         ('%s -f -' % taropts[ilocal]).split()
 300                 )
 301         rune = 'set -e; ' + rune
 302         downcmdl = downs['shstring'] + [rune]
 303
 304         if upp: cmdls = (downcmdl, localcmdl)
 305         else: cmdls = (localcmdl, downcmdl)
 306
 307         debug(`["cmdls", `cmdls`]`)
 308         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 309
 310         subprocs = [None,None]
 311         debug(" +< %s" % string.join(cmdls[0]))
 312         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 313                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 314         debug(" +> %s" % string.join(cmdls[1]))
 315         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 316                         stdout=deststdout, preexec_fn=preexecfn)
 317         subprocs[0].stdout.close()
 318         timeout_start(copy_timeout)
 319         for sdn in [1,0]:
 320                 debug(" +"+"<>"[sdn]+"?");
 321                 status = subprocs[sdn].wait()
 322                 if not (status==0 or (sdn==0 and status==-13)):
 323                         timeout_stop()
 324                         bomb("%s %s failed, status %d" %
 325                                 (wh, ['source','destination'][sdn], status))
 326         timeout_stop()
 327
 328 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 329 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 330
 331 def command():
 332         sys.stdout.flush()
 333         ce = sys.stdin.readline()
 334         if not ce: bomb('end of file - caller quit?')
 335         ce = ce.rstrip().split()
 336         c = map(urllib.unquote, ce)
 337         if not c: bomb('empty commands are not permitted')
 338         debug('executing '+string.join(ce))
 339         c_lookup = c[0].replace('-','_')
 340         try: f = globals()['cmd_'+c_lookup]
 341         except KeyError: bomb("unknown command `%s'" % ce[0])
 342         try:
 343                 r = f(c, ce)
 344                 if not r: r = []
 345                 r.insert(0, 'ok')
 346         except FailedCmd, fc:
 347                 r = fc.e
 348         print string.join(r)
 349
 350 signal_list = [ signal.SIGHUP, signal.SIGTERM,
 351                 signal.SIGINT, signal.SIGPIPE ]
 352
 353 def sethandlers(f):
 354         for signum in signal_list: signal.signal(signum, f)
 355
 356 def cleanup():
 357         global downtmp, cleaning
 358         debug("cleanup...");
 359         sethandlers(signal.SIG_DFL)
 360         cleaning = True
 361         if downtmp:
 362                 caller.hook_cleanup()
 363         cleaning = False
 364         downtmp = False
 365
 366 def error_cleanup():
 367         try:
 368                 ok = False
 369                 try:
 370                         cleanup()
 371                         ok = True
 372                 except Quit, q:
 373                         print >> sys.stderr, q.m
 374                 except:
 375                         print >> sys.stderr, "Unexpected cleanup error:"
 376                         traceback.print_exc()
 377                         print >> sys.stderr, ''
 378                 if not ok:
 379                         print >> sys.stderr, ("while cleaning up"
 380                                 " because of another error:")
 381         except:
 382                 pass
 383
 384 def prepare():
 385         global downtmp, cleaning
 386         downtmp = None
 387         def handler(sig, *any):
 388                 cleanup()
 389                 os.kill(os.getpid(), sig)
 390         sethandlers(handler)
 391
 392 def mainloop():
 393         try:
 394                 while True: command()
 395         except Quit, q:
 396                 error_cleanup()
 397                 if q.m: print >> sys.stderr, q.m
 398                 sys.exit(q.ec)
 399         except:
 400                 error_cleanup()
 401                 print >> sys.stderr, "Unexpected error:"
 402                 traceback.print_exc()
 403                 sys.exit(16)
 404
 405 def main():
 406         signal.signal(signal.SIGALRM, alarm_handler)
 407         ok()
 408         prepare()
 409         mainloop()