lib/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import errno
  33 import re as regexp
  34
  35 from Autopkgtest import *
  36
  37 debuglevel = None
  38 progname = "<VirtSubproc>"
  39 devnull_read = file('/dev/null','r')
  40 caller = __main__
  41 copy_timeout = 300
  42
  43 class Quit:
  44         def __init__(q,ec,m): q.ec = ec; q.m = m
  45
  46 class Timeout: pass
  47 def alarm_handler(*a): raise Timeout()
  48 def timeout_start(to): signal.alarm(to)
  49 def timeout_stop(): signal.alarm(0)
  50
  51 class FailedCmd:
  52         def __init__(fc,e): fc.e = e
  53
  54 def debug(m):
  55         if not debuglevel: return
  56         print >> sys.stderr, progname+": debug:", m
  57
  58 def bomb(m):
  59         raise Quit(12, progname+": failure: %s" % m)
  60
  61 def ok(): print 'ok'
  62
  63 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  64         if len(c) < 1+nargs:
  65                 bomb("too few arguments to command `%s'" % ce[0])
  66         if noptargs is not None and len(c) > 1+nargs+noptargs:
  67                 bomb("too many arguments to command `%s'" % ce[0])
  68
  69 def cmd_capabilities(c, ce):
  70         cmdnumargs(c, ce)
  71         return caller.hook_capabilities() + ['execute-debug']
  72
  73 def cmd_quit(c, ce):
  74         cmdnumargs(c, ce)
  75         raise Quit(0, '')
  76
  77 def cmd_close(c, ce):
  78         cmdnumargs(c, ce)
  79         if not downtmp: bomb("`close' when not open")
  80         cleanup()
  81
  82 def cmd_print_auxverb_command(c, ce): return print_command('auxverb', c, ce)
  83 def cmd_print_shstring_command(c, ce): return print_command('shstring', c, ce)
  84
  85 def print_command(which, c, ce):
  86         global downs
  87         cmdnumargs(c, ce)
  88         if not downtmp: bomb("`print-%s-command' when not open" % which)
  89         cl = downs[which]
  90         if not len(cl):
  91                 cl = ['sh','-c','exec "$@"','x'] + cl
  92         return [','.join(map(urllib.quote, cl))]
  93
  94 def preexecfn():
  95         caller.hook_forked_inchild()
  96
  97 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  98         debug(" ++ %s" % string.join(popenargs[0]))
  99         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
 100         if instr is None: popenargsk['stdin'] = devnull_read
 101         timeout_start(timeout)
 102         (out, err) = sp.communicate(instr)
 103         timeout_stop()
 104         if err: bomb("%s unexpectedly produced stderr output `%s'" %
 105                         (what, err))
 106         status = sp.wait()
 107         return (status, out)
 108
 109 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
 110         cmdl = cmd_string.split()
 111
 112         if downp: perhaps_down = downs['auxverb']
 113         else: perhaps_down = []
 114
 115         if outp: stdout = subprocess.PIPE
 116         else: stdout = None
 117
 118         cmd = cmdl + cmd_list
 119         if len(perhaps_down): cmd = perhaps_down + cmd
 120
 121         (status, out) = execute_raw(cmdl[0], None, timeout,
 122                                 cmd, stdout=stdout)
 123
 124         if status: bomb("%s%s failed (exit status %d)" %
 125                         ((downp and "(down) " or ""), cmdl[0], status))
 126
 127         if outp and out and out[-1]=='\n': out = out[:-1]
 128         return out
 129
 130 def cmd_open(c, ce):
 131         global downtmp
 132         cmdnumargs(c, ce)
 133         if downtmp: bomb("`open' when already open")
 134         caller.hook_open()
 135         opened1()
 136         downtmp = caller.hook_downtmp()
 137         return opened2()
 138
 139 def downtmp_mktemp():
 140         global downtmp
 141         return execute('mktemp -t -d', downp=True, outp=True)
 142
 143 def downtmp_remove():
 144         global downtmp
 145         execute('rm -rf --', [downtmp], downp=True)
 146
 147 perl_quote_re = regexp.compile('[^-+=_.,;:() 0-9a-zA-Z]')
 148 def perl_quote_1chargroup(m): return '\\x%02x' % ord(m.group(0))
 149 def perl_quote(s): return '"'+perl_quote_re.sub(perl_quote_1chargroup, s)+'"'
 150
 151 def opened1():
 152         global down, downkind, downs
 153         debug("downkind = %s, down = %s" % (downkind, `down`))
 154         if downkind == 'auxverb':
 155                 downs = { 'auxverb': down,
 156                           'shstring': down + ['sh','-c'] }
 157         elif downkind == 'shstring':
 158                 downs = { 'shstring': down,
 159                           'auxverb': ['perl','-e','''
 160                         @cmd=('''+(','.join(map(perl_quote,down)))+''');
 161                         my $shstring = pop @ARGV;
 162                         s/'/'\\\\''/g foreach @ARGV;
 163                         push @cmd, "'$_'" foreach @ARGV;
 164                         my $argv0=$cmd[0];
 165                         exec $argv0 @cmd;
 166                         die "$argv0: $!";
 167                 '''] }
 168         debug("downs = %s" % `downs`)
 169
 170 def opened2():
 171         global downtmp, downs
 172         debug("downtmp = %s" % (downtmp))
 173         return [downtmp]
 174
 175 def cmd_revert(c, ce):
 176         global downtmp
 177         cmdnumargs(c, ce)
 178         if not downtmp: bomb("`revert' when not open")
 179         if not 'revert' in caller.hook_capabilities():
 180                 bomb("`revert' when `revert' not advertised")
 181         caller.hook_revert()
 182         opened1()
 183         downtmp = caller.hook_downtmp()
 184         return opened2()
 185
 186 def cmd_execute(c, ce):
 187         cmdnumargs(c, ce, 5, None)
 188         if not downtmp: bomb("`execute' when not open" % which)
 189         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 190         debug_g = None
 191         timeout = 0
 192         envs = []
 193         for kw in ce[6:]:
 194                 if kw.startswith('debug='):
 195                         if debug_g: bomb("multiple debug= in execute")
 196                         m = debug_re.match(kw)
 197                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 198                         debug_g = m.groups()
 199                 elif kw.startswith('timeout='):
 200                         try: timeout = int(kw[8:],0)
 201                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 202                 elif kw.startswith('env='):
 203                         es = kw[4:]; eq = es.find('=')
 204                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 205                         envs.append((es[:eq], es[eq+1:]))
 206                 else: bomb("invalid execute kw arg `%s'" % kw)
 207
 208         rune = 'set -e; exec '
 209
 210         stdout = None
 211         tfd = None
 212         if debug_g:
 213                 rune += " 3>&1"
 214
 215         for ioe in range(3):
 216                 rune += " %d%s%s" % (ioe, '<>'[ioe>0],
 217                                         shellquote_arg(ce[ioe+2]))
 218         if debug_g:
 219                 (tfd,hfd) = m.groups()
 220                 tfd = int(tfd)
 221                 rune += " %d>&3 3>&-" % tfd
 222                 stdout = int(hfd)
 223
 224         rune += '; '
 225
 226         rune += 'cd %s; ' % shellquote_arg(ce[5])
 227
 228         for e in envs:
 229                 (en, ev) = map(urllib.unquote,e)
 230                 rune += "%s=%s " % (en, shellquote_arg(ev))
 231
 232         rune += 'exec ' + shellquote_cmdl(map(urllib.unquote, ce[1].split(',')))
 233
 234         cmdl = downs['shstring'] + [rune]
 235
 236         stdout_copy = None
 237         try:
 238                 if type(stdout) == type(2):
 239                         stdout_copy = os.dup(stdout)
 240                 try:
 241                         (status, out) = execute_raw('target-cmd', None,
 242                                 timeout, cmdl, stdout=stdout_copy,
 243                                 stdin=devnull_read, stderr=subprocess.PIPE)
 244                 except Timeout:
 245                         raise FailedCmd(['timeout'])
 246         finally:
 247                 if stdout_copy is not None: os.close(stdout_copy)
 248
 249         if out: bomb("target command unexpected produced stdout"
 250                         " visible to us `%s'" % out)
 251         return [`status`]
 252
 253 def copyupdown(c, ce, upp):
 254         cmdnumargs(c, ce, 2)
 255         if not downtmp: bomb("`copyup'/`copydown' when not open" % which)
 256         isrc = 0
 257         idst = 1
 258         ilocal = 0 + upp
 259         iremote = 1 - upp
 260         wh = ce[0]
 261         sd = c[1:]
 262         if not sd[0] or not sd[1]:
 263                 bomb("%s paths must be nonempty" % wh)
 264         dirsp = sd[0][-1]=='/'
 265         functions = "import errno\n"
 266         if dirsp != (sd[1][-1]=='/'):
 267                 bomb("% paths must agree about directoryness"
 268                         " (presence or absence of trailing /)" % wh)
 269         localfd = None
 270         deststdout = devnull_read
 271         srcstdin = devnull_read
 272         remfileq = shellquote_arg(sd[iremote])
 273         if not dirsp:
 274                 modestr = ''
 275                 rune = 'cat %s%s' % ('><'[upp], remfileq)
 276                 if upp:
 277                         deststdout = file(sd[idst], 'w')
 278                 else:
 279                         srcstdin = file(sd[isrc], 'r')
 280                         status = os.fstat(srcstdin.fileno())
 281                         if status.st_mode & 0111:
 282                                 rune += '; chmod +x -- %s' % (remfileq)
 283                 localcmdl = ['cat']
 284         else:
 285                 taropts = [None, None]
 286                 taropts[isrc] = '-c .'
 287                 taropts[idst] = '-p -x --no-same-owner'
 288
 289                 rune = 'cd %s; tar %s -f -' % (remfileq, taropts[iremote])
 290                 if upp:
 291                         try: os.mkdir(sd[ilocal])
 292                         except (IOError,OSError), oe:
 293                                 if oe.errno != errno.EEXIST: raise
 294                 else:
 295                         rune = ('if ! test -d %s; then mkdir -- %s; fi; ' % (
 296                                                         remfileq,remfileq)
 297                                 ) + rune
 298
 299                 localcmdl = ['tar','-C',sd[ilocal]] + (
 300                         ('%s -f -' % taropts[ilocal]).split()
 301                 )
 302         rune = 'set -e; ' + rune
 303         downcmdl = downs['shstring'] + [rune]
 304
 305         if upp: cmdls = (downcmdl, localcmdl)
 306         else: cmdls = (localcmdl, downcmdl)
 307
 308         debug(`["cmdls", `cmdls`]`)
 309         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 310
 311         subprocs = [None,None]
 312         debug(" +< %s" % string.join(cmdls[0]))
 313         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 314                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 315         debug(" +> %s" % string.join(cmdls[1]))
 316         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 317                         stdout=deststdout, preexec_fn=preexecfn)
 318         subprocs[0].stdout.close()
 319         timeout_start(copy_timeout)
 320         for sdn in [1,0]:
 321                 debug(" +"+"<>"[sdn]+"?");
 322                 status = subprocs[sdn].wait()
 323                 if not (status==0 or (sdn==0 and status==-13)):
 324                         timeout_stop()
 325                         bomb("%s %s failed, status %d" %
 326                                 (wh, ['source','destination'][sdn], status))
 327         timeout_stop()
 328
 329 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 330 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 331
 332 def command():
 333         sys.stdout.flush()
 334         ce = sys.stdin.readline()
 335         if not ce: bomb('end of file - caller quit?')
 336         ce = ce.rstrip().split()
 337         c = map(urllib.unquote, ce)
 338         if not c: bomb('empty commands are not permitted')
 339         debug('executing '+string.join(ce))
 340         c_lookup = c[0].replace('-','_')
 341         try: f = globals()['cmd_'+c_lookup]
 342         except KeyError: bomb("unknown command `%s'" % ce[0])
 343         try:
 344                 r = f(c, ce)
 345                 if not r: r = []
 346                 r.insert(0, 'ok')
 347         except FailedCmd, fc:
 348                 r = fc.e
 349         print string.join(r)
 350
 351 signal_list = [ signal.SIGHUP, signal.SIGTERM,
 352                 signal.SIGINT, signal.SIGPIPE ]
 353
 354 def sethandlers(f):
 355         for signum in signal_list: signal.signal(signum, f)
 356
 357 def cleanup():
 358         global downtmp, cleaning
 359         debug("cleanup...");
 360         sethandlers(signal.SIG_DFL)
 361         cleaning = True
 362         if downtmp:
 363                 caller.hook_cleanup()
 364         cleaning = False
 365         downtmp = False
 366
 367 def error_cleanup():
 368         try:
 369                 ok = False
 370                 try:
 371                         cleanup()
 372                         ok = True
 373                 except Quit, q:
 374                         print >> sys.stderr, q.m
 375                 except:
 376                         print >> sys.stderr, "Unexpected cleanup error:"
 377                         traceback.print_exc()
 378                         print >> sys.stderr, ''
 379                 if not ok:
 380                         print >> sys.stderr, ("while cleaning up"
 381                                 " because of another error:")
 382         except:
 383                 pass
 384
 385 def prepare():
 386         global downtmp, cleaning
 387         downtmp = None
 388         def handler(sig, *any):
 389                 cleanup()
 390                 os.kill(os.getpid(), sig)
 391         sethandlers(handler)
 392
 393 def mainloop():
 394         try:
 395                 while True: command()
 396         except Quit, q:
 397                 error_cleanup()
 398                 if q.m: print >> sys.stderr, q.m
 399                 sys.exit(q.ec)
 400         except:
 401                 error_cleanup()
 402                 print >> sys.stderr, "Unexpected error:"
 403                 traceback.print_exc()
 404                 sys.exit(16)
 405
 406 def main():
 407         signal.signal(signal.SIGALRM, alarm_handler)
 408         ok()
 409         prepare()
 410         mainloop()