lib/VirtSubproc.py

   1 # VirtSubproc is part of autopkgtest
   2 # autopkgtest is a tool for testing Debian binary packages
   3 #
   4 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 #
  20 # See the file CREDITS for a full list of credits information (often
  21 # installed as /usr/share/doc/autopkgtest/CREDITS).
  22
  23 import __main__
  24
  25 import sys
  26 import os
  27 import string
  28 import urllib
  29 import signal
  30 import subprocess
  31 import traceback
  32 import errno
  33 import re as regexp
  34
  35 from Autopkgtest import *
  36
  37 debuglevel = None
  38 progname = "<VirtSubproc>"
  39 devnull_read = file('/dev/null','r')
  40 caller = __main__
  41 copy_timeout = 300
  42
  43 class Quit:
  44         def __init__(q,ec,m): q.ec = ec; q.m = m
  45
  46 class Timeout: pass
  47 def alarm_handler(*a): raise Timeout()
  48 def timeout_start(to): signal.alarm(to)
  49 def timeout_stop(): signal.alarm(0)
  50
  51 class FailedCmd:
  52         def __init__(fc,e): fc.e = e
  53
  54 def debug(m):
  55         if not debuglevel: return
  56         print >> sys.stderr, progname+": debug:", m
  57
  58 def bomb(m):
  59         raise Quit(12, progname+": failure: %s" % m)
  60
  61 def ok(): print 'ok'
  62
  63 def cmdnumargs(c, ce, nargs=0, noptargs=0):
  64         if len(c) < 1+nargs:
  65                 bomb("too few arguments to command `%s'" % ce[0])
  66         if noptargs is not None and len(c) > 1+nargs+noptargs:
  67                 bomb("too many arguments to command `%s'" % ce[0])
  68
  69 def cmd_capabilities(c, ce):
  70         cmdnumargs(c, ce)
  71         return caller.hook_capabilities() + ['execute-debug']
  72
  73 def cmd_quit(c, ce):
  74         cmdnumargs(c, ce)
  75         raise Quit(0, '')
  76
  77 def cmd_close(c, ce):
  78         cmdnumargs(c, ce)
  79         if not downtmp: bomb("`close' when not open")
  80         cleanup()
  81
  82 def cmd_print_auxverb_command(c, ce): return print_command('auxverb', c, ce)
  83 def cmd_print_shstring_command(c, ce): return print_command('shstring', c, ce)
  84
  85 def print_command(which, c, ce):
  86         global downs
  87         cmdnumargs(c, ce)
  88         if not downtmp: bomb("`print-%s-command' when not open" % which)
  89         cl = downs[which]
  90         if not len(cl):
  91                 cl = ['sh','-c','exec "$@"','x'] + cl
  92         return [','.join(map(urllib.quote, cl))]
  93
  94 def preexecfn():
  95         caller.hook_forked_inchild()
  96
  97 def execute_raw(what, instr, timeout, *popenargs, **popenargsk):
  98         debug(" ++ %s" % string.join(popenargs[0]))
  99         sp = subprocess.Popen(preexec_fn=preexecfn, *popenargs, **popenargsk)
 100         if instr is None: popenargsk['stdin'] = devnull_read
 101         timeout_start(timeout)
 102         (out, err) = sp.communicate(instr)
 103         timeout_stop()
 104         if err: bomb("%s unexpectedly produced stderr output `%s'" %
 105                         (what, err))
 106         status = sp.wait()
 107         return (status, out)
 108
 109 def execute(cmd_string, cmd_list=[], downp=False, outp=False, timeout=0):
 110         cmdl = cmd_string.split()
 111
 112         if downp: perhaps_down = downs['auxverb']
 113         else: perhaps_down = []
 114
 115         if outp: stdout = subprocess.PIPE
 116         else: stdout = None
 117
 118         cmd = cmdl + cmd_list
 119         if len(perhaps_down): cmd = perhaps_down + cmd
 120
 121         (status, out) = execute_raw(cmdl[0], None, timeout,
 122                                 cmd, stdout=stdout)
 123
 124         if status: bomb("%s%s failed (exit status %d)" %
 125                         ((downp and "(down) " or ""), cmdl[0], status))
 126
 127         if outp and out and out[-1]=='\n': out = out[:-1]
 128         return out
 129
 130 def cmd_open(c, ce):
 131         global downtmp
 132         cmdnumargs(c, ce)
 133         if downtmp: bomb("`open' when already open")
 134         caller.hook_open()
 135         opened1()
 136         downtmp = caller.hook_downtmp()
 137         return opened2()
 138
 139 def downtmp_mktemp():
 140         global downtmp
 141         return execute('mktemp -t -d', downp=True, outp=True)
 142
 143 def downtmp_remove():
 144         global downtmp
 145         execute('rm -rf --', [downtmp], downp=True)
 146
 147 perl_quote_re = regexp.compile('[^-+=_.,;:() 0-9a-zA-Z]')
 148 def perl_quote_1chargroup(m): return '\\x%02x' % ord(m.group(0))
 149 def perl_quote(s): return '"'+perl_quote_re.sub(perl_quote_1chargroup, s)+'"'
 150
 151 def opened1():
 152         global down, downkind, downs
 153         debug("downkind = %s, down = %s" % (downkind, `down`))
 154         if downkind == 'auxverb':
 155                 downs = { 'auxverb': down,
 156                           'shstring': down + ['sh','-c'] }
 157         elif downkind == 'shstring':
 158                 downs = { 'shstring': down,
 159                           'auxverb': ['perl','-e','''
 160                         @cmd=('''+(','.join(map(perl_quote,down)))+''');
 161                         my $shstring = pop @ARGV;
 162                         s/'/'\\\\''/g foreach @ARGV;
 163                         push @cmd, "'$_'" foreach @ARGV;
 164                         my $argv0=$cmd[0];
 165                         exec $argv0 @cmd;
 166                         die "$argv0: $!";
 167                 '''] }
 168         debug("downs = %s" % `downs`)
 169
 170 def opened2():
 171         global downtmp, downs
 172         debug("downtmp = %s" % (downtmp))
 173         return [downtmp]
 174
 175 def cmd_revert(c, ce):
 176         global downtmp
 177         cmdnumargs(c, ce)
 178         if not downtmp: bomb("`revert' when not open")
 179         if not 'revert' in caller.hook_capabilities():
 180                 bomb("`revert' when `revert' not advertised")
 181         caller.hook_revert()
 182         opened1()
 183         downtmp = caller.hook_downtmp()
 184         return opened2()
 185
 186 def cmd_execute(c, ce):
 187         cmdnumargs(c, ce, 5, None)
 188         if not downtmp: bomb("`execute' when not open" % which)
 189         debug_re = regexp.compile('debug=(\d+)\-(\d+)$')
 190         debug_g = None
 191         timeout = 0
 192         envs = []
 193         for kw in ce[6:]:
 194                 if kw.startswith('debug='):
 195                         if debug_g: bomb("multiple debug= in execute")
 196                         m = debug_re.match(kw)
 197                         if not m: bomb("invalid execute debug arg `%s'" % kw)
 198                         debug_g = m.groups()
 199                 elif kw.startswith('timeout='):
 200                         try: timeout = int(kw[8:],0)
 201                         except ValueError: bomb("invalid timeout arg `%s'" %kw)
 202                 elif kw.startswith('env='):
 203                         es = kw[4:]; eq = es.find('=')
 204                         if eq <= 0: bomb("invalid env arg `%s'" % kw)
 205                         envs.append((es[:eq], es[eq+1:]))
 206                 else: bomb("invalid execute kw arg `%s'" % kw)
 207
 208         rune = 'set -e; exec '
 209
 210         stdout = None
 211         tfd = None
 212         if debug_g:
 213                 rune += " 3>&1"
 214
 215         for ioe in range(3):
 216                 rune += " %d%s%s" % (ioe, '<>'[ioe>0],
 217                                         shellquote_arg(ce[ioe+2]))
 218         if debug_g:
 219                 (tfd,hfd) = m.groups()
 220                 tfd = int(tfd)
 221                 rune += " %d>&3 3>&-" % tfd
 222                 stdout = int(hfd)
 223
 224         rune += '; '
 225
 226         rune += 'cd %s; ' % shellquote_arg(ce[5])
 227
 228         for e in envs:
 229                 (en, ev) = map(urllib.unquote,e)
 230                 rune += "%s=%s " % (en, shellquote_arg(ev))
 231
 232         rune += 'exec ' + shellquote_cmdl(map(urllib.unquote, ce[1].split(',')))
 233
 234         cmdl = downs['shstring'] + [rune]
 235
 236         stdout_copy = None
 237         try:
 238                 if type(stdout) == type(2):
 239                         stdout_copy = os.dup(stdout)
 240                 try:
 241                         (status, out) = execute_raw('target-cmd', None,
 242                                 timeout, cmdl, stdout=stdout_copy,
 243                                 stdin=devnull_read, stderr=subprocess.PIPE)
 244                 except Timeout:
 245                         raise FailedCmd(['timeout'])
 246         finally:
 247                 if stdout_copy is not None: os.close(stdout_copy)
 248
 249         if out: bomb("target command unexpected produced stdout"
 250                         " visible to us `%s'" % out)
 251         return [`status`]
 252
 253 def copyupdown(c, ce, upp):
 254         cmdnumargs(c, ce, 2)
 255         if not downtmp: bomb("`copyup'/`copydown' when not open" % which)
 256         isrc = 0
 257         idst = 1
 258         ilocal = 0 + upp
 259         iremote = 1 - upp
 260         wh = ce[0]
 261         sd = c[1:]
 262         sde = ce[1:]
 263         if not sd[0] or not sd[1]:
 264                 bomb("%s paths must be nonempty" % wh)
 265         dirsp = sd[0][-1]=='/'
 266         functions = "import errno\n"
 267         if dirsp != (sd[1][-1]=='/'):
 268                 bomb("% paths must agree about directoryness"
 269                         " (presence or absence of trailing /)" % wh)
 270         localfd = None
 271         deststdout = devnull_read
 272         srcstdin = devnull_read
 273         remfileq = shellquote_arg(sde[iremote])
 274         if not dirsp:
 275                 modestr = ''
 276                 rune = 'cat %s%s' % ('><'[upp], remfileq)
 277                 if upp:
 278                         deststdout = file(sd[idst], 'w')
 279                 else:
 280                         srcstdin = file(sd[isrc], 'r')
 281                         status = os.fstat(srcstdin.fileno())
 282                         if status.st_mode & 0111:
 283                                 rune += '; chmod +x -- %s' % (remfileq)
 284                 localcmdl = ['cat']
 285         else:
 286                 taropts = [None, None]
 287                 taropts[isrc] = '-c .'
 288                 taropts[idst] = '-p -x --no-same-owner'
 289
 290                 rune = 'cd %s; tar %s -f -' % (remfileq, taropts[iremote])
 291                 if upp:
 292                         try: os.mkdir(sd[ilocal])
 293                         except (IOError,OSError), oe:
 294                                 if oe.errno != errno.EEXIST: raise
 295                 else:
 296                         rune = ('if ! test -d %s; then mkdir -- %s; fi; ' % (
 297                                                         remfileq,remfileq)
 298                                 ) + rune
 299
 300                 localcmdl = ['tar','-C',sd[ilocal]] + (
 301                         ('%s -f -' % taropts[ilocal]).split()
 302                 )
 303         rune = 'set -e; ' + rune
 304         downcmdl = downs['shstring'] + [rune]
 305
 306         if upp: cmdls = (downcmdl, localcmdl)
 307         else: cmdls = (localcmdl, downcmdl)
 308
 309         debug(`["cmdls", `cmdls`]`)
 310         debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`)
 311
 312         subprocs = [None,None]
 313         debug(" +< %s" % string.join(cmdls[0]))
 314         subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin,
 315                         stdout=subprocess.PIPE, preexec_fn=preexecfn)
 316         debug(" +> %s" % string.join(cmdls[1]))
 317         subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout,
 318                         stdout=deststdout, preexec_fn=preexecfn)
 319         subprocs[0].stdout.close()
 320         timeout_start(copy_timeout)
 321         for sdn in [1,0]:
 322                 debug(" +"+"<>"[sdn]+"?");
 323                 status = subprocs[sdn].wait()
 324                 if not (status==0 or (sdn==0 and status==-13)):
 325                         timeout_stop()
 326                         bomb("%s %s failed, status %d" %
 327                                 (wh, ['source','destination'][sdn], status))
 328         timeout_stop()
 329
 330 def cmd_copydown(c, ce): copyupdown(c, ce, False)
 331 def cmd_copyup(c, ce): copyupdown(c, ce, True)
 332
 333 def command():
 334         sys.stdout.flush()
 335         ce = sys.stdin.readline()
 336         if not ce: bomb('end of file - caller quit?')
 337         ce = ce.rstrip().split()
 338         c = map(urllib.unquote, ce)
 339         if not c: bomb('empty commands are not permitted')
 340         debug('executing '+string.join(ce))
 341         c_lookup = c[0].replace('-','_')
 342         try: f = globals()['cmd_'+c_lookup]
 343         except KeyError: bomb("unknown command `%s'" % ce[0])
 344         try:
 345                 r = f(c, ce)
 346                 if not r: r = []
 347                 r.insert(0, 'ok')
 348         except FailedCmd, fc:
 349                 r = fc.e
 350         print string.join(r)
 351
 352 signal_list = [ signal.SIGHUP, signal.SIGTERM,
 353                 signal.SIGINT, signal.SIGPIPE ]
 354
 355 def sethandlers(f):
 356         for signum in signal_list: signal.signal(signum, f)
 357
 358 def cleanup():
 359         global downtmp, cleaning
 360         debug("cleanup...");
 361         sethandlers(signal.SIG_DFL)
 362         cleaning = True
 363         if downtmp:
 364                 caller.hook_cleanup()
 365         cleaning = False
 366         downtmp = False
 367
 368 def error_cleanup():
 369         try:
 370                 ok = False
 371                 try:
 372                         cleanup()
 373                         ok = True
 374                 except Quit, q:
 375                         print >> sys.stderr, q.m
 376                 except:
 377                         print >> sys.stderr, "Unexpected cleanup error:"
 378                         traceback.print_exc()
 379                         print >> sys.stderr, ''
 380                 if not ok:
 381                         print >> sys.stderr, ("while cleaning up"
 382                                 " because of another error:")
 383         except:
 384                 pass
 385
 386 def prepare():
 387         global downtmp, cleaning
 388         downtmp = None
 389         def handler(sig, *any):
 390                 cleanup()
 391                 os.kill(os.getpid(), sig)
 392         sethandlers(handler)
 393
 394 def mainloop():
 395         try:
 396                 while True: command()
 397         except Quit, q:
 398                 error_cleanup()
 399                 if q.m: print >> sys.stderr, q.m
 400                 sys.exit(q.ec)
 401         except:
 402                 error_cleanup()
 403                 print >> sys.stderr, "Unexpected error:"
 404                 traceback.print_exc()
 405                 sys.exit(16)
 406
 407 def main():
 408         signal.signal(signal.SIGALRM, alarm_handler)
 409         ok()
 410         prepare()
 411         mainloop()