From: Ian Jackson Date: Thu, 12 Oct 2006 18:19:06 +0000 (+0100) Subject: === modified file 'debian/changelog' X-Git-Tag: converted-from-bzr~32^3~77 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=d708e283f07d72ec9c69dafeabe83e2912d07a95;p=autopkgtest.git === modified file 'debian/changelog' --- debian/changelog +++ debian/changelog @@ -1,3 +1,12 @@ +autopkgtest (0.6.2~iwj) unstable; urgency=low + + * Split VirtSubproc off from adt-virt-chroot in preparation + for adt-virt-xenlvm. + * Concatenate arguments to dchroot et al, to stop them looking inside + the args and getting confused. + + -- Ian Jackson Thu, 12 Oct 2006 19:18:40 +0100 + autopkgtest (0.6.1) edgy; urgency=low * autopkgtest-xenlvm: filter output from debootstrap to make --- diff --git a/Makefile b/Makefile index ecc624d..c9f0a22 100644 --- a/Makefile +++ b/Makefile @@ -25,15 +25,18 @@ include settings.make programs = virt-chroot/adt-virt-chroot \ runner/adt-run +pythonfiles = virt-chroot/VirtSubproc.py + all: cd xen && $(MAKE) install-here: - $(INSTALL_DIRS) -d $(bindir) $(docdir) $(man1dir) + $(INSTALL_DIRS) -d $(bindir) $(docdir) $(man1dir) $(pythondir) set -e; for f in $(programs); do \ $(INSTALL_PROGRAM) $$f $(bindir); \ test ! -f $$f.1 || $(INSTALL_DOC) $$f.1 $(man1dir); \ done + $(INSTALL_DATA) $(pythonfiles) $(pythondir) $(INSTALL_DOC) CREDITS debian/changelog $(docdir) install: install-here diff --git a/debian/changelog b/debian/changelog index e792fe0..838f543 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +autopkgtest (0.6.2~iwj) unstable; urgency=low + + * Split VirtSubproc off from adt-virt-chroot in preparation + for adt-virt-xenlvm. + * Concatenate arguments to dchroot et al, to stop them looking inside + the args and getting confused. + + -- Ian Jackson Thu, 12 Oct 2006 19:18:40 +0100 + autopkgtest (0.6.1) edgy; urgency=low * autopkgtest-xenlvm: filter output from debootstrap to make diff --git a/settings.make b/settings.make index a8d78d7..ea7f67e 100644 --- a/settings.make +++ b/settings.make @@ -6,6 +6,7 @@ man1dir = $(mandir)/man1 pkgname = autopkgtest docdir = $(share)/doc/$(pkgname) sharedir = $(share)/$(pkgname) +pythondir = $(share)/$(pkgname)/python sharedir_lnfrom = $(share)/$(pkgname) etcdir = /etc etcconfdir = $(etcdir)/autopkgtest diff --git a/virt-chroot/VirtSubproc.py b/virt-chroot/VirtSubproc.py new file mode 100644 index 0000000..e30a5fb --- /dev/null +++ b/virt-chroot/VirtSubproc.py @@ -0,0 +1,316 @@ +# VirtSubproc is part of autopkgtest +# autopkgtest is a tool for testing Debian binary packages +# +# autopkgtest is Copyright (C) 2006 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +# See the file CREDITS for a full list of credits information (often +# installed as /usr/share/doc/autopkgtest/CREDITS). + +import __main__ + +import sys +import os +import string +import urllib +import signal +import subprocess +import traceback + +debuglevel = None +progname = "" +devnull_read = file('/dev/null','r') +caller = __main__ + +class Quit: + def __init__(q,ec,m): q.ec = ec; q.m = m + +def debug(m): + if not debuglevel: return + print >> sys.stderr, progname+": debug:", m + +def bomb(m): + raise Quit(12, progname+": failure: %s" % m) + +def ok(): print 'ok' + +def cmdnumargs(c, ce, nargs=0): + if len(c) == nargs + 1: return + bomb("wrong number of arguments to command `%s'" % ce[0]) + +def cmd_capabilities(c, ce): + cmdnumargs(c, ce) + +def cmd_quit(c, ce): + cmdnumargs(c, ce) + raise Quit(0, '') + +def execute_raw(what, instr, *popenargs, **popenargsk): + debug(" ++ %s" % string.join(popenargs[0])) + sp = subprocess.Popen(*popenargs, **popenargsk) + if instr is None: popenargsk['stdin'] = devnull_read + (out, err) = sp.communicate(instr) + if err: bomb("%s unexpectedly produced stderr output `%s'" % + (what, err)) + status = sp.wait() + return (status, out) + +def execute(cmd_string, cmd_list=[], downp=False, outp=False): + cmdl = cmd_string.split() + + if downp: perhaps_down = down + else: downp = [] + + if outp: stdout = subprocess.PIPE + else: stdout = None + + cmd = cmdl + cmd_list + if len(perhaps_down): cmd = perhaps_down + [' '.join(cmd)] + + (status, out) = execute_raw(cmdl[0], None, cmd, stdout=stdout) + + if status: bomb("%s%s failed (exit status %d)" % + ((downp and "(down) " or ""), cmdl[0], status)) + + if outp and out and out[-1]=='\n': out = out[:-1] + return out + +def cmd_open(c, ce): + global downtmp + cmdnumargs(c, ce) + if downtmp: bomb("`open' when already open") + downtmp = caller.hook_open() + return [downtmp] + +def cmd_close(c, ce): + global downtmp + cmdnumargs(c, ce) + if not downtmp: bomb("`close' when not open") + cleanup() + +def cmd_stop(c, ce): + global downtmp + cmdnumargs(c, ce, 1) + if not downtmp: bomb("`stop' when not open") + caller.hook_stop() + cleanup() + +def down_python_script(gobody, functions=''): + # Many things are made much harder by the inability of + # dchroot, ssh, et al, to cope without mangling the arguments. + # So we run a sub-python on the testbed and feed it a script + # on stdin. The sub-python decodes the arguments. + + script = ( "import urllib\n" + "import os\n" + "def setfd(fd,fnamee,write,mode=0666):\n" + " fname = urllib.unquote(fnamee)\n" + " if write: rw = os.O_WRONLY|os.O_CREAT\n" + " else: rw = os.O_RDONLY\n" + " nfd = os.open(fname, rw, mode)\n" + " os.dup2(nfd,fd)\n" + + functions + + "def go():\n" ) + script += ( " os.environ['TMPDIR']= urllib.unquote('%s')\n" % + urllib.quote(downtmp) ) + script += ( " os.chdir(os.environ['TMPDIR'])\n" ) + script += ( gobody + + "go()\n" ) + + debug("+P ...\n"+script) + + scripte = urllib.quote(script) + cmdl = down + ['python','-c', + "'import urllib; s = urllib.unquote(%s); exec s'" % + ('"%s"' % scripte)] + return cmdl + +def cmd_execute(c, ce): + cmdnumargs(c, ce, 5) + gobody = " import sys\n" + for ioe in range(3): + gobody += " setfd(%d,'%s',%d)\n" % ( + ioe, ce[ioe+2], ioe>0 ) + gobody += " os.chdir(urllib.unquote('" + ce[5] +"'))\n" + gobody += " cmd = '%s'\n" % ce[1] + gobody += (" cmd = cmd.split(',')\n" + " cmd = map(urllib.unquote, cmd)\n" + " c0 = cmd[0]\n" + " if '/' in c0:\n" + " if not os.access(c0, os.X_OK):\n" + " status = os.stat(c0)\n" + " mode = status.st_mode | 0111\n" + " os.chmod(c0, mode)\n" + " try: os.execvp(c0, cmd)\n" + " except OSError, e:\n" + " print >>sys.stderr, \"%s: %s\" % (\n" + " (c0, os.strerror(e.errno)))\n" + " os._exit(127)\n") + cmdl = down_python_script(gobody) + + (status, out) = execute_raw('sub-python', None, cmdl, + stdin=devnull_read, stderr=subprocess.PIPE) + if out: bomb("sub-python unexpected produced stdout" + " visible to us `%s'" % out) + return [`status`] + +def copyupdown(c, ce, upp): + cmdnumargs(c, ce, 2) + isrc = 0 + idst = 1 + ilocal = 0 + upp + iremote = 1 - upp + wh = ce[0] + sd = c[1:] + sde = ce[1:] + if not sd[0] or not sd[1]: + bomb("%s paths must be nonempty" % wh) + dirsp = sd[0][-1]=='/' + functions = "import errno\n" + if dirsp != (sd[1][-1]=='/'): + bomb("% paths must agree about directoryness" + " (presence or absence of trailing /)" % wh) + localfd = None + deststdout = devnull_read + srcstdin = devnull_read + preexecfns = [None, None] + if not dirsp: + modestr = '' + if upp: + deststdout = file(sd[idst], 'w') + else: + srcstdin = file(sd[isrc], 'r') + status = os.fstat(srcstdin.fileno()) + if status.st_mode & 0111: modestr = ',0777' + gobody = " setfd(%s,'%s',%s%s)\n" % ( + 1-upp, sde[iremote], not upp, modestr) + gobody += " os.execvp('cat', ['cat'])\n" + localcmdl = ['cat'] + else: + gobody = " dir = urllib.unquote('%s')\n" % sde[iremote] + if upp: + try: os.mkdir(sd[ilocal]) + except OSError, oe: + if oe.errno != errno.EEXIST: raise + else: + gobody += (" try: os.mkdir(dir)\n" + " except OSError, oe:\n" + " if oe.errno != errno.EEXIST: raise\n") + gobody +=( " os.chdir(dir)\n" + " tarcmd = 'tar -f -'.split()\n") + localcmdl = 'tar -f -'.split() + taropts = [None, None] + taropts[isrc] = '-c .' + taropts[idst] = '-p -x --no-same-owner' + gobody += " tarcmd += '%s'.split()\n" % taropts[iremote] + localcmdl += ['-C',sd[ilocal]] + localcmdl += taropts[ilocal].split() + gobody += " os.execvp('tar', tarcmd)\n"; + + downcmdl = down_python_script(gobody, functions) + + if upp: cmdls = (downcmdl, localcmdl) + else: cmdls = (localcmdl, downcmdl) + + debug(`["cmdls", `cmdls`]`) + debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`) + + subprocs = [None,None] + debug(" +< %s" % string.join(cmdls[0])) + subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin, + stdout=subprocess.PIPE, preexec_fn=preexecfns[0]) + debug(" +> %s" % string.join(cmdls[1])) + subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout, + stdout=deststdout, preexec_fn=preexecfns[1]) + for sdn in [1,0]: + status = subprocs[sdn].wait() + if status: bomb("%s %s failed, status %d" % + (wh, ['source','destination'][sdn], status)) + +def cmd_copydown(c, ce): copyupdown(c, ce, False) +def cmd_copyup(c, ce): copyupdown(c, ce, True) + +def command(): + sys.stdout.flush() + ce = sys.stdin.readline() + if not ce: bomb('end of file - caller quit?') + ce = ce.rstrip().split() + c = map(urllib.unquote, ce) + if not c: bomb('empty commands are not permitted') + debug('executing '+string.join(ce)) + try: f = globals()['cmd_'+c[0]] + except KeyError: bomb("unknown command `%s'" % ce[0]) + r = f(c, ce) + if not r: r = [] + r.insert(0, 'ok') + ru = map(urllib.quote, r) + print string.join(ru) + +def cleanup(): + global downtmp, cleaning + cleaning = True + if downtmp: caller.hook_cleanup() + cleaning = False + downtmp = False + +def error_cleanup(): + try: + ok = False + try: + cleanup() + ok = True + except Quit, q: + print >> sys.stderr, q.m + except: + print >> sys.stderr, "Unexpected cleanup error:" + traceback.print_exc() + print >> sys.stderr, '' + if not ok: + print >> sys.stderr, ("while cleaning up" + " because of another error:") + except: + pass + +def prepare(): + global downtmp, cleaning + downtmp = None + signal_list = [ signal.SIGHUP, signal.SIGTERM, + signal.SIGINT, signal.SIGPIPE ] + def sethandlers(f): + for signum in signal_list: signal.signal(signum, f) + def handler(sig, *any): + sethandlers(signal.SIG_DFL) + cleanup() + os.kill(os.getpid(), sig) + sethandlers(handler) + +def mainloop(): + try: + while True: command() + except Quit, q: + error_cleanup() + if q.m: print >> sys.stderr, q.m + sys.exit(q.ec) + except: + error_cleanup() + print >> sys.stderr, "Unexpected error:" + traceback.print_exc() + sys.exit(16) + +def main(): + ok() + prepare() + mainloop() diff --git a/virt-chroot/adt-virt-chroot b/virt-chroot/adt-virt-chroot index e4f14d1..e5651e1 100755 --- a/virt-chroot/adt-virt-chroot +++ b/virt-chroot/adt-virt-chroot @@ -24,26 +24,14 @@ import sys import os -import string -import urllib -import signal -import subprocess -import traceback -from optparse import OptionParser - -devnull_read = file('/dev/null','r') - -debuglevel = None - -class Quit: - def __init__(q,ec,m): q.ec = ec; q.m = m -def bomb(m): - raise Quit(12, "adt-virt-chroot: failure: %s" % m) +try: our_base = os.environ['AUTOPKGTEST_BASE'] +except KeyError: our_base = '/usr/share/autopkgtest'; +sys.path.insert(1, our_base+'/python') -def debug(m): - if not debuglevel: return - print >> sys.stderr, "adt-virt-chroot: debug:", m +import string +from optparse import OptionParser +import VirtSubproc as vsp def parse_args(): global down, debuglevel @@ -59,7 +47,7 @@ def parse_args(): (opts,args) = parser.parse_args() if len(args) != 1: pe("need exactly one arg, chroot specification") - debuglevel = opts.debug + vsp.debuglevel = opts.debug chroot_arg = args[0] if not chroot_arg: pe("chroot specification may not be empty") @@ -71,272 +59,21 @@ def parse_args(): if opts.gain_root != None: down = opts.gain_root.split() + down - debug("down = %s" % string.join(down)) - -def ok(): print 'ok' - -def cmdnumargs(c, ce, nargs=0): - if len(c) == nargs + 1: return - bomb("wrong number of arguments to command `%s'" % ce[0]) - -def cmd_capabilities(c, ce): - cmdnumargs(c, ce) + vsp.debug("down = %s" % string.join(down)) + vsp.down = down -def cmd_quit(c, ce): - cmdnumargs(c, ce) - raise Quit(0, '') - -def execute_raw(what, instr, *popenargs, **popenargsk): - debug(" ++ %s" % string.join(popenargs[0])) - sp = subprocess.Popen(*popenargs, **popenargsk) - if instr is None: popenargsk['stdin'] = devnull_read - (out, err) = sp.communicate(instr) - if err: bomb("%s unexpectedly produced stderr output `%s'" % - (what, err)) - status = sp.wait() - return (status, out) - -def execute(cmd_string, cmd_list=[], downp=False, outp=False): - cmdl = cmd_string.split() - - if downp: perhaps_down = down - else: downp = [] - - if outp: stdout = subprocess.PIPE - else: stdout = None - - cmd = perhaps_down + cmdl + cmd_list - (status, out) = execute_raw(cmdl[0], None, cmd, stdout=stdout) - - if status: bomb("%s%s failed (exit status %d)" % - ((downp and "(down) " or ""), cmdl[0], status)) - - if outp and out and out[-1]=='\n': out = out[:-1] - return out - -def cmd_open(c, ce): +def hook_open(): global downtmp - cmdnumargs(c, ce) - if downtmp: bomb("`open' when already open") - execute('true', downp=True) - downtmp = execute('mktemp -t -d', downp=True, outp=True) - return [downtmp] + vsp.execute('true', downp=True) + downtmp = vsp.execute('mktemp -t -d', downp=True, outp=True) + return downtmp -def cmd_close(c, ce): - global downtmp - cmdnumargs(c, ce) - if not downtmp: bomb("`close' when not open") - cleanup() - -def cmd_stop(c, ce): - global downtmp - cmdnumargs(c, ce, 1) - if not downtmp: bomb("`stop' when not open") - execute('rm -rf --', c[1:2]) +def hook_stop(): + vsp.execute('rm -rf --', c[1:2]) os.mkdir(c[1]) - cleanup() - -def down_python_script(gobody, functions=''): - # Many things are made much harder by dchroot's inability to - # cope without mangling the arguments. So we run a - # sub-python on the testbed and feed it a script on stdin. - # The sub-python decodes the arguments. - - script = ( "import urllib\n" - "import os\n" - "def setfd(fd,fnamee,write,mode=0666):\n" - " fname = urllib.unquote(fnamee)\n" - " if write: rw = os.O_WRONLY|os.O_CREAT\n" - " else: rw = os.O_RDONLY\n" - " nfd = os.open(fname, rw, mode)\n" - " os.dup2(nfd,fd)\n" - + functions + - "def go():\n" ) - script += ( " os.environ['TMPDIR']= urllib.unquote('%s')\n" % - urllib.quote(downtmp) ) - script += ( " os.chdir(os.environ['TMPDIR'])\n" ) - script += ( gobody + - "go()\n" ) - - debug("+P ...\n"+script) - - scripte = urllib.quote(script) - cmdl = down + ['python','-c', - "'import urllib; s = urllib.unquote(%s); exec s'" % - ('"%s"' % scripte)] - return cmdl - -def cmd_execute(c, ce): - cmdnumargs(c, ce, 5) - gobody = " import sys\n" - for ioe in range(3): - gobody += " setfd(%d,'%s',%d)\n" % ( - ioe, ce[ioe+2], ioe>0 ) - gobody += " os.chdir(urllib.unquote('" + ce[5] +"'))\n" - gobody += " cmd = '%s'\n" % ce[1] - gobody += (" cmd = cmd.split(',')\n" - " cmd = map(urllib.unquote, cmd)\n" - " c0 = cmd[0]\n" - " if '/' in c0:\n" - " if not os.access(c0, os.X_OK):\n" - " status = os.stat(c0)\n" - " mode = status.st_mode | 0111\n" - " os.chmod(c0, mode)\n" - " try: os.execvp(c0, cmd)\n" - " except OSError, e:\n" - " print >>sys.stderr, \"%s: %s\" % (\n" - " (c0, os.strerror(e.errno)))\n" - " os._exit(127)\n") - cmdl = down_python_script(gobody) - - (status, out) = execute_raw('sub-python', None, cmdl, - stdin=devnull_read, stderr=subprocess.PIPE) - if out: bomb("sub-python unexpected produced stdout" - " visible to us `%s'" % out) - return [`status`] - -def copyupdown(c, ce, upp): - cmdnumargs(c, ce, 2) - isrc = 0 - idst = 1 - ilocal = 0 + upp - iremote = 1 - upp - wh = ce[0] - sd = c[1:] - sde = ce[1:] - if not sd[0] or not sd[1]: - bomb("%s paths must be nonempty" % wh) - dirsp = sd[0][-1]=='/' - functions = "import errno\n" - if dirsp != (sd[1][-1]=='/'): - bomb("% paths must agree about directoryness" - " (presence or absence of trailing /)" % wh) - localfd = None - deststdout = devnull_read - srcstdin = devnull_read - preexecfns = [None, None] - if not dirsp: - modestr = '' - if upp: - deststdout = file(sd[idst], 'w') - else: - srcstdin = file(sd[isrc], 'r') - status = os.fstat(srcstdin.fileno()) - if status.st_mode & 0111: modestr = ',0777' - gobody = " setfd(%s,'%s',%s%s)\n" % ( - 1-upp, sde[iremote], not upp, modestr) - gobody += " os.execvp('cat', ['cat'])\n" - localcmdl = ['cat'] - else: - gobody = " dir = urllib.unquote('%s')\n" % sde[iremote] - if upp: - try: os.mkdir(sd[ilocal]) - except OSError, oe: - if oe.errno != errno.EEXIST: raise - else: - gobody += (" try: os.mkdir(dir)\n" - " except OSError, oe:\n" - " if oe.errno != errno.EEXIST: raise\n") - gobody +=( " os.chdir(dir)\n" - " tarcmd = 'tar -f -'.split()\n") - localcmdl = 'tar -f -'.split() - taropts = [None, None] - taropts[isrc] = '-c .' - taropts[idst] = '-p -x --no-same-owner' - gobody += " tarcmd += '%s'.split()\n" % taropts[iremote] - localcmdl += ['-C',sd[ilocal]] - localcmdl += taropts[ilocal].split() - gobody += " os.execvp('tar', tarcmd)\n"; - downcmdl = down_python_script(gobody, functions) - - if upp: cmdls = (downcmdl, localcmdl) - else: cmdls = (localcmdl, downcmdl) - - debug(`["cmdls", `cmdls`]`) - debug(`["srcstdin", `srcstdin`, "deststdout", `deststdout`, "devnull_read", devnull_read]`) - - subprocs = [None,None] - debug(" +< %s" % string.join(cmdls[0])) - subprocs[0] = subprocess.Popen(cmdls[0], stdin=srcstdin, - stdout=subprocess.PIPE, preexec_fn=preexecfns[0]) - debug(" +> %s" % string.join(cmdls[1])) - subprocs[1] = subprocess.Popen(cmdls[1], stdin=subprocs[0].stdout, - stdout=deststdout, preexec_fn=preexecfns[1]) - for sdn in [1,0]: - status = subprocs[sdn].wait() - if status: bomb("%s %s failed, status %d" % - (wh, ['source','destination'][sdn], status)) - -def cmd_copydown(c, ce): copyupdown(c, ce, False) -def cmd_copyup(c, ce): copyupdown(c, ce, True) - -def command(): - sys.stdout.flush() - ce = sys.stdin.readline() - if not ce: bomb('end of file - caller quit?') - ce = ce.rstrip().split() - c = map(urllib.unquote, ce) - if not c: bomb('empty commands are not permitted') - debug('executing '+string.join(ce)) - try: f = globals()['cmd_'+c[0]] - except KeyError: bomb("unknown command `%s'" % ce[0]) - r = f(c, ce) - if not r: r = [] - r.insert(0, 'ok') - ru = map(urllib.quote, r) - print string.join(ru) - -def cleanup(): - global downtmp, cleaning - cleaning = True - if downtmp: execute('rm -rf --', [downtmp], downp=True) - cleaning = False - downtmp = False - -def error_cleanup(): - try: - ok = False - try: - cleanup() - ok = True - except Quit, q: - print >> sys.stderr, q.m - except: - print >> sys.stderr, "Unexpected cleanup error:" - traceback.print_exc() - print >> sys.stderr, '' - if not ok: - print >> sys.stderr, ("while cleaning up" - " because of another error:") - except: - pass - -def prepare(): - global downtmp, cleaning - downtmp = None - signal_list = [ signal.SIGHUP, signal.SIGTERM, - signal.SIGINT, signal.SIGPIPE ] - def sethandlers(f): - for signum in signal_list: signal.signal(signum, f) - def handler(sig, *any): - sethandlers(signal.SIG_DFL) - cleanup() - os.kill(os.getpid(), sig) - sethandlers(handler) +def hook_cleanup(): + vsp.execute('rm -rf --', [downtmp], downp=True) parse_args() -ok() -prepare() - -try: - while True: command() -except Quit, q: - error_cleanup() - if q.m: print >> sys.stderr, q.m - sys.exit(q.ec) -except: - error_cleanup() - print >> sys.stderr, "Unexpected error:" - traceback.print_exc() - sys.exit(16) +vsp.main()