chiark / gitweb /
* adt-xenlvm-with-testbed: sleep 1 after xm destroy, which is racy.
[autopkgtest.git] / runner / adt-run
index a01649f319082170257b20e50fda6ead567c214f..7787e5761971fb566993e8e87bd40058ed5d01b4 100755 (executable)
@@ -3,7 +3,7 @@
 # adt-run is part of autopkgtest
 # autopkgtest is a tool for testing Debian binary packages
 #
-# autopkgtest is Copyright (C) 2006 Canonical Ltd.
+# autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -45,24 +45,130 @@ signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
 tmpdir = None          # pathstring on host
 testbed = None         # Testbed
 errorcode = 0          # exit status that we are going to use
+timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
 binaries = None                # Binaries (.debs we have registered)
+build_essential = ["build-essential"]
+
+#---------- output handling
+#
+#  There are at least the following kinds of output:
+#
+#   1. stderr output which consists of
+#   1a. our errors
+#   1b. stuff printed to stderr by the virtualisation server
+#   1c. stuff printed to stderr by our short-lived subprocesseses
+#           which we don't expect to fail
+#
+#   2. trace information, which consists of
+#   2a. our trace information from calls to debug()
+#   2b. progress information and stderr output from
+#        general scripts we run on the host
+#   2c. progress information and stderr output from things
+#        we run on the testbed including builds
+#   2d. stderr and stdout output from actual tests
+#
+#   xxxx
+#   3. actual test results (printed to our stdout)
+#
+# Cloning of 1a and 2a, where necessary, is done by us writing
+# the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
+# is done by forking off a copy of ourselves to do plumbing,
+# which copy we wait for at the appropriate point.
+
+class DummyOpts:
+ def __init__(do): do.debuglevel = 0
+
+opts = DummyOpts()
+trace_stream = None
+summary_stream = None
+
+def pstderr(m):
+       print >>sys.stderr, m
+       if trace_stream is not None: print >>trace_stream, m
+
+def debug(m, minlevel=0):
+       if opts.debuglevel < minlevel: return
+       if opts.quiet and trace_stream is None: return
+       p = 'adt-run: trace'
+       if minlevel: p += `minlevel`
+       p += ': '
+       for l in m.rstrip('\n').split('\n'):
+               s = p + l
+               if not opts.quiet: print >>sys.stderr, s
+               if trace_stream is not None: print >>trace_stream, s
+
+def debug_file(hp, minlevel=0):
+       if opts.debuglevel < minlevel: return
+       def do_copy(stream, what):
+               rc = subprocess.call(['cat',hp], stdout=stream)
+               if rc: bomb('cat failed copying data from %s'
+                           ' to %s, exit code %d' % (hp, what, rc))
+       if not opts.quiet: do_copy(sys.stderr, 'stderr')
+       if trace_stream is not None: do_copy(trace_stream, 'trace log')
+
+class Errplumb:
+ def __init__(ep, critical=False):
+       to_stderr = critical or not opts.quiet
+       count = to_stderr + (trace_stream is not None)
+       if count == 0:
+               ep.stream = open('/dev/null','w')
+               ep._sp = None
+       elif count == 1:
+               if to_stderr: ep.stream = os.dup(2)
+               else: ep.stream = trace_stream
+               ep._sp = None
+       else:
+               ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
+                       stdin=subprocess.PIPE, stdout=trace_stream,
+                       close_fds=True)
+               ep.stream = ep._sp.stdin
+ def wait(ep):
+       if ep._sp is None: return
+       if type(ep.stream) == type(2):
+               os.close(ep.stream)
+               ep.stream = ()
+       ep._sp.stdin.close()
+       rc = ep._sp.wait()
+       if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
+       ep._sp = None
+
+def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
+       if dbg is not None:
+               if isinstance(dbg,tuple): (what,script) = dbg
+               else: (what,script) = (dbg,None)
+               debug_subprocess(what, cmdl, script=script)
+       ep = Errplumb(critical)
+       running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
+       output = running.communicate()[0]
+       rc = running.wait()
+       ep.wait()
+       return (rc, output)
+
+def psummary(m):
+       if summary_stream is not None: print >>summary_stream, m
+
+def preport(m):
+       print m
+       sys.stdout.flush()
+       if trace_stream is not None: print >>trace_stream, m
+       psummary(m)
+
+def report(tname, result):
+       preport('%-20s %s' % (tname, result))
 
 #---------- errors we define
 
 class Quit:
        def __init__(q,ec,m): q.ec = ec; q.m = m
 
-def bomb(m, se=''):
-       print >>sys.stderr, se
+def bomb(m):
        raise Quit(20, "unexpected error: %s" % m)
 
-def badpkg(m, se=''):
-       print >>sys.stderr, se
-       print 'blame: ', ' '.join(testbed.blamed)
+def badpkg(m):
+       preport('blame: ' + ' '.join(testbed.blamed))
+       preport('badpkg: ' + m)
        raise Quit(12, "erroneous package: %s" % m)
 
-def report(tname, result): print '%-20s %s' % (tname, result)
-
 class Unsupported:
  def __init__(u, lno, m):
        if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
@@ -72,21 +178,19 @@ class Unsupported:
        errorcode != 2
        report(tname, 'SKIP %s' % u.m)
 
-def debug(m):
-       global opts
-       if not opts.debug: return
-       for l in m.rstrip('\n').split('\n'):
-               print >>sys.stderr, 'atd-run: debug:', l
+#---------- convenience function
 
 def mkdir_okexist(pathname, mode=02755):
        try:
                os.mkdir(pathname, mode)
-       except OSError, oe:
+       except (IOError,OSError), oe:
                if oe.errno != errno.EEXIST: raise
 
 def rmtree(what, pathname):
-       debug('/ %s rmtree %s' % (what, pathname))
-       shutil.rmtree(pathname)
+       debug('/ %s rmtree %s' % (what, pathname), 2)
+       try: shutil.rmtree(pathname)
+       except (IOError,OSError), oe:
+               if oe.errno != errno.EEXIST: raise
 
 def debug_subprocess(what, cmdl=None, script=None):
        o = '$ '+what+':'
@@ -97,11 +201,12 @@ def debug_subprocess(what, cmdl=None, script=None):
                        ol.append(x.    replace('\\','\\\\').
                                        replace(' ','\\ ')      )
                o += ' '+ ' '.join(ol)
-       if script is not None:
-               o += '\n'
+       debug(o)
+       if script is not None and opts.debuglevel >= 1:
+               o = ''
                for l in script.rstrip('\n').split('\n'):
                        o += '$     '+l+'\n'
-       debug(o)
+               debug(o,1)
 
 def flatten(l):
        return reduce((lambda a,b: a + b), l, []) 
@@ -123,6 +228,8 @@ class AutoFile:
        p.spec = None
        p.dir = ''
 
+ def __repr__(p):
+       return "<AF@%s>" % p.__str__()
  def __str__(p):
        def ptbp(tbp):
                if p.path[tbp] is None: return '-'+p.dir
@@ -130,7 +237,8 @@ class AutoFile:
                else: return p.path[tbp]+p.dir+'!'
        out = p.what
        if p.spec is not None:
-               if p.spec_tbp: out += '#'
+               if not hasattr(p,'spec_tb'): out += '~'
+               elif p.spec_tbp: out += '#'
                else: out += '='
                out += p.spec
        out += ':'
@@ -141,7 +249,8 @@ class AutoFile:
 
  def _wrong(p, how):
        xtra = ''
-       if p.spec is not None: xtra = ' spec[%s]=%s' % (p.spec, p.spec_tb)
+       if p.spec is not None:
+               xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
        raise ("internal error: %s (%s)" % (how, str(p)))
 
  def _ensure_path(p, tbp):
@@ -167,9 +276,9 @@ class AutoFile:
                                'test -d "$1" || mkdir -p "$1"',
                                'x', p.path[tbp]]
                        tf_what = urllib.quote(p.what).replace('/','%2F')
-                       (rc,se) = testbed.execute('mkdir-'+tf_what, cmdl)
+                       rc = testbed.execute('mkdir-'+tf_what, cmdl)
                        if rc: bomb('failed to create directory %s' %
-                               p.path[tbp], se)
+                               p.path[tbp])
 
        p.file[tbp] = p.path[tbp]
        return p.path[tbp]
@@ -194,7 +303,7 @@ class AutoFile:
        p._debug('invalidated %s' % 'HT'[tbp])
 
  def _debug(p, m):
-       debug('/ %s#%x: %s' % (p.what, id(p), m))
+       debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 
  def _constructed(p):
        p._debug('constructed: '+str(p))
@@ -204,7 +313,7 @@ class AutoFile:
        for tbp in [False,True]:
         for pf in [p.path, p.file]:
                if pf[tbp] is None: continue
-               if not pf[tbp]: bomb('empty path specified for '+p.what)
+               if not pf[tbp]: bomb('empty path specified for %s' % p.what)
                if p.dir and pf[tbp].endswith('/'):
                        pf[tbp] = pf[tbp].rstrip('/')
                        if not pf[tbp]: pf[tbp] = '/'
@@ -294,9 +403,13 @@ class Action:
        a.af = af
        a.ah = arghandling
        a.what = what
+       a.missing_tests_control = False
+ def __repr__(a):
+       return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 
 def parse_args():
-       global opts
+       global opts, timeouts
+       global n_non_actions # argh, stupid python scoping rules
        usage = "%prog <options> --- <virt-server>..."
        parser = OptionParser(usage=usage)
        pa = parser.add_option
@@ -308,7 +421,8 @@ def parse_args():
                'deb_forbuilds': 'auto',
                'deb_fortests': 'auto',
                'tb': False,
-               'override_control': None
+               'override_control': None,
+               'set_lang': 'C'
        }
        initial_arghandling = arghandling.copy()
        n_non_actions = 0
@@ -317,7 +431,7 @@ def parse_args():
        # actions (ie, test sets to run, sources to build, binaries to use):
 
        def cb_action(op,optstr,value,parser, long,kindpath,is_act):
-               print >>sys.stderr, "cb_action", is_act
+               global n_non_actions
                parser.largs.append((value,kindpath))
                n_non_actions += not(is_act)
 
@@ -326,7 +440,10 @@ def parse_args():
                        nargs=1, type='string',
                        callback_args=(long,kindpath,is_act), help=help)
 
-       pa_action('build-tree',         'TREE', '@/',
+       pa_action('built-tree',         'TREE', '@/',
+               help='run tests from build tree TREE')
+
+       pa_action('unbuilt-tree',       'TREE', '@//',
                help='run tests from build tree TREE')
 
        pa_action('source',             'DSC', '@.dsc',
@@ -337,6 +454,15 @@ def parse_args():
               help='use binary package DEB according'
                    ' to most recent --binaries-* settings')
 
+       def cb_actnoarg(op,optstr,value,parser, largsentry):
+               parser.largs.append(largsentry)
+       pa('','--instantiate', action='callback', callback=cb_actnoarg,
+               callback_args=((None, ('instantiate',)),),
+               help='instantiate testbed now (during testing phase)'
+                    ' and install packages'
+                    ' selected for automatic installation, even'
+                    ' if this might apparently not be required otherwise')
+
        pa_action('override-control',   'CONTROL', ('control',), is_act=0,
               help='run tests from control file CONTROL instead,'
                    ' (applies to next test suite only)')
@@ -384,7 +510,6 @@ def parse_args():
                     ' according to most recent --binaries-* settings')
        pa_setah('--no-built-binaries', ['dsc_filter'], '_',
                help='from subsequent sources, do not use any binaries')
-
        #---- binary package processing settings:
 
        def pa_setahbins(long,toset,how):
@@ -406,7 +531,7 @@ def parse_args():
 
        def cb_path(op,optstr,value,parser, constructor,long,dir):
                name = long.replace('-','_')
-               af = constructor(arghandling['tb'], value, long, dir)
+               af = constructor(long, value,arghandling['tb'])
                setattr(parser.values, name, af)
 
        def pa_path(long, constructor, help, dir=False):
@@ -418,15 +543,34 @@ def parse_args():
        pa_path('output-dir', OutputDir, dir=True,
                help='write stderr/out files in PATH')
 
+       pa('--leave-lang', dest='set_lang', action='store_false',
+               help="leave LANG on testbed set to testbed's default")
+       pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
+               help='set LANG on testbed to LANGVAL', default='C')
+
        pa('','--tmp-dir',              type='string', dest='tmpdir',
                help='write temporary files to TMPDIR, emptying it'
                     ' beforehand and leaving it behind at the end')
+       pa('','--log-file',             type='string', dest='logfile',
+               help='write the log LOGFILE, emptying it beforehand,'
+                    ' instead of using OUTPUT-DIR/log or TMPDIR/log')
+       pa('','--summary-file',         type='string', dest='summary',
+               help='write a summary report to SUMMARY,'
+                    ' emptying it beforehand')
+
+       for k in timeouts.keys():
+               pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
+                       metavar='T', help='set %s timeout to T')
+       pa('','--timeout-factor',       type='float', dest='timeout_factor',
+                       metavar='FACTOR', default=1.0,
+                       help='multiply all default timeouts by FACTOR')
 
        pa('','--user',                 type='string', dest='user',
                help='run tests as USER (needs root on testbed)')
        pa('','--gain-root',            type='string', dest='gainroot',
                help='prefix debian/rules binary with GAINROOT')
-       pa('-d', '--debug', action='store_true', dest='debug');
+       pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
+       pa('-d', '--debug', action='count', dest='debuglevel', default=0);
        pa('','--gnupg-home',           type='string', dest='gnupghome',
                default='~/.autopkgtest/gpg',
                help='use GNUPGHOME rather than ~/.autopkgtest (for'
@@ -457,6 +601,11 @@ def parse_args():
        if n_non_actions >= len(parser.largs):
                parser.error('nothing to do specified')
 
+       for k in timeouts.keys():
+               t = getattr(opts,'timeout_'+k)
+               if t is None: t = timeouts[k] * opts.timeout_factor
+               timeouts[k] = int(t)
+
        arghandling = initial_arghandling
        opts.actions = []
        ix = 0
@@ -477,22 +626,27 @@ def parse_args():
                if type(kindpath) is tuple:             kind = kindpath[0]
                elif kindpath.endswith('.deb'):         kind = 'deb'
                elif kindpath.endswith('.dsc'):         kind = 'dsc'
+               elif kindpath.endswith('//'):
+                       kind = 'ubtree'
+                       constructor = InputDir
                elif kindpath.endswith('/'):
                        kind = 'tree'
                        constructor = InputDir
-               else: parser.error("do not know how to handle filename \`%s';"
-                       " specify --source --binary or --build-tree")
+               else: parser.error("do not know how to handle filename `%s';"
+                       " specify --source --binary or --build-tree" %
+                       kindpath)
 
                what = '%s%s' % (kind,ix); ix += 1
 
                if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
                else: fwhatx = '-'+kind
 
-               af = constructor(what+fwhatx, pathstr, arghandling['tb'])
+               if pathstr is None: af = None
+               else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
                opts.actions.append(Action(kind, af, arghandling, what))
 
-def finalise_options():
-       global opts, tb, tmpdir
+def setup_trace():
+       global trace_stream, tmpdir, summary_stream
 
        if opts.tmpdir is not None:
                rmtree('tmpdir(specified)',opts.tmpdir)
@@ -502,6 +656,21 @@ def finalise_options():
                assert(tmpdir is None)
                tmpdir = tempfile.mkdtemp()
 
+       if opts.logfile is None:
+               if opts.output_dir is not None and opts.output_dir.spec_tbp:
+                       opts.logfile = opts.output_dir.spec + '/log'
+               elif opts.tmpdir is not None:
+                       opts.logfile = opts.tmpdir + '/log'
+       if opts.logfile is not None:
+               trace_stream = open(opts.logfile, 'w', 0)
+       if opts.summary is not None:
+               summary_stream = open(opts.summary, 'w', 0)
+
+       debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
+
+def finalise_options():
+       global opts, tb, build_essential
+
        if opts.user is None and 'root-on-testbed' not in testbed.caps:
                opts.user = ''
 
@@ -519,7 +688,7 @@ def finalise_options():
 
        if opts.user:
                if 'root-on-testbed' not in testbed.caps:
-                       print >>sys.stderr, ("warning: virtualisation"
+                       pstderr("warning: virtualisation"
                                " system does not offer root on testbed,"
                                " but --user option specified: failure likely")
                opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
@@ -531,6 +700,7 @@ def finalise_options():
                if (opts.user or
                    'root-on-testbed' not in testbed.caps):
                        opts.gainroot = 'fakeroot'
+                       build_essential += ['fakeroot']
 
        if opts.gnupghome.startswith('~/'):
                try: home = os.environ['HOME']
@@ -553,17 +723,18 @@ class Testbed:
        tb.blamed = []
        tb._ephemeral = []
        tb._debug('init')
- def _debug(tb, m):
-       debug('** '+m)
+       tb._need_reset_apt = False
+ def _debug(tb, m, minlevel=0):
+       debug('** '+m, minlevel)
  def start(tb):
        tb._debug('start')
        p = subprocess.PIPE
        debug_subprocess('vserver', opts.vserver)
+       tb._errplumb = Errplumb(True)
        tb.sp = subprocess.Popen(opts.vserver,
-               stdin=p, stdout=p, stderr=None)
+               stdin=p, stdout=p, stderr=tb._errplumb.stream)
        tb.expect('ok')
        tb.caps = tb.commandr('capabilities')
-       tb._need_reset_apt = False
  def stop(tb):
        tb._debug('stop')
        tb.close()
@@ -576,6 +747,7 @@ class Testbed:
                ec = tb.sp.wait()
        if ec:
                tb.bomb('testbed gave exit status %d after quit' % ec)
+       tb._errplumb.wait()
  def open(tb):
        tb._debug('open, scratch=%s' % tb.scratch)
        if tb.scratch is not None: return
@@ -588,11 +760,11 @@ class Testbed:
  def reset_apt(tb):
        if not tb._need_reset_apt: return
        what = 'aptget-update-reset'
-       cmdl = ['apt-get','-qy','update']
-       (rc,se) = tb.execute(what, cmdl)
+       cmdl = ['sh','-c','apt-get -qy update 2>&1']
+       rc = tb.execute(what, cmdl, kind='install')
        if rc:
-               print >>sys.stderr, se, ("\n" "warning: failed to restore"
-                               " testbed apt cache, exit code %d" % rc)
+               pstderr("\n" "warning: failed to restore"
+                       " testbed apt cache, exit code %d" % rc)
        tb._need_reset_apt = False
  def close(tb):
        tb._debug('close, scratch=%s' % tb.scratch)
@@ -600,22 +772,28 @@ class Testbed:
        tb.scratch = None
        if tb.sp is None: return
        tb.command('close')
- def prepare(tb, deps_new):
-       tb._debug('prepare, modified=%s, deps_processed=%s, deps_new=%s' %
-               (tb.modified, tb.deps_processed, deps_new))
+ def prepare1(tb, deps_new):
+       tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
+               (tb.modified, tb.deps_processed, deps_new), 1)
        if 'revert' in tb.caps and (tb.modified or
            [d for d in tb.deps_processed if d not in deps_new]):
+               for af in tb._ephemeral: af.read(False)
                tb._debug('reset **')
-               tb.command('reset')
+               tb.command('revert')
                tb.blamed = []
                for af in tb._ephemeral: af.invalidate(True)
-       binaries.publish()
        tb.modified = False
+ def prepare2(tb, deps_new):
+       tb._debug('prepare2, deps_new=%s' % deps_new, 1)
+       binaries.publish()
        tb._install_deps(deps_new)
+ def prepare(tb, deps_new):
+       tb.prepare1(deps_new)
+       tb.prepare2(deps_new)
  def register_ephemeral(tb, af):
        tb._ephemeral.append(af)
  def _install_deps(tb, deps_new):
-       tb._debug(' installing dependencies '+`deps_new`)
+       tb._debug(' installing dependencies '+`deps_new`, 1)
        tb.deps_processed = deps_new
        if not deps_new: return
        dstr = ', '.join(deps_new)
@@ -627,14 +805,14 @@ class Testbed:
                ]])
        cmdl = ['python','-c',script]
        what = 'install-deps'
-       debug_subprocess(what, cmdl, script=script)
-       (rc,se) = testbed.execute(what, cmdl)
-       if rc: badpkg('dependency install failed, exit code %d' % rc, se)
+       rc = testbed.execute(what+'-debinstall', cmdl, script=script,
+                               kind='install')
+       if rc: badpkg('dependency install failed, exit code %d' % rc)
  def needs_reset(tb):
-       tb._debug('needs_reset, previously=%s' % tb.modified)
+       tb._debug('needs_reset, previously=%s' % tb.modified, 1)
        tb.modified = True
  def blame(tb, m):
-       tb._debug('blame += %s' % m)
+       tb._debug('blame += %s' % m, 1)
        tb.blamed.append(m)
  def bomb(tb, m):
        tb._debug('bomb %s' % m)
@@ -642,14 +820,14 @@ class Testbed:
                tb.sp.stdout.close()
                tb.sp.stdin.close()
                ec = tb.sp.wait()
-               if ec: print >>sys.stderr, ('adt-run: testbed failing,'
-                       ' exit status %d' % ec)
+               if ec: pstderr('adt-run: testbed failing,'
+                               ' exit status %d' % ec)
        tb.sp = None
        raise Quit(16, 'testbed failed: %s' % m)
  def send(tb, string):
        tb.sp.stdin
        try:
-               debug('>> '+string)
+               debug('>> '+string, 2)
                print >>tb.sp.stdin, string
                tb.sp.stdin.flush()
                tb.lastsend = string
@@ -662,7 +840,7 @@ class Testbed:
        if not l: tb.bomb('unexpected eof from the testbed')
        if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
        l = l.rstrip('\n')
-       debug('<< '+l)
+       debug('<< '+l, 2)
        ll = l.split()
        if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
        if ll[0] != keyword:
@@ -693,24 +871,59 @@ class Testbed:
  def commandr1(tb, cmd, args=()):
        rl = tb.commandr(cmd, args, 1)
        return rl[0]
- def execute(tb, what, cmdargs,
+ def execute(tb, what, cmdl,
                si='/dev/null', so='/dev/null', se=None, cwd=None,
-               dump_fd=None):
+               script=False, tmpdir=None, kind='short'):
+       # Options for script:
+       #   False - do not call debug_subprocess, no synch. reporting required
+       #   None or string - call debug_subprocess with that value,
+       #                       plumb stderr through synchronously if possible
+       # Options for se:
+       #   None - usual Errplumb (output is of kind 2c)
+       #   string - path on testbed (output is of kind 2d)
+
+       timeout = timeouts[kind]
+
+       if script is not False: debug_subprocess(what, cmdl, script=script)
        if cwd is None: cwd = tb.scratch.write(True)
-       se_use = se
-       if se_use is None:
-               se_af = TemporaryFile('xerr-'+what)
-               se_use = se_af.write(True)
+
+       xdump = None
+       if se is None:
+               ep = Errplumb()
+               se_catch = TemporaryFile(what+'-xerr')
+               se_use = se_catch.write(True)
+               if not opts.quiet: xdump = 'debug=2-2'
+               elif trace_stream is not None:
+                       xdump = 'debug=2-%d' % trace_stream.fileno()
+       else:
+               ep = None
+               se_catch = None
+               se_use = se
+
        cmdl = [None,
-               ','.join(map(urllib.quote, cmdargs)),
+               ','.join(map(urllib.quote, cmdl)),
                si, so, se_use, cwd]
-       if dump_fd is not None: cmdl += ['debug=%d-%d' % (dump_fd,2)]
+
+       if timeout is not None and timeout > 0:
+               cmdl.append('timeout=%d' % timeout)
+
+       if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
+       if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
+       if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
+       if opts.set_lang is not False:
+               cmdl.append('env=LANG=%s' % opts.set_lang)
+
        rc = tb.commandr1('execute', cmdl)
        try: rc = int(rc)
        except ValueError: bomb("execute for %s gave invalid response `%s'"
                                        % (what,rc))
-       if se is not None: return rc
-       return (rc, file(se_af.read()).read())
+
+       if se_catch is not None:
+               debug_file(se_catch.read())
+       if ep is not None:
+               ep.wait()
+
+       return rc
 
 #---------- representation of test control files: Field*, Test, etc.
 
@@ -743,31 +956,48 @@ class Restriction:
  def __init__(r,rname,base): pass
 
 class Restriction_rw_build_tree(Restriction): pass
-class Restriction_rw_tests_tree(Restriction): pass
 class Restriction_breaks_testbed(Restriction):
  def __init__(r, rname, base):
        if 'revert' not in testbed.caps:
                raise Unsupported(f.lno,
-                       'Test breaks testbed but testbed cannot reset')
+                       'Test breaks testbed but testbed cannot revert')
+class Restriction_needs_root(Restriction):
+ def __init__(r, rname, base):
+       if 'root-on-testbed' not in testbed.caps:
+               raise Unsupported(f.lno,
+                       'Test needs root on testbed which is not available')
 
 class Field_Restrictions(FieldBase):
  def parse(f):
        for wle in f.words():
                (lno, rname) = wle
-               rname = rname.replace('-','_')
-               try: rclass = globals()['Restriction_'+rname]
+               nrname = rname.replace('-','_')
+               try: rclass = globals()['Restriction_'+nrname]
                except KeyError: raise Unsupported(lno,
                        'unknown restriction %s' % rname)
-               r = rclass(rname, f.base)
+               r = rclass(nrname, f.base)
+               f.base['restriction_names'].append(rname)
                f.base['restrictions'].append(r)
 
+class Field_Features(FieldIgnore):
+ def parse(f):
+       for wle in f.words():
+               (lno, fname) = wle
+               f.base['feature_names'].append(fname)
+               nfname = fname.replace('-','_')
+               try: fclass = globals()['Feature_'+nfname]
+               except KeyError: continue
+               ft = fclass(nfname, f.base)
+               f.base['features'].append(ft)
+
 class Field_Tests(FieldIgnore): pass
 
 class Field_Depends(FieldBase):
  def parse(f):
+       print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
        dl = map(lambda x: x.strip(),
-               flatten(map(lambda x: x.split(','), f.vl)))
-       re = regexp.compile('[^-.+:~]')
+               flatten(map(lambda (lno, v): v.split(','), f.vl)))
+       re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
        for d in dl:
                if re.search(d):
                        badpkg("Test Depends field contains dependency"
@@ -783,15 +1013,18 @@ class Field_Tests_directory(FieldBase):
 
 def run_tests(stanzas, tree):
        global errorcode, testbed
+       if stanzas == ():
+               report('*', 'SKIP no tests in this package')
+               errorcode |= 8
        for stanza in stanzas:
                tests = stanza[' tests']
                if not tests:
-                       report('*', 'SKIP no tests in this package')
+                       report('*', 'SKIP package has metadata but no tests')
                        errorcode |= 8
                for t in tests:
                        t.prepare()
                        t.run(tree)
-                       if 'breaks-testbed' in t.restrictions:
+                       if 'breaks-testbed' in t.restriction_names:
                                testbed.needs_reset()
                testbed.needs_reset()
 
@@ -820,45 +1053,80 @@ class Test:
        dn = []
        for d in t.depends:
                t._debug(' processing dependency '+d)
-               if not '*' in d:
+               if not '@' in d:
                        t._debug('  literal dependency '+d)
                        dn.append(d)
                else:
                        for (pkg,bin) in t.act.binaries:
-                               d = d.replace('*',pkg)
+                               d = d.replace('@',pkg)
                                t._debug('  synthesised dependency '+d)
                                dn.append(d)
        testbed.prepare(dn)
  def run(t, tree):
-       t._debug('running')
+       t._debug('[----------------------------------------')
        def stdouterr(oe):
                idstr = t.what + '-' + oe
-               if opts.output_dir is not None and opts.output_dir.tb:
+               if opts.output_dir is not None and opts.output_dir.spec_tbp:
                        use_dir = opts.output_dir
                else:
                        use_dir = testbed.scratch
                return RelativeOutputFile(idstr, use_dir, idstr)
 
-       t.act.work.write(True)
+       if hasattr(t.act,'work'): t.act.work.write(True)
+       tree.read(True)
 
        af = RelativeInputFile(t.what, tree, t.path)
        so = stdouterr('stdout')
        se = stdouterr('stderr')
-       rc = testbed.execute('test-'+t.what,
-               [opts.user_wrap(af.read(True))],
-               so=so.write(True), se=se.write(True), cwd=tree.read(True))
-                       
-       stab = os.stat(se.read())
+
+       tf = af.read(True)
+       tmpdir = None
+
+       rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
+       if rc: bomb('failed to chmod +x %s' % tf)
+
+       if 'needs-root' not in t.restriction_names and opts.user is not None:
+               tfl = ['su',opts.user,'-c',tf]
+               tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
+               script = 'rm -rf -- "$1"; mkdir -- "$1"'
+               if opts.user: script += '; chown %s "$1"' % opts.user
+               if 'rw-build-tree' in t.restriction_names:
+                       script += '; chown -R %s "$2"' % opts.user
+               rc = testbed.execute('mktmpdir-'+t.what,
+                       ['sh','-xec',script,'x',tmpdir,tree.read(True)])
+               if rc: bomb("could not create test tmpdir `%s', exit code %d"
+                               % (tmpdir, rc))
+       else:
+               tfl = [tf]
+
+       rc = testbed.execute('test-'+t.what, tfl,
+               so=so.write(True), se=se.write(True), cwd=tree.read(True),
+               tmpdir=tmpdir, kind='test')
+
+       so_read = so.read()
+       se_read = se.read()
+
+       t._debug(' - - - - - - - - - - results - - - - - - - - - -')
+       stab = os.stat(se_read)
        if stab.st_size != 0:
-               l = file(se.read()).readline()
+               l = open(se_read).readline()
                l = l.rstrip('\n \t\r')
-               if len(l) > 40: l = l[:40] + '...'
+               if len(l) > 35: l = l[:35] + '...'
                t.reportfail('status: %d, stderr: %s' % (rc, l))
+               t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
+               debug_file(se_read)
        elif rc != 0:
                t.reportfail('non-zero exit status %d' % rc)
        else:
                t.report('PASS')
 
+       stab = os.stat(so_read)
+       if stab.st_size != 0:
+               t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
+               debug_file(so_read)
+
+       t._debug('----------------------------------------]')
+
 def read_control(act, tree, control_override):
        stanzas = [ ]
 
@@ -866,11 +1134,13 @@ def read_control(act, tree, control_override):
                control_af = control_override
                testbed.blame('arg:'+control_override.spec)
        else:
+               if act.missing_tests_control:
+                       return ()
                control_af = RelativeInputFile(act.what+'-testcontrol',
                        tree, 'debian/tests/control')
        try:
-               control = file(control_af.read(), 'r')
-       except OSError, oe:
+               control = open(control_af.read(), 'r')
+       except (IOError,OSError), oe:
                if oe[0] != errno.ENOENT: raise
                return []
 
@@ -924,9 +1194,12 @@ def read_control(act, tree, control_override):
                        tnames = map((lambda lt: lt[1]), tnames)
                        tnames = string.join(tnames).split()
                        base = {
+                               'restriction_names': [],
                                'restrictions': [],
+                               'feature_names': [],
+                               'features': [],
                                'testsdir': 'debian/tests',
-                               'depends' : '*'
+                               'depends' : '@'
                        }
                        for fname in stz.keys():
                                if fname.startswith(' '): continue
@@ -947,25 +1220,31 @@ def read_control(act, tree, control_override):
        return stanzas
 
 def print_exception(ei, msgprefix=''):
-       if msgprefix: print >>sys.stderr, msgprefix
+       if msgprefix: pstderr(msgprefix)
        (et, q, tb) = ei
        if et is Quit:
-               print >>sys.stderr, 'adt-run:', q.m
+               pstderr('adt-run: ' + q.m)
+               psummary('quitting: '+q.m)
                return q.ec
        else:
-               print >>sys.stderr, "adt-run: unexpected, exceptional, error:"
-               traceback.print_exc()
+               pstderr("adt-run: unexpected, exceptional, error:")
+               psummary('quitting: unexpected error, consult transcript')
+               traceback.print_exc(None, sys.stderr)
+               if trace_stream is not None:
+                       traceback.print_exc(None, trace_stream)
                return 20
 
 def cleanup():
+       global trace_stream
        try:
-               rm_ec = 0
-               if opts.tmpdir is None and tmpdir is not None:
-                       rmtree('tmpdir', tmpdir)
                if testbed is not None:
                        testbed.reset_apt()
                        testbed.stop()
-               if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
+               if opts.tmpdir is None and tmpdir is not None:
+                       rmtree('tmpdir', tmpdir)
+               if trace_stream is not None:
+                       trace_stream.close()
+                       trace_stream = None
        except:
                print_exception(sys.exc_info(),
                        '\nadt-run: error cleaning up:\n')
@@ -975,9 +1254,7 @@ def cleanup():
 
 def determine_package(act):
        cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
-       running = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-       output = running.communicate()[0]
-       rc = running.wait()
+       (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
        if rc: badpkg('failed to parse binary package, code %d' % rc)
        re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
        act.pkg = None
@@ -1002,7 +1279,7 @@ class Binaries:
                for x in ['pubring','secring']:
                        os.stat(opts.gnupghome + '/' + x + '.gpg')
                ok = True
-       except OSError, oe:
+       except (IOError,OSError), oe:
                if oe.errno != errno.ENOENT: raise
 
        if ok: b._debug('no key generation needed')
@@ -1018,8 +1295,8 @@ class Binaries:
        mkdir_okexist(opts.gnupghome, 0700)
 
        script = '''
+  exec >&2
   cd "$1"
-  exec >key-gen-log 2>&1
   cat <<"END" >key-gen-params
 Key-Type: DSA
 Key-Length: 1024
@@ -1032,19 +1309,13 @@ END
   gpg --homedir="$1" --batch --gen-key key-gen-params
                '''
        cmdl = ['sh','-ec',script,'x',opts.gnupghome]
-       debug_subprocess('genkey', cmdl, script=script)
-       rc = subprocess.call(cmdl)
-       if rc:
-               try:
-                       f = open(opts.gnupghome+'/key-gen-log')
-                       tp = file.read()
-               except OSError, e: tp = e
-               print >>sys.stderr, tp
-               bomb('key generation failed, code %d' % rc)
+       rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
+       if rc: bomb('key generation failed, code %d' % rc)
 
  def apt_configs(b):
        return {
                "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
+               "Debug::pkgProblemResolver": "true",
        }
 
  def apt_pkg_gdebi_script(b, arg, middle):
@@ -1066,7 +1337,9 @@ END
                'print res',
                'print d.missingDeps',
                'print d.requiredChanges',
-               'assert(res)',
+               'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
+                       ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
+                         'd._failureString)',
                'cache.commit()',
                ''
                ]
@@ -1075,7 +1348,7 @@ END
        ag = ['apt-get','-qy']
        for kv in b.apt_configs().iteritems():
                ag += ['-o', '%s=%s' % kv]
-       return ag
+       return ' '.join(ag)
 
  def reset(b):
        b._debug('reset')
@@ -1098,11 +1371,11 @@ END
        dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
 
        try: os.remove(dest.write())
-       except OSError, oe:
+       except (IOError,OSError), oe:
                if oe.errno != errno.ENOENT: raise e
 
        try: os.link(af.read(), dest.write())
-       except OSError, oe:
+       except (IOError,OSError), oe:
                if oe.errno != errno.EXDEV: raise e
                shutil.copy(af.read(), dest)
 
@@ -1115,6 +1388,7 @@ END
        b._debug('publish')
 
        script = '''
+  exec >&2
   cd "$1"
   apt-ftparchive packages . >Packages
   gzip <Packages >Packages.gz
@@ -1124,8 +1398,7 @@ END
   gpg --homedir="$2" --batch --export >archive-key.pgp
                '''
        cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
-       debug_subprocess('ftparchive', cmdl, script)
-       rc = subprocess.call(cmdl)
+       rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
        if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
 
        b.dir.invalidate(True)
@@ -1133,28 +1406,28 @@ END
 
        so = TemporaryFile('vlds')
        script = '''
-  apt-key add archive-key.pgp >&2
+  exec 3>&1 >&2
+  apt-key add archive-key.pgp
   echo "deb file://'''+apt_source+''' /" >sources.list
   cat /etc/apt/sources.list >>sources.list
   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
   fi
-  '''+ ' '.join(b.apt_get()) +''' update >&2
-  cat /var/lib/dpkg/status
+  '''+ b.apt_get() +''' update >&2
+  cat /var/lib/dpkg/status >&3
                '''
        testbed.mungeing_apt()
-       debug_subprocess('apt-key', script=script)
-       (rc,se) = testbed.execute('apt-key',
-                               ['sh','-ec',script],
-                               so=so.write(True), cwd=b.dir.write(True))
-       if rc: bomb('apt setup failed with exit code %d' % rc, se)
+       rc = testbed.execute('apt-key', ['sh','-ec',script],
+                               so=so.write(True), cwd=b.dir.write(True),
+                               script=script, kind='install')
+       if rc: bomb('apt setup failed with exit code %d' % rc)
 
        testbed.blamed += b.blamed
 
        b._debug('publish reinstall checking...')
        pkgs_reinstall = set()
        pkg = None
-       for l in file(so.read()):
+       for l in open(so.read()):
                if l.startswith('Package: '):
                        pkg = l[9:].rstrip()
                elif l.startswith('Status: install '):
@@ -1165,152 +1438,259 @@ END
        if pkgs_reinstall:
                for pkg in pkgs_reinstall: testbed.blame(pkg)
                what = 'apt-get-reinstall'
-               cmdl = (b.apt_get() + ['--reinstall','install'] +
-                       [pkg for pkg in pkgs_reinstall])
-               debug_subprocess(what, cmdl)
-               (rc,se) = testbed.execute(what, cmdl)
+               cmdl = (b.apt_get() + ' --reinstall install '+
+                       ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
+               cmdl = ['sh','-c',cmdl]
+               rc = testbed.execute(what, cmdl, script=None, kind='install')
                if rc: badpkg("installation of basic binarries failed,"
-                               " exit code %d" % rc, se)
+                               " exit code %d" % rc)
 
        b._debug('publish install...')
        for pkg in b.install:
                what = 'apt-get-install-%s' % pkg
                testbed.blame(pkg)
-               cmdl = b.apt_get() + ['install',pkg]
-               debug_subprocess(what, cmdl)
-               (rc,se) = testbed.execute(what, cmdl)
+               cmdl = b.apt_get() + ' install ' + pkg + ' >&2'
+               cmdl = ['sh','-c',cmdl]
+               rc = testbed.execute(what, cmdl, script=None, kind='install')
                if rc: badpkg("installation of %s failed, exit code %d"
-                               % (pkg, rc), se)
+                               % (pkg, rc))
 
        b._debug('publish done')
 
 #---------- processing of sources (building)
 
-def source_rules_command(act,script,what,which,work,results_lines=0):
-       if opts.debug:
-               trace = "%s-%s-trace" % (what,which)
-               script = [      "mkfifo -m600 "+trace,
-                               "tee <"+trace+" /dev/stderr >&4 &",
-                               "exec >"+trace+" 2>&1"  ] + script
-
+def source_rules_command(act,script,what,which,work,cwd,
+                               results_lines=0,xargs=[]):
        script = [      "exec 3>&1 >&2",
                        "set -x"        ] + script
        script = '\n'.join(script)
        so = TemporaryFile('%s-%s-results' % (what,which))
-       se = TemporaryFile('%s-%s-log' % (what,which))
-       debug_subprocess('source-rules-command/%s/%s' % (act.what, which),
-                       script=script)
        rc = testbed.execute('%s-%s' % (what,which),
-                       ['sh','-xec',script],
-                       so=so.write(True), se=se.write(True),
-                       cwd= work.write(True), dump_fd=4)
-       results = file(so.read()).read().rstrip('\n').split("\n")
-       se = file(se.read()).read()
-       if rc: badpkg("%s failed with exit code %d" % (which,rc), se)
+                       ['sh','-ec',script]+xargs, script=script,
+                       so=so.write(True), cwd=cwd, kind='build')
+       results = open(so.read()).read().rstrip('\n')
+       if len(results): results = results.split("\n")
+       else: results = []
+       if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
        if results_lines is not None and len(results) != results_lines:
                badpkg("got %d lines of results from %s where %d expected"
-                       % (len(results), which, results_lines), se)
+                       % (len(results), which, results_lines))
        if results_lines==1: return results[0]
        return results
 
-def build_source(act):
+def build_source(act, control_override):
        act.blame = 'arg:'+act.af.spec
        testbed.blame(act.blame)
+       testbed.prepare1([])
        testbed.needs_reset()
 
        what = act.what
-       dsc = act.af
-       basename = dsc.spec
-
-       dsc_file = open(dsc.read())
-       in_files = False
-       fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
-       for l in dsc_file:
-               l = l.rstrip('\n')
-               if l.startswith('Files:'): in_files = True; continue
-               elif l.startswith('#'): pass
-               elif not l.startswith(' '):
-                       in_files = False
-                       if l.startswith('Source:'):
-                               act.blame = 'dsc:'+l[7:].strip()
-                               testbed.blame(act.blame)
-               if not in_files: continue
-
-               m = fre.match(l)
-               if not m: badpkg(".dsc contains unparseable line"
-                               " in Files: `%s'" % l)
-               leaf = m.groups(0)[0]
-               subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
-                               sibling=True)
-               subfile.read(True)
-       dsc.read(True)
+       basename = act.af.spec
+       debiancontrol = None
+       act.binaries = []
 
-       script = binaries.apt_pkg_gdebi_script(
-               dsc.read(True), [[
-               'from GDebi.DscSrcPackage import DscSrcPackage',
-               'd = DscSrcPackage(cache, arg)',
-               'res = d.checkDeb()',
-                ],[
-               'from GDebi.DebPackage import DebPackage',
-               'd = DebPackage(cache)',
-               'res = d.satisfyDependsStr("build-essential")',
-               ]])
-       cmdl = ['python','-c',script]
-       whatp = what+'-builddeps'
-       debug_subprocess(whatp, cmdl, script=script)
-       (rc,se) = testbed.execute(what, cmdl)
-       if rc: badpkg('build-depends install failed, exit code %d' % rc, se)
+       def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
+
+       if act.kind == 'dsc':
+               dsc = act.af
+               dsc_file = open(dsc.read())
+               in_files = False
+               fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
+               for l in dsc_file:
+                       l = l.rstrip('\n')
+                       if l.startswith('Files:'): in_files = True; continue
+                       elif l.startswith('#'): pass
+                       elif not l.startswith(' '):
+                               in_files = False
+                               if l.startswith('Source:'):
+                                       act.blame = 'dsc:'+l[7:].strip()
+                                       testbed.blame(act.blame)
+                       if not in_files: continue
+
+                       m = fre.match(l)
+                       if not m: badpkg(".dsc contains unparseable line"
+                                       " in Files: `%s'" % l)
+                       leaf = m.groups(0)[0]
+                       subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
+                                       sibling=True)
+                       subfile.read(True)
+               dsc.read(True)
+
+       if act.kind == 'ubtree':
+               debiancontrol = RelativeInputFile(what+'-debiancontrol',
+                       act.af, 'debian/control')
+               dsc = TemporaryFile(what+'-fakedsc')
+               dsc_w = open(dsc.write(), 'w')
+               for l in open(debiancontrol.read()):
+                       l = l.rstrip('\n')
+                       if not len(l): break
+                       print >>dsc_w, l
+               print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
+               dsc_w.close()
+
+       if act.kind == 'dsc':
+               testbed.prepare2([])
+               script = binaries.apt_pkg_gdebi_script('', [[
+                               'from GDebi.DebPackage import DebPackage',
+                               'd = DebPackage(cache)',
+                               'res = d.satisfyDependsStr("dpkg-dev")',
+                       ]])
+               cmdl = ['python','-c',script]
+               whatp = what+'-dpkgsource'
+               rc = testbed.execute(what, cmdl, script=script, kind='install')
+               if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
 
        work = TemporaryDir(what+'-build')
+       act.work = work
 
-       script = [
-                       'cd '+work.write(True),
-               ]
-       if opts.user: script += [
-                       'chown '+opts.user+' .',
-                       'dsc=dsc.read(True) '+
-                               opts.user_wrap('dpkg-source -x $dsc')
-               ]
-       else: script += [
-                       'dpkg-source -x '+dsc.read(True),
+       tmpdir = work.write(True)+'/tmpdir'
+       tmpdir_script = [
+                       'TMPDIR="$1"',
+                       'rm -rf -- "$TMPDIR"',
+                       'export TMPDIR',
+                       opts.user_wrap('mkdir -- "$TMPDIR"'),
                ]
+
+       if act.kind == 'ubtree':
+               spec = '%s/real-tree' % work.write(True)
+               create_command = '''
+                       rm -rf "$spec"
+                       mkdir "$spec"
+                       cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
+                       '''
+               initcwd = act.af.read(True)
+
+       if act.kind == 'dsc':
+               spec = dsc.read(True)
+               create_command = '''
+                       dpkg-source -x $spec
+                       '''
+               initcwd = work.write(True)
+
+       script = [
+               'spec="$2"',
+               'origpwd=`pwd`',
+               'cd '+work.write(True)
+       ]
+
+       if opts.user:
+               script += ([ 'chown '+opts.user+' .' ] +
+                       tmpdir_script +
+                       [ 'spec="$spec" origpwd="$origpwd" '
+                               +opts.user_wrap(create_command) ])
+       else:
+               script += (tmpdir_script +
+                       [ create_command ])
+
        script += [
-                       'cd */.',
-                       'dpkg-checkbuilddeps',
+                       'cd [a-z0-9]*-*/.',
                        'pwd >&3',
-                       opts.user_wrap('debian/rules build'),
+                       'set +e; test -f debian/tests/control; echo $? >&3'
                ]
-       result_pwd = source_rules_command(act,script,what,'build',work,1)
+       (result_pwd, control_test_rc) = source_rules_command(
+                       act,script,what,'extract',work,
+                       cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
+
+       filter = act.ah['dsc_filter']
 
-       if os.path.dirname(result_pwd)+'/' != work.read(True):
-               badpkg("results dir `%s' is not in expected parent dir `%s'"
-                       % (result_pwd, work.read(True)))
+       if control_test_rc == '1': act.missing_tests_control = True
+
+       # For optional builds:
+       #
+       # We might need to build the package because:
+       #   - we want its binaries (filter isn't _ and at least one of the
+       #       deb_... isn't ignore)
+       #   - the test control file says so
+       #       (assuming we have any tests)
+
+       class NeedBuildException: pass
+       def build_needed(m):
+               debug_b('build needed for %s' % m)
+               raise NeedBuildException()
+
+       try:
+               if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
+                                     act.ah['deb_fortests'] != 'ignore'):
+                       build_needed('binaries')
+
+               result_pwd_af = InputDir(what+'-treeforcontrol',
+                       result_pwd, True)
+               stanzas = read_control(act, result_pwd_af, control_override)
+               for stanza in stanzas:
+                       for t in stanza[' tests']:
+                               if 'no-build-needed' not in t.feature_names:
+                                       build_needed('test %s' % t.tname)
+                               for d in t.depends:
+                                       if '@' in d:
+                                               build_needed('test %s '
+                                                'dependency %s' % (t.tname,d))
+
+               debug_b('build not needed')
+               built = False
+
+       except NeedBuildException:
+
+               if act.kind != 'dsc':
+                       testbed.prepare2([])
+
+               script = binaries.apt_pkg_gdebi_script(
+                       dsc.read(True), [[
+                       'from GDebi.DscSrcPackage import DscSrcPackage',
+                       'd = DscSrcPackage(cache, arg)',
+                       'res = d.checkDeb()',
+                        ],[
+                       'from GDebi.DebPackage import DebPackage',
+                       'd = DebPackage(cache)',
+                       'res = d.satisfyDependsStr("'+
+                                       ','.join(build_essential)+
+                               '")',
+                       ]])
+
+               cmdl = ['python','-c',script]
+               whatp = what+'-builddeps'
+               rc = testbed.execute(what, cmdl, script=script, kind='install')
+               if rc: badpkg('build-depends install failed,'
+                             ' exit code %d' % rc)
+
+               script = tmpdir_script + [
+                       'cd "$2"',
+                       'dpkg-checkbuilddeps',
+                       opts.user_wrap('debian/rules build'),
+                       ]
+               source_rules_command(act,script,what,'build',work,
+                               cwd=initcwd, xargs=['x',tmpdir,result_pwd])
+
+               if os.path.dirname(result_pwd)+'/' != work.read(True):
+                       badpkg("results dir `%s' is not in expected parent"
+                              " dir `%s'" % (result_pwd, work.read(True)))
+
+               built = True
 
-       act.work = work
        act.tests_tree = InputDir(what+'-tests-tree',
                                work.read(True)+os.path.basename(result_pwd),
                                True)
        if act.ah['dsc_tests']:
-               act.tests_tree.read()
                testbed.register_ephemeral(act.work)
                testbed.register_ephemeral(act.tests_tree)
 
+       if not built:
+               act.blamed = []
+               return
+
        act.blamed = copy.copy(testbed.blamed)
 
-       def debug_b(m): debug('* <dsc:%s> %s' % (act.what, m))
-       act.binaries = []
-       filter = act.ah['dsc_filter']
        debug_b('filter=%s' % filter)
        if filter != '_':
-               script = [
-                       'cd '+work.write(True)+'/*/.',
+               script = tmpdir_script + [
+                       'cd '+work.write(True)+'/[a-z0-9]*-*/.',
                        opts.user_wrap(opts.gainroot+' debian/rules binary'),
                        'cd ..',
                        'echo *.deb >&3',
                        ]
                result_debs = source_rules_command(act,script,what,
-                               'binary',work,1)
-               if result_debs == '*': debs = []
+                               'binary',work,work.write(True),
+                               results_lines=1, xargs=['x',tmpdir])
+               if result_debs == '*.deb': debs = []
                else: debs = result_debs.split(' ')
                debug_b('debs='+`debs`)
                re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
@@ -1347,24 +1727,34 @@ def process_actions():
        testbed.open()
        binaries = Binaries()
 
+       for act in opts.actions:
+               if act.af is not None and not act.af.spec_tbp:
+                       testbed.register_ephemeral(act.af)
+
        binaries.reset()
+       control_override = None
 
        debug_a1('builds ...')
        for act in opts.actions:
                debug_a2('%s %s' %
                        (act.kind, act.what))
 
-               testbed.prepare([])
+               if act.kind == 'control':
+                       control_override = act.af
                if act.kind == 'deb':
                        testbed.blame('arg:'+act.af.spec)
                        determine_package(act)
                        testbed.blame('deb:'+act.pkg)
                        binaries.register(act,act.pkg,act.af,
                                'forbuilds',testbed.blamed)
-               if act.kind == 'dsc':
-                       build_source(act)
+               if act.kind == 'dsc' or act.kind == 'ubtree':
+                       build_source(act, control_override)
                if act.kind == 'tree':
                        act.binaries = []
+               if act.kind.endswith('tree') or act.kind == 'dsc':
+                       control_override = None
+               if act.kind == 'instantiate':
+                       pass
 
        debug_a1('builds done.')
 
@@ -1381,10 +1771,11 @@ def process_actions():
                if act.kind == 'deb':
                        binaries.register(act,act.pkg,act.af,'fortests',
                                ['deb:'+act.pkg])
-               if act.kind == 'dsc':
+               if act.kind == 'dsc' or act.kind == 'ubtree':
                        for (pkg,bin) in act.binaries:
                                binaries.register(act,pkg,bin,'fortests',
                                        act.blamed)
+               if act.kind == 'dsc':
                        if act.ah['dsc_tests']:
                                debug_a3('read control ...')
                                stanzas = read_control(act, act.tests_tree,
@@ -1393,12 +1784,14 @@ def process_actions():
                                debug_a3('run_tests ...')
                                run_tests(stanzas, act.tests_tree)
                        control_override = None
-               if act.kind == 'tree':
+               if act.kind == 'tree' or act.kind == 'ubtree':
                        testbed.blame('arg:'+act.af.spec)
                        stanzas = read_control(act, act.af, control_override)
                        debug_a3('run_tests ...')
                        run_tests(stanzas, act.af)
                        control_override = None
+               if act.kind == 'instantiate':
+                       testbed.prepare([])
        debug_a1('tests done.')
 
 def main():
@@ -1409,6 +1802,7 @@ def main():
        except SystemExit, se:
                os._exit(20)
        try:
+               setup_trace()
                testbed = Testbed()
                testbed.start()
                finalise_options()