runner/adt-run

   1 #!/usr/bin/python2.6
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 try: our_base = os.environ['AUTOPKGTEST_BASE']+'/lib'
  44 except KeyError: our_base = '/usr/share/autopkgtest/python';
  45 sys.path.insert(1, our_base)
  46
  47 from Autopkgtest import *
  48
  49 #---------- global variables
  50
  51 tmp = None              # pathstring on host
  52 testbed = None          # Testbed
  53 errorcode = 0           # exit status that we are going to use
  54 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  55 binaries = None         # Binaries (.debs we have registered)
  56 build_essential = ["build-essential"]
  57 paths = []
  58 os.putenv("APT_LISTBUGS_FRONTEND", "none")    # do not consider using apt-listbugs
  59 os.putenv("APT_LISTCHANGES_FRONTEND", "none") # do not consider using apt-listchanges
  60
  61 #---------- output handling
  62 #
  63 #  There are at least the following kinds of output:
  64 #
  65 #   1. stderr output which consists of
  66 #   1a. our errors
  67 #   1b. stuff printed to stderr by the virtualisation server
  68 #   1c. stuff printed to stderr by our short-lived subprocesseses
  69 #           which we don't expect to fail
  70 #
  71 #   2. trace information, which consists of
  72 #   2a. our trace information from calls to debug()
  73 #   2b. progress information and stderr output from
  74 #        general scripts we run on the host
  75 #   2c. progress information and stderr output from things
  76 #        we run on the testbed including builds
  77 #   2d. stderr and stdout output from actual tests
  78 #
  79 #   xxxx
  80 #   3. actual test results (printed to our stdout)
  81 #
  82 # Cloning of 1a and 2a, where necessary, is done by us writing
  83 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  84 # is done by forking off a copy of ourselves to do plumbing,
  85 # which copy we wait for at the appropriate point.
  86
  87 class DummyOpts:
  88  def __init__(do): do.debuglevel = 0
  89
  90 opts = DummyOpts()
  91 trace_stream = None
  92 summary_stream = None
  93
  94 def pstderr(m):
  95         print >>sys.stderr, m
  96         if trace_stream is not None: print >>trace_stream, m
  97
  98 def debug(m, minlevel=0):
  99         if opts.debuglevel < minlevel: return
 100         if opts.quiet and trace_stream is None: return
 101         p = 'adt-run: trace'
 102         if minlevel: p += `minlevel`
 103         p += ': '
 104         for l in m.rstrip('\n').split('\n'):
 105                 s = p + l
 106                 if not opts.quiet: print >>sys.stderr, s
 107                 if trace_stream is not None: print >>trace_stream, s
 108
 109 def debug_file(hp, minlevel=0):
 110         if opts.debuglevel < minlevel: return
 111         def do_copy(stream, what):
 112                 rc = subprocess.call(['cat',hp], stdout=stream)
 113                 if rc: bomb('cat failed copying data from %s'
 114                             ' to %s, exit code %d' % (hp, what, rc))
 115         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 116         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 117
 118 class Errplumb:
 119  def __init__(ep, critical=False):
 120         to_stderr = critical or not opts.quiet
 121         count = to_stderr + (trace_stream is not None)
 122         if count == 0:
 123                 ep.stream = open('/dev/null','w')
 124                 ep._sp = None
 125         elif count == 1:
 126                 if to_stderr: ep.stream = os.dup(2)
 127                 else: ep.stream = trace_stream
 128                 ep._sp = None
 129         else:
 130                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 131                         stdin=subprocess.PIPE, stdout=trace_stream,
 132                         close_fds=True)
 133                 ep.stream = ep._sp.stdin
 134  def wait(ep):
 135         if ep._sp is None: return
 136         if type(ep.stream) == type(2):
 137                 os.close(ep.stream)
 138                 ep.stream = ()
 139         ep._sp.stdin.close()
 140         rc = ep._sp.wait()
 141         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 142         ep._sp = None
 143
 144 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 145         if dbg is not None:
 146                 if isinstance(dbg,tuple): (what,script) = dbg
 147                 else: (what,script) = (dbg,None)
 148                 debug_subprocess(what, cmdl, script=script)
 149         ep = Errplumb(critical)
 150         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 151         output = running.communicate()[0]
 152         rc = running.wait()
 153         ep.wait()
 154         return (rc, output)
 155
 156 def psummary(m):
 157         if summary_stream is not None: print >>summary_stream, m
 158
 159 def preport(m):
 160         print m
 161         sys.stdout.flush()
 162         if trace_stream is not None: print >>trace_stream, m
 163         psummary(m)
 164
 165 def report(tname, result):
 166         preport('%-20s %s' % (tname, result))
 167
 168 #---------- errors we define
 169
 170 class Quit:
 171         def __init__(q,ec,m): q.ec = ec; q.m = m
 172
 173 def bomb(m):
 174         raise Quit(20, "unexpected error: %s" % m)
 175
 176 def badpkg(m):
 177         preport('blame: ' + ' '.join(testbed.blamed))
 178         preport('badpkg: ' + m)
 179         raise Quit(12, "erroneous package: %s" % m)
 180
 181 class Unsupported:
 182  def __init__(u, lno, m):
 183         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 184         else: u.m = m
 185  def report(u, tname):
 186         global errorcode
 187         errorcode != 2
 188         report(tname, 'SKIP %s' % u.m)
 189
 190 #---------- convenience function
 191
 192 def mkdir_okexist(pathname, mode=02755):
 193         try:
 194                 os.mkdir(pathname, mode)
 195         except (IOError,OSError), oe:
 196                 if oe.errno != errno.EEXIST: raise
 197
 198 def rmtree(what, pathname):
 199         debug('/ %s rmtree %s' % (what, pathname), 2)
 200         try: shutil.rmtree(pathname)
 201         except (IOError,OSError), oe:
 202                 if oe.errno != errno.EEXIST: raise
 203
 204 def debug_subprocess(what, cmdl=None, script=None):
 205         o = '$ '+what+':'
 206         if cmdl is not None:
 207                 ol = []
 208                 for x in cmdl:
 209                         if x is script: x = '<SCRIPT>'
 210                         ol.append(x.    replace('\\','\\\\').
 211                                         replace(' ','\\ ')      )
 212                 o += ' '+ ' '.join(ol)
 213         debug(o)
 214         if script is not None and opts.debuglevel >= 1:
 215                 o = ''
 216                 for l in script.rstrip('\n').split('\n'):
 217                         o += '$     '+l+'\n'
 218                 debug(o,1)
 219
 220 def flatten(l):
 221         return reduce((lambda a,b: a + b), l, [])
 222
 223 #---------- fancy automatic file-copying class
 224
 225 class AutoFile:
 226         # p.what
 227         # p.path[tb]    None or path    not None => path known
 228         # p.file[tb]    None or path    not None => file exists
 229         # p.ephem[tb]   boolean         True => destroyed by tb reset
 230         # p.spec        string or None
 231         # p.spec_tbp    True or False, or not set if spec is None
 232         # p.dir         '' or '/'
 233
 234  def __init__(p, what):
 235         p.what = what
 236         p.path = [None,None]
 237         p.file = [None,None]
 238         p.ephem = [False,False]
 239         p.spec = None
 240         p.spec_tbp = None
 241         p.dir = ''
 242
 243  def __repr__(p):
 244         return "<AF@%s>" % p.__str__()
 245  def __str__(p):
 246         def ptbp(tbp):
 247                 if p.path[tbp] is None: return '-'+p.dir
 248                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 249                 else: return p.path[tbp]+p.dir+'!'
 250         out = p.what
 251         if p.spec is not None:
 252                 if not hasattr(p,'spec_tb'): out += '~'
 253                 elif p.spec_tbp: out += '#'
 254                 else: out += '='
 255                 out += p.spec
 256         out += ':'
 257         out += ptbp(False)
 258         out += '|'
 259         out += ptbp(True)
 260         return out
 261
 262  def _wrong(p, how):
 263         xtra = ''
 264         if p.spec is not None:
 265                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 266         raise Exception("internal error: %s (%s)" % (how, str(p)))
 267
 268  def _ensure_path(p, tbp):
 269         if p.path[tbp] is None:
 270                 if '/' in p.what:
 271                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 272                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 273                 if not tbp:
 274                         p.path[tbp] = tmp+'/'+p.what
 275                 else:
 276                         p.ephem[tbp] = True
 277                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 278
 279  def write(p, tbp=False):
 280         p._debug('write %s...' % 'HT'[tbp])
 281         p._ensure_path(tbp)
 282
 283         if p.dir and not p.file[tbp]:
 284                 if not tbp:
 285                         p._debug('mkdir H')
 286                         mkdir_okexist(p.path[tbp])
 287                 else:
 288                         cmdl = ['sh','-ec',
 289                                 'test -d "$1" || mkdir -p "$1"',
 290                                 'x', p.path[tbp]]
 291                         tf_what = urllib.quote(p.what).replace('/','%2F')
 292                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 293                         if rc: bomb('failed to create directory %s' %
 294                                 p.path[tbp])
 295
 296         p.file[tbp] = p.path[tbp]
 297         return p.path[tbp]
 298
 299  def read(p, tbp=False):
 300         p._debug('read %s...' % 'HT'[tbp])
 301         p._ensure_path(tbp)
 302
 303         if p.file[tbp] is None:
 304                 if p.file[not tbp] is None:
 305                         p._wrong("requesting read but nonexistent")
 306                 cud = ['copyup','copydown'][tbp]
 307                 src = p.file[not tbp] + p.dir
 308                 dst = p.path[tbp] + p.dir
 309                 testbed.command(cud, (src, dst))
 310                 p.file[tbp] = p.path[tbp]
 311
 312         return p.file[tbp] + p.dir
 313
 314  def invalidate(p, tbp=False):
 315         if p.path[tbp] is not None:
 316                 p._debug('invalidating %s' % 'HT'[tbp])
 317         p.file[tbp] = None
 318         if p.spec_tbp != tbp or p.spec is None:
 319                 p.path[tbp] = None
 320
 321  def invalidate_ephem(p, tbp=False):
 322         if p.ephem[tbp]:
 323                 p.invalidate(tbp)
 324
 325  def _debug(p, m):
 326         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 327
 328  def _constructed(p):
 329         p._debug('constructed: '+str(p))
 330         p._check()
 331         paths.append(p)
 332
 333  def _check(p):
 334         for tbp in [False,True]:
 335          for pf in [p.path, p.file]:
 336                 if pf[tbp] is None: continue
 337                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 338                 if p.dir and pf[tbp].endswith('/'):
 339                         pf[tbp] = pf[tbp].rstrip('/')
 340                         if not pf[tbp]: pf[tbp] = '/'
 341                 if not p.dir and pf[tbp].endswith('/'):
 342                         bomb("directory `%s' specified for "
 343                              "non-directory %s" % (pf[tbp], p.what))
 344
 345  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 346         AutoFile.__init__(p,what)
 347         sh_on = ''; sh_sibl = ''
 348         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 349         if sibling: sh_sibl=' (sibling)'
 350         parent._debug('using as base: %s: %s%s%s' %
 351                         (str(parent), leaf, sh_on, sh_sibl))
 352         if not sibling and not parent.dir:
 353                 parent._wrong('asked for non-sibling relative path of non-dir')
 354         if sibling: trim = os.path.dirname
 355         else: trim = lambda x: x
 356         for tbp in [False,True]:
 357                 p.ephem[tbp] = parent.ephem[tbp]
 358                 if parent.path[tbp] is None: continue
 359                 trimmed = trim(parent.path[tbp])
 360                 if trimmed: trimmed += '/'
 361                 p.path[tbp] = trimmed + leaf
 362                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 363                         p.file[tbp] = p.path[tbp]
 364
 365 class InputFile(AutoFile):
 366  def _init(p, what, spec, spec_tbp=False):
 367         AutoFile.__init__(p, what)
 368         p.spec = spec
 369         p.spec_tbp = spec_tbp
 370         p.path[spec_tbp] = spec
 371         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 372  def __init__(p, what, spec, spec_tbp=False):
 373         p._init(what,spec,spec_tbp)
 374         p._constructed()
 375
 376 class InputDir(InputFile):
 377  def __init__(p, what, spec, spec_tbp=False):
 378         InputFile._init(p,what,spec,spec_tbp)
 379         p.dir = '/'
 380         p._constructed()
 381
 382 class OutputFile(AutoFile):
 383  def _init(p, what, spec, spec_tbp=False):
 384         AutoFile.__init__(p, what)
 385         p.spec = spec
 386         p.spec_tbp = spec_tbp
 387         p.path[spec_tbp] = spec
 388  def __init__(p, what, spec, spec_tbp=False):
 389         p._init(what,spec,spec_tbp)
 390         p._constructed()
 391
 392 class OutputDir(OutputFile):
 393  def __init__(p, what, spec, spec_tbp=False):
 394         OutputFile._init(p,what,spec,spec_tbp)
 395         p.dir = '/'
 396         p._constructed()
 397
 398 class RelativeInputDir(AutoFile):
 399  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 400         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 401         p.dir = '/'
 402         p._constructed()
 403
 404 class RelativeInputFile(AutoFile):
 405  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 406         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 407         p._constructed()
 408
 409 class RelativeOutputFile(AutoFile):
 410  def __init__(p, what, parent, leaf, sibling=False):
 411         p._relative_init(what, parent, leaf, None, False, sibling)
 412         p._constructed()
 413
 414 class TemporaryFile(AutoFile):
 415  def __init__(p, what):
 416         AutoFile.__init__(p, what)
 417         p._constructed()
 418
 419 class TemporaryDir(AutoFile):
 420  def __init__(p, what):
 421         AutoFile.__init__(p, what)
 422         p.dir = '/'
 423         p._constructed()
 424
 425 #---------- parsing and representation of the arguments
 426
 427 class Action:
 428  def __init__(a, kind, af, arghandling, what):
 429         # extra attributes get added during processing
 430         a.kind = kind
 431         a.af = af
 432         a.ah = arghandling
 433         a.what = what
 434         a.missing_tests_control = False
 435  def __repr__(a):
 436         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 437
 438 def parse_args():
 439         global opts, timeouts
 440         global n_non_actions # argh, stupid python scoping rules
 441         usage = "%prog <options> --- <virt-server>..."
 442         parser = OptionParser(usage=usage)
 443         pa = parser.add_option
 444         pe = parser.add_option
 445
 446         arghandling = {
 447                 'dsc_tests': True,
 448                 'dsc_filter': '*',
 449                 'deb_forbuilds': 'auto',
 450                 'deb_fortests': 'auto',
 451                 'tb': False,
 452                 'override_control': None,
 453                 'set_lang': 'C'
 454         }
 455         initial_arghandling = arghandling.copy()
 456         n_non_actions = 0
 457
 458         #----------
 459         # actions (ie, test sets to run, sources to build, binaries to use):
 460
 461         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 462                 global n_non_actions
 463                 parser.largs.append((value,kindpath))
 464                 n_non_actions += not(is_act)
 465
 466         def pa_action(long, metavar, kindpath, help, is_act=True):
 467                 pa('','--'+long, action='callback', callback=cb_action,
 468                         nargs=1, type='string',
 469                         callback_args=(long,kindpath,is_act), help=help)
 470
 471         pa_action('built-tree',         'TREE', '@/',
 472                 help='run tests from build tree TREE')
 473
 474         pa_action('unbuilt-tree',       'TREE', '@//',
 475                 help='run tests from build tree TREE')
 476
 477         pa_action('source',             'DSC', '@.dsc',
 478                 help='build DSC and use its tests and/or'
 479                     ' generated binary packages')
 480
 481         pa_action('binary',             'DEB', '@.deb',
 482                help='use binary package DEB according'
 483                     ' to most recent --binaries-* settings')
 484
 485         def cb_actnoarg(op,optstr,value,parser, largsentry):
 486                 parser.largs.append(largsentry)
 487         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 488                 callback_args=((None, ('instantiate',)),),
 489                 help='instantiate testbed now (during testing phase)'
 490                      ' and install packages'
 491                      ' selected for automatic installation, even'
 492                      ' if this might apparently not be required otherwise')
 493
 494         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 495                help='run tests from control file CONTROL instead,'
 496                     ' (applies to next test suite only)')
 497
 498         #----------
 499         # argument handling settings (what ways to use action
 500         #  arguments, and pathname processing):
 501
 502         def cb_setah(option, opt_str, value, parser, toset,setval):
 503                 if type(setval) == list:
 504                         if not value in setval:
 505                                 parser.error('value for %s option (%s) is not '
 506                                  'one of the permitted values (%s)' %
 507                                  (value, opt_str, setval.join(' ')))
 508                 elif setval is not None:
 509                         value = setval
 510                 for v in toset:
 511                         arghandling[v] = value
 512                 parser.largs.append(arghandling.copy())
 513
 514         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 515                 type = metavar
 516                 if type is not None: type = 'string'
 517                 pa('',long, action='callback', callback=cb_setah,
 518                    callback_args=(affected,effect), **kwargs)
 519
 520         #---- paths: host or testbed:
 521         #
 522         pa_setah('--paths-testbed', ['tb'],True,
 523                 help='subsequent path specifications refer to the testbed')
 524         pa_setah('--paths-host', ['tb'],False,
 525                 help='subsequent path specifications refer to the host')
 526
 527         #---- source processing settings:
 528
 529         pa_setah('--sources-tests', ['dsc_tests'],True,
 530                 help='run tests from builds of subsequent sources')
 531         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 532                 help='do not run tests from builds of subsequent sources')
 533
 534         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 535                 type='string', metavar='PATTERN-LIST',
 536                 help='from subsequent sources, use binaries matching'
 537                      ' PATTERN-LIST (comma-separated glob patterns)'
 538                      ' according to most recent --binaries-* settings')
 539         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 540                 help='from subsequent sources, do not use any binaries')
 541         #---- binary package processing settings:
 542
 543         def pa_setahbins(long,toset,how):
 544          pa_setah(long, toset,['ignore','auto','install'],
 545                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 546                 help=how+' ignore binaries, install them as needed'
 547                         ' for dependencies, or unconditionally install'
 548                         ' them, respectively')
 549         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 550         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 551         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 552
 553         #----------
 554         # general options:
 555
 556         def cb_vserv(op,optstr,value,parser):
 557                 parser.values.vserver = list(parser.rargs)
 558                 del parser.rargs[:]
 559
 560         def cb_path(op,optstr,value,parser, constructor,long,dir):
 561                 name = long.replace('-','_')
 562                 af = constructor(long, value,arghandling['tb'])
 563                 setattr(parser.values, name, af)
 564
 565         def pa_path(long, constructor, help, dir=False):
 566                 pa('','--'+long, action='callback', callback=cb_path,
 567                         callback_args=(constructor,long,dir),
 568                         nargs=1, type='string',
 569                         help=help, metavar='PATH')
 570
 571         pa_path('output-dir', OutputDir, dir=True,
 572                 help='write stderr/out files in PATH')
 573
 574         pa('--leave-lang', dest='set_lang', action='store_false',
 575                 help="leave LANG on testbed set to testbed's default")
 576         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 577                 help='set LANG on testbed to LANGVAL', default='C')
 578
 579         pa('','--tmp-dir',              type='string', dest='tmp',
 580                 help='write temporary files to TMP, emptying it'
 581                      ' beforehand and leaving it behind at the end')
 582         pa('','--log-file',             type='string', dest='logfile',
 583                 help='write the log LOGFILE, emptying it beforehand,'
 584                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 585         pa('','--summary-file',         type='string', dest='summary',
 586                 help='write a summary report to SUMMARY,'
 587                      ' emptying it beforehand')
 588
 589         for k in timeouts.keys():
 590                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 591                         metavar='T', help='set %s timeout to T')
 592         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 593                         metavar='FACTOR', default=1.0,
 594                         help='multiply all default timeouts by FACTOR')
 595
 596         pa('','--user',                 type='string', dest='user',
 597                 help='run tests as USER (needs root on testbed)')
 598         pa('','--gain-root',            type='string', dest='gainroot',
 599                 help='prefix debian/rules binary with GAINROOT')
 600         pa('-q', '--quiet', action='store_true', dest='quiet', default=False);
 601         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 602         pa('','--gnupg-home',           type='string', dest='gnupghome',
 603                 default='~/.autopkgtest/gpg',
 604                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 605                         " signing private apt archive);"
 606                         " `fresh' means generate new key each time.")
 607
 608         #----------
 609         # actual meat:
 610
 611         class SpecialOption(optparse.Option): pass
 612         vs_op = SpecialOption('','--VSERVER-DUMMY')
 613         vs_op.action = 'callback'
 614         vs_op.type = None
 615         vs_op.default = None
 616         vs_op.nargs = 0
 617         vs_op.callback = cb_vserv
 618         vs_op.callback_args = ( )
 619         vs_op.callback_kwargs = { }
 620         vs_op.help = 'introduces virtualisation server and args'
 621         vs_op._short_opts = []
 622         vs_op._long_opts = ['---']
 623
 624         pa(vs_op)
 625
 626         (opts,args) = parser.parse_args()
 627         if not hasattr(opts,'vserver'):
 628                 parser.error('you must specifiy --- <virt-server>...')
 629         if n_non_actions >= len(parser.largs):
 630                 parser.error('nothing to do specified')
 631
 632         for k in timeouts.keys():
 633                 t = getattr(opts,'timeout_'+k)
 634                 if t is None: t = timeouts[k] * opts.timeout_factor
 635                 timeouts[k] = int(t)
 636
 637         arghandling = initial_arghandling
 638         opts.actions = []
 639         ix = 0
 640         for act in args:
 641                 if type(act) == dict:
 642                         arghandling = act
 643                         continue
 644                 elif type(act) == tuple:
 645                         pass
 646                 elif type(act) == str:
 647                         act = (act,act)
 648                 else:
 649                         raise Exception("unknown action in list `%s' having"
 650                               " type `%s'" % (act, type(act)))
 651                 (pathstr, kindpath) = act
 652
 653                 constructor = InputFile
 654                 if type(kindpath) is tuple:             kind = kindpath[0]
 655                 elif kindpath.endswith('.deb'):         kind = 'deb'
 656                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 657                 elif kindpath.endswith('//'):
 658                         kind = 'ubtree'
 659                         constructor = InputDir
 660                 elif kindpath.endswith('/'):
 661                         kind = 'tree'
 662                         constructor = InputDir
 663                 else: parser.error("do not know how to handle filename `%s';"
 664                         " specify --source --binary or --build-tree" %
 665                         kindpath)
 666
 667                 what = '%s%s' % (kind,ix); ix += 1
 668
 669                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 670                 else: fwhatx = '-'+kind
 671
 672                 if pathstr is None: af = None
 673                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 674                 opts.actions.append(Action(kind, af, arghandling, what))
 675
 676 def setup_trace():
 677         global trace_stream, tmp, summary_stream
 678
 679         if opts.tmp is not None:
 680                 rmtree('tmp(specified)',opts.tmp)
 681                 mkdir_okexist(opts.tmp, 0700)
 682                 tmp = opts.tmp
 683         else:
 684                 assert(tmp is None)
 685                 tmp = tempfile.mkdtemp()
 686
 687         if opts.logfile is None:
 688                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 689                         opts.logfile = opts.output_dir.spec + '/log'
 690                 elif opts.tmp is not None:
 691                         opts.logfile = opts.tmp + '/log'
 692         if opts.logfile is not None:
 693                 trace_stream = open(opts.logfile, 'w', 0)
 694         if opts.summary is not None:
 695                 summary_stream = open(opts.summary, 'w', 0)
 696
 697         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 698
 699 def finalise_options():
 700         global opts, tb, build_essential
 701
 702         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 703                 opts.user = ''
 704
 705         if opts.user is None:
 706                 su = 'suggested-normal-user='
 707                 ul = [
 708                         e[len(su):]
 709                         for e in testbed.caps
 710                         if e.startswith(su)
 711                         ]
 712                 if ul:
 713                         opts.user = ul[0]
 714                 else:
 715                         opts.user = ''
 716
 717         if opts.user:
 718                 if 'root-on-testbed' not in testbed.caps:
 719                         pstderr("warning: virtualisation"
 720                                 " system does not offer root on testbed,"
 721                                 " but --user option specified: failure likely")
 722                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 723         else:
 724                 opts.user_wrap = lambda x: x
 725
 726         if opts.gainroot is None:
 727                 opts.gainroot = ''
 728                 if (opts.user or
 729                     'root-on-testbed' not in testbed.caps):
 730                         opts.gainroot = 'fakeroot'
 731                         build_essential += ['fakeroot']
 732
 733         if opts.gnupghome.startswith('~/'):
 734                 try: home = os.environ['HOME']
 735                 except KeyError:
 736                         parser.error("HOME environment variable"
 737                                 " not set, needed for --gnupghome=`%s"
 738                                 % opts.gnupghome)
 739                 opts.gnupghome = home + opts.gnupghome[1:]
 740         elif opts.gnupghome == 'fresh':
 741                 opts.gnupghome = None
 742
 743 #---------- testbed management - the Testbed class
 744
 745 class Testbed:
 746  def __init__(tb):
 747         tb.sp = None
 748         tb.lastsend = None
 749         tb.scratch = None
 750         tb.modified = False
 751         tb.blamed = []
 752         tb._debug('init')
 753         tb._need_reset_apt = False
 754  def _debug(tb, m, minlevel=0):
 755         debug('** '+m, minlevel)
 756  def start(tb):
 757         tb._debug('start')
 758         p = subprocess.PIPE
 759         debug_subprocess('vserver', opts.vserver)
 760         tb._errplumb = Errplumb(True)
 761         tb.sp = subprocess.Popen(opts.vserver,
 762                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 763         tb.expect('ok')
 764         tb.caps = tb.commandr('capabilities')
 765  def stop(tb):
 766         tb._debug('stop')
 767         tb.close()
 768         if tb.sp is None: return
 769         ec = tb.sp.returncode
 770         if ec is None:
 771                 tb.sp.stdout.close()
 772                 tb.send('quit')
 773                 tb.sp.stdin.close()
 774                 ec = tb.sp.wait()
 775         if ec:
 776                 tb.bomb('testbed gave exit status %d after quit' % ec)
 777         tb._errplumb.wait()
 778  def open(tb):
 779         tb._debug('open, scratch=%s' % tb.scratch)
 780         if tb.scratch is not None: return
 781         pl = tb.commandr('open')
 782         tb._opened(pl)
 783  def _opened(tb, pl):
 784         tb.scratch = InputDir('tb-scratch', pl[0], True)
 785         tb.deps_processed = []
 786         for af in paths: af.invalidate_ephem(True)
 787         tb._auxverbscript_make()
 788  def _auxverbscript_make(tb):
 789         pec = tb.commandr('print-auxverb-command')
 790         if len(pec) < 1: tb.bomb('too few results from print-execute-command')
 791         cmdl = map(urllib.unquote, pec[0].split(','))
 792
 793         tb._debug('cmdl = %s' % (`cmdl`))
 794
 795         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 796         print >>open(tb.ec_auxverbscript.write(),'w'), (
 797 '''#!/bin/sh
 798 set -e
 799 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 800         # This is a pretty nasty hack.  Hopefully it can go away
 801         #  eventually.  See #635763.
 802         set -- dpkg --print-architecture
 803 fi
 804 exec '''+shellquote_cmdl(cmdl)+' "$@"'+"\n"
 805                 )
 806         os.chmod(tb.ec_auxverbscript.write(), 0755)
 807
 808  def mungeing_apt(tb):
 809         if not 'revert' in tb.caps:
 810                 tb._need_reset_apt = True
 811  def reset_apt(tb):
 812         if not tb._need_reset_apt: return
 813         what = 'aptget-update-reset'
 814         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 815         rc = tb.execute(what, cmdl, kind='install')
 816         if rc:
 817                 pstderr("\n" "warning: failed to restore"
 818                         " testbed apt cache, exit code %d" % rc)
 819         what = 'aptconf-reset'
 820         cmdl = ['rm','-f','/etc/apt/apt.conf.d/90autopkgtest',
 821                 '/etc/apt/sources.list.d/autopkgtest.list',
 822                 '/etc/apt/preferences.d/90autopkgtest']
 823         rc = tb.execute(what, cmdl, kind='install')
 824         if rc:
 825                 pstderr("\n" "warning: failed to reset changes"
 826                         " made to testbed apt configuration, exit code %d" % rc)
 827         tb._need_reset_apt = False
 828  def close(tb):
 829         tb._debug('close, scratch=%s' % tb.scratch)
 830         if tb.scratch is None: return
 831         tb.scratch = None
 832         if tb.sp is None: return
 833         tb.command('close')
 834  def prepare1(tb, deps_new):
 835         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 836                 (tb.modified, tb.deps_processed, deps_new), 1)
 837         if 'revert' in tb.caps and (tb.modified or
 838             [d for d in tb.deps_processed if d not in deps_new]):
 839                 for af in paths: af.read(False)
 840                 tb._debug('reset **')
 841                 pl = tb.commandr('revert')
 842                 tb._opened(pl)
 843         tb.modified = False
 844  def prepare2(tb, deps_new):
 845         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 846         binaries.publish()
 847         tb._install_deps(deps_new)
 848  def prepare(tb, deps_new):
 849         tb.prepare1(deps_new)
 850         tb.prepare2(deps_new)
 851  def _install_deps(tb, deps_new):
 852         tb._debug(' installing dependencies '+`deps_new`, 1)
 853         tb.deps_processed = deps_new
 854         if not deps_new: return
 855         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 856  def needs_reset(tb):
 857         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 858         tb.modified = True
 859  def blame(tb, m):
 860         tb._debug('blame += %s' % m, 1)
 861         tb.blamed.append(m)
 862  def bomb(tb, m):
 863         tb._debug('bomb %s' % m)
 864         if tb.sp is not None:
 865                 tb.sp.stdout.close()
 866                 tb.sp.stdin.close()
 867                 ec = tb.sp.wait()
 868                 if ec: pstderr('adt-run: testbed failing,'
 869                                 ' exit status %d' % ec)
 870         tb.sp = None
 871         raise Quit(16, 'testbed failed: %s' % m)
 872  def send(tb, string):
 873         tb.sp.stdin
 874         try:
 875                 debug('>> '+string, 2)
 876                 print >>tb.sp.stdin, string
 877                 tb.sp.stdin.flush()
 878                 tb.lastsend = string
 879         except:
 880                 (type, value, dummy) = sys.exc_info()
 881                 tb.bomb('cannot send to testbed: %s' % traceback.
 882                         format_exception_only(type, value))
 883  def expect(tb, keyword, nresults=None):
 884         l = tb.sp.stdout.readline()
 885         if not l: tb.bomb('unexpected eof from the testbed')
 886         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 887         l = l.rstrip('\n')
 888         debug('<< '+l, 2)
 889         ll = l.split()
 890         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 891         if ll[0] != keyword:
 892                 if tb.lastsend is None:
 893                         tb.bomb("got banner `%s', expected `%s...'" %
 894                                 (l, keyword))
 895                 else:
 896                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 897                                 (tb.lastsend, l, keyword))
 898         ll = ll[1:]
 899         if nresults is not None and len(ll) != nresults:
 900                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 901                         " expected %d result parameters" %
 902                         (tb.lastsend, l, len(ll), nresults))
 903         return ll
 904  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 905         # pass args=[None,...] or =(None,...) to avoid more url quoting
 906         if type(cmd) is str: cmd = [cmd]
 907         if len(args) and args[0] is None: args = args[1:]
 908         else: args = map(urllib.quote, args)
 909         al = cmd + args
 910         tb.send(string.join(al))
 911         ll = tb.expect('ok', nresults)
 912         if unquote: ll = map(urllib.unquote, ll)
 913         return ll
 914  def command(tb, cmd, args=()):
 915         tb.commandr(cmd, args, 0)
 916  def commandr1(tb, cmd, args=()):
 917         rl = tb.commandr(cmd, args, 1)
 918         return rl[0]
 919  def execute(tb, what, cmdl,
 920                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 921                 script=False, xenv=[], kind='short'):
 922         # Options for script:
 923         #   False - do not call debug_subprocess, no synch. reporting required
 924         #   None or string - call debug_subprocess with that value,
 925         #                       plumb stderr through synchronously if possible
 926         # Options for se:
 927         #   None - usual Errplumb (output is of kind 2c)
 928         #   string - path on testbed (output is of kind 2d)
 929
 930         timeout = timeouts[kind]
 931
 932         if script is not False: debug_subprocess(what, cmdl, script=script)
 933         if cwd is None: cwd = tb.scratch.write(True)
 934
 935         xdump = None
 936         if se is None:
 937                 ep = Errplumb()
 938                 se_catch = TemporaryFile(what+'-xerr')
 939                 se_use = se_catch.write(True)
 940                 if not opts.quiet: xdump = 'debug=2-2'
 941                 elif trace_stream is not None:
 942                         xdump = 'debug=2-%d' % trace_stream.fileno()
 943         else:
 944                 ep = None
 945                 se_catch = None
 946                 se_use = se
 947
 948         cmdl = [None,
 949                 ','.join(map(urllib.quote, cmdl)),
 950                 si, so, se_use, cwd]
 951
 952         if timeout is not None and timeout > 0:
 953                 cmdl.append('timeout=%d' % timeout)
 954
 955         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 956         for e in xenv: cmdl.append('env=%s' % e)
 957         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 958         if opts.set_lang is not False:
 959                 cmdl.append('env=LANG=%s' % opts.set_lang)
 960
 961         rc = tb.commandr1('execute', cmdl)
 962         try: rc = int(rc)
 963         except ValueError: bomb("execute for %s gave invalid response `%s'"
 964                                         % (what,rc))
 965
 966         if se_catch is not None:
 967                 debug_file(se_catch.read())
 968         if ep is not None:
 969                 ep.wait()
 970
 971         return rc
 972
 973  def satisfy_dependencies_string(tb, deps, what):
 974         # Must have called Binaries.configure_apt
 975         debug('dependencies: %s: satisfying %s' % (what,deps))
 976         dsc = TemporaryFile('deps.dsc')
 977         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 978         # pbuilder-satisfydepends has a bug where it ignores the
 979         #  Build-Depends if it's the last line in the dsc (#635696)
 980         tb.satisfy_dependencies_dsc(dsc, what)
 981
 982  def satisfy_dependencies_dsc(tb, dsc, what):
 983         # Must have called Binaries.configure_apt
 984         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 985                 '--binary-all', # --check-key
 986                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 987                 '-c',dsc.read()
 988         ]
 989         # The --internal-chrootexec option is really handy but
 990         #  perhaps we are not supposed to use it ?  See also #635697.
 991         debug('dependencies: %s: running %s' % (what,`cmdl`))
 992         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 993         if rc: badpkg('dependency install failed, exit code %d' % rc)
 994
 995 #---------- representation of test control files: Field*, Test, etc.
 996
 997 class FieldBase:
 998  def __init__(f, fname, stz, base, tnames, vl):
 999         assert(vl)
1000         f.stz = stz
1001         f.base = base
1002         f.tnames = tnames
1003         f.vl = vl
1004  def words(f):
1005         def distribute(vle):
1006                 (lno, v) = vle
1007                 r = v.split()
1008                 r = map((lambda w: (lno, w)), r)
1009                 return r
1010         return flatten(map(distribute, f.vl))
1011  def atmostone(f):
1012         if len(vl) == 1:
1013                 (f.lno, f.v) = vl[0]
1014         else:
1015                 raise Unsupported(f.vl[1][0],
1016                         'only one %s field allowed' % fn)
1017         return f.v
1018
1019 class FieldIgnore(FieldBase):
1020  def parse(f): pass
1021
1022 class Restriction:
1023  def __init__(r,rname,base): pass
1024
1025 class Restriction_rw_build_tree(Restriction): pass
1026 class Restriction_build_needed(Restriction): pass
1027 class Restriction_breaks_testbed(Restriction):
1028  def __init__(r, rname, base):
1029         if 'revert-full-system' not in testbed.caps:
1030                 raise Unsupported(-1,
1031         'Test breaks testbed but testbed does not advertise revert-full-system')
1032 class Restriction_needs_root(Restriction):
1033  def __init__(r, rname, base):
1034         if 'root-on-testbed' not in testbed.caps:
1035                 raise Unsupported(-1,
1036                         'Test needs root on testbed which is not available')
1037
1038 class Field_Restrictions(FieldBase):
1039  def parse(f):
1040         for wle in f.words():
1041                 (lno, rname) = wle
1042                 nrname = rname.replace('-','_')
1043                 try: rclass = globals()['Restriction_'+nrname]
1044                 except KeyError: raise Unsupported(lno,
1045                         'unknown restriction %s' % rname)
1046                 r = rclass(nrname, f.base)
1047                 f.base['restriction_names'].append(rname)
1048                 f.base['restrictions'].append(r)
1049
1050 class Field_Features(FieldIgnore):
1051  def parse(f):
1052         for wle in f.words():
1053                 (lno, fname) = wle
1054                 f.base['feature_names'].append(fname)
1055                 nfname = fname.replace('-','_')
1056                 try: fclass = globals()['Feature_'+nfname]
1057                 except KeyError: continue
1058                 ft = fclass(nfname, f.base)
1059                 f.base['features'].append(ft)
1060
1061 class Field_Tests(FieldIgnore): pass
1062
1063 class Field_Depends(FieldBase):
1064  def parse(f):
1065         debug("Field_DependS: %s %s %s %s" % (f.stz, f.base, f.tnames, f.vl), 2)
1066         dl = map(lambda x: x.strip(),
1067                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1068         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1069         for d in dl:
1070                 if re.search(d):
1071                         badpkg("Test Depends field contains dependency"
1072                                " `%s' with invalid characters" % d)
1073         f.base['depends'] = dl
1074
1075 class Field_Tests_directory(FieldBase):
1076  def parse(f):
1077         td = atmostone(f)
1078         if td.startswith('/'): raise Unspported(f.lno,
1079                 'Tests-Directory may not be absolute')
1080         f.base['testsdir'] = td
1081
1082 def run_tests(stanzas, tree):
1083         global errorcode, testbed
1084         if stanzas == ():
1085                 report('*', 'SKIP no tests in this package')
1086                 errorcode |= 8
1087         for stanza in stanzas:
1088                 tests = stanza[' tests']
1089                 if not tests:
1090                         report('*', 'SKIP package has metadata but no tests')
1091                         errorcode |= 8
1092                 for t in tests:
1093                         t.prepare()
1094                         t.run(tree)
1095                         if 'breaks-testbed' in t.restriction_names:
1096                                 testbed.needs_reset()
1097                 testbed.needs_reset()
1098
1099 class Test:
1100  def __init__(t, tname, base, act):
1101         if '/' in tname: raise Unsupported(base[' lno'],
1102                 'test name may not contain / character')
1103         for k in base: setattr(t,k,base[k])
1104         t.tname = tname
1105         t.act = act
1106         t.what = act.what+'t-'+tname
1107         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1108         else: t.path = tname
1109         t._debug('constructed; path=%s' % t.path)
1110         t._debug(' .depends=%s' % t.depends)
1111  def _debug(t, m):
1112         debug('& %s: %s' % (t.what, m))
1113  def report(t, m):
1114         report(t.what, m)
1115  def reportfail(t, m):
1116         global errorcode
1117         errorcode |= 4
1118         report(t.what, 'FAIL ' + m)
1119  def prepare(t):
1120         t._debug('preparing')
1121         dn = []
1122         for d in t.depends:
1123                 t._debug(' processing dependency '+d)
1124                 if not '@' in d:
1125                         t._debug('  literal dependency '+d)
1126                         dn.append(d)
1127                 else:
1128                         for (pkg,bin) in t.act.binaries:
1129                                 dp = d.replace('@',pkg)
1130                                 t._debug('  synthesised dependency '+dp)
1131                                 dn.append(dp)
1132         testbed.prepare(dn)
1133  def run(t, tree):
1134         t._debug('[----------------------------------------')
1135         def stdouterr(oe):
1136                 idstr = t.what + '-' + oe
1137                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1138                         use_dir = opts.output_dir
1139                 else:
1140                         use_dir = testbed.scratch
1141                 return RelativeOutputFile(idstr, use_dir, idstr)
1142
1143         if hasattr(t.act,'work'): t.act.work.write(True)
1144         tree.read(True)
1145
1146         af = RelativeInputFile(t.what, tree, t.path)
1147         so = stdouterr('stdout')
1148         se = stdouterr('stderr')
1149
1150         tf = af.read(True)
1151         xenv = []
1152
1153         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1154         if rc: bomb('failed to chmod +x %s' % tf)
1155
1156         testtmp = '%s%s-testtmp' % (testbed.scratch.read(True), t.what)
1157         script = 'rm -rf -- "$@"; mkdir -- "$@"'
1158
1159         if 'needs-root' not in t.restriction_names and opts.user is not None:
1160                 tfl = ['su',opts.user,'-c',tf]
1161                 if opts.user:
1162                         script += '; chown %s "$@"' % opts.user
1163                         if 'rw-build-tree' in t.restriction_names:
1164                                 script += '; chown -R %s "$2"' % opts.user
1165         else:
1166                 tfl = [tf]
1167
1168         test_tmpdir = testtmp+'/tmpdir'; xenv.append('TMPDIR=%s' % test_tmpdir)
1169         test_addtmp = testtmp+'/addtmp'; xenv.append('ADTTMP=%s' % test_adttmp)
1170         rc = testbed.execute('mktmpdir-'+t.what,
1171                 ['sh','-xec',script,'x',
1172                         test_tmpdir, test_adttmp,
1173                         tree.read(True)])
1174         if rc: bomb("could not create test tmp dirs in `%s', exit code %d"
1175                         % (testtmp, rc))
1176
1177         rc = testbed.execute('test-'+t.what, tfl,
1178                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1179                 xenv=xenv, kind='test')
1180
1181         so_read = so.read()
1182         se_read = se.read()
1183
1184         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1185         stab = os.stat(se_read)
1186         if stab.st_size != 0:
1187                 l = open(se_read).readline()
1188                 l = l.rstrip('\n \t\r')
1189                 if len(l) > 35: l = l[:35] + '...'
1190                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1191                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1192                 debug_file(se_read)
1193         elif rc != 0:
1194                 t.reportfail('non-zero exit status %d' % rc)
1195         else:
1196                 t.report('PASS')
1197
1198         stab = os.stat(so_read)
1199         if stab.st_size != 0:
1200                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1201                 debug_file(so_read)
1202
1203         t._debug('----------------------------------------]')
1204
1205 def read_control(act, tree, control_override):
1206         stanzas = [ ]
1207
1208         if control_override is not None:
1209                 control_af = control_override
1210                 testbed.blame('arg:'+control_override.spec)
1211         else:
1212                 if act.missing_tests_control:
1213                         return ()
1214                 control_af = RelativeInputFile(act.what+'-testcontrol',
1215                         tree, 'debian/tests/control')
1216         try:
1217                 control = open(control_af.read(), 'r')
1218         except (IOError,OSError), oe:
1219                 if oe[0] != errno.ENOENT: raise
1220                 return []
1221
1222         lno = 0
1223         def badctrl(m): testbed.bomb('tests/control line %d: %s' % (lno, m))
1224         stz = { }       # stz[field_name][index] = (lno, value)
1225                         # special field names:
1226                         # stz[' lno'] = number
1227                         # stz[' tests'] = list of Test objects
1228                         # empty dictionary means we're between stanzas
1229         def in_stanza(stz):
1230                 return stz.has_key(' lno')
1231         def end_stanza(stz):
1232                 if not in_stanza(stz): return
1233                 stz[' errs'] = 0
1234                 stanzas.append(stz.copy())
1235                 stz.clear()
1236                 hcurrent = None
1237
1238         initre = regexp.compile('([A-Z][-0-9a-zA-Z]*)\s*\:\s*(.*)$')
1239         while 1:
1240                 l = control.readline()
1241                 if not l: break
1242                 lno += 1
1243                 if not l.endswith('\n'): badctrl('unterminated line')
1244                 if regexp.compile('\s*\#').match(l): continue
1245                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1246                 initmat = initre.match(l)
1247                 if initmat:
1248                         (fname, l) = initmat.groups()
1249                         fname = string.capwords(fname)
1250                         if not in_stanza(stz):
1251                                 stz = { ' lno': lno, ' tests': [] }
1252                         if not stz.has_key(fname): stz[fname] = [ ]
1253                         hcurrent = stz[fname]
1254                 elif regexp.compile('\s').match(l):
1255                         if not hcurrent: badctrl('unexpected continuation')
1256                 else:
1257                         badctrl('syntax error')
1258                 hcurrent.append((lno, l))
1259         end_stanza(stz)
1260
1261         def testbadctrl(stz, lno, m):
1262                 report_badctrl(lno, m)
1263                 stz[' errs'] += 1
1264
1265         for stz in stanzas:
1266                 try:
1267                         try: tnames = stz['Tests']
1268                         except KeyError:
1269                                 tnames = ['*']
1270                                 raise Unsupported(stz[' lno'],
1271                                         'no Tests field')
1272                         tnames = map((lambda lt: lt[1]), tnames)
1273                         tnames = string.join(tnames).split()
1274                         base = {
1275                                 'restriction_names': [],
1276                                 'restrictions': [],
1277                                 'feature_names': [],
1278                                 'features': [],
1279                                 'testsdir': 'debian/tests',
1280                                 'depends' : '@'
1281                         }
1282                         for fname in stz.keys():
1283                                 if fname.startswith(' '): continue
1284                                 vl = stz[fname]
1285                                 try: fclass = globals()['Field_'+
1286                                         fname.replace('-','_')]
1287                                 except KeyError: raise Unsupported(vl[0][0],
1288                                         'unknown metadata field %s' % fname)
1289                                 f = fclass(stz, fname, base, tnames, vl)
1290                                 f.parse()
1291                         for tname in tnames:
1292                                 t = Test(tname, base, act)
1293                                 stz[' tests'].append(t)
1294                 except Unsupported, u:
1295                         for tname in tnames: u.report(tname)
1296                         continue
1297
1298         return stanzas
1299
1300 def print_exception(ei, msgprefix=''):
1301         if msgprefix: pstderr(msgprefix)
1302         (et, q, tb) = ei
1303         if et is Quit:
1304                 pstderr('adt-run: ' + q.m)
1305                 psummary('quitting: '+q.m)
1306                 return q.ec
1307         else:
1308                 pstderr("adt-run: unexpected, exceptional, error:")
1309                 psummary('quitting: unexpected error, consult transcript')
1310                 traceback.print_exc(None, sys.stderr)
1311                 if trace_stream is not None:
1312                         traceback.print_exc(None, trace_stream)
1313                 return 20
1314
1315 def cleanup():
1316         global trace_stream
1317         try:
1318                 if testbed is not None:
1319                         testbed.reset_apt()
1320                         testbed.stop()
1321                 if opts.tmp is None and tmp is not None:
1322                         rmtree('tmp', tmp)
1323                 if trace_stream is not None:
1324                         trace_stream.close()
1325                         trace_stream = None
1326         except:
1327                 print_exception(sys.exc_info(),
1328                         '\nadt-run: error cleaning up:\n')
1329                 os._exit(20)
1330
1331 #---------- registration, installation etc. of .deb's: Binaries
1332
1333 def determine_package(act):
1334         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1335         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1336         if rc: badpkg('failed to parse binary package, code %d' % rc)
1337         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1338         act.pkg = None
1339         for l in output.split('\n'):
1340                 m = re.match(l)
1341                 if not m: continue
1342                 if act.pkg: badpkg('two Package: lines in control file')
1343                 act.pkg = m.groups()[0]
1344         if not act.pkg: badpkg('no good Package: line in control file')
1345
1346 class Binaries:
1347  def __init__(b, tb):
1348         b.dir = TemporaryDir('binaries')
1349         b.dir.write()
1350         ok = False
1351
1352         if opts.gnupghome is None:
1353                 opts.gnupghome = tmp+'/gnupg'
1354
1355         b._debug('initialising')
1356         try:
1357                 for x in ['pubring','secring']:
1358                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1359                 ok = True
1360         except (IOError,OSError), oe:
1361                 if oe.errno != errno.ENOENT: raise
1362
1363         if ok: b._debug('no key generation needed')
1364         else: b.genkey()
1365
1366  def _debug(b, s):
1367         debug('* '+s)
1368
1369  def genkey(b):
1370         b._debug('preparing for key generation')
1371
1372         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1373         mkdir_okexist(opts.gnupghome, 0700)
1374
1375         script = '''
1376   exec >&2
1377   cd "$1"
1378   cat <<"END" >key-gen-params
1379 Key-Type: DSA
1380 Key-Length: 1024
1381 Key-Usage: sign
1382 Name-Real: autopkgtest per-run key
1383 Name-Comment: do not trust this key
1384 Name-Email: autopkgtest@example.com
1385 END
1386   set -x
1387   gpg --homedir="$1" --batch --gen-key key-gen-params
1388                 '''
1389         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1390         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1391         if rc: bomb('key generation failed, code %d' % rc)
1392
1393  def apt_configs(b):
1394         return {
1395                 "Debug::pkgProblemResolver": "true",
1396                 "APT::Get::force-yes" : "true",
1397                 "APT::Get::Assume-Yes" : "true",
1398                 "quiet" : "true",
1399         }
1400
1401  def _configure_apt(b, tb):
1402         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1403                         True)
1404         f = open(config.write(),'w')
1405         for (k,v) in b.apt_configs().iteritems():
1406                 print >>f, '%s { "%s"; };' % (k, v)
1407         f.close()
1408         config.read(True)
1409
1410         prefs = OutputFile('apt-prefs','/etc/apt/preferences.d/90autopkgtest',
1411                         True)
1412         print >>open(prefs.write(),'w'), '''
1413 Package: *
1414 Pin: origin ""
1415 Pin-Priority: 1002
1416 '''
1417         prefs.read(True)
1418
1419  def _apt_get(b):
1420         ag = ['apt-get','-q']
1421         for kv in b.apt_configs().iteritems():
1422                 ag += ['-o', '%s=%s' % kv]
1423         return ' '.join(ag)
1424
1425  def reset(b):
1426         b._debug('reset')
1427         rmtree('binaries', b.dir.read())
1428         b.dir.invalidate()
1429         b.dir.write()
1430         b.install = []
1431         b.blamed = []
1432         b.registered = set()
1433
1434  def register(b, act, pkg, af, forwhat, blamed):
1435         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1436                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1437
1438         if act.ah['deb_'+forwhat] == 'ignore': return
1439
1440         b.blamed += testbed.blamed
1441
1442         leafname = pkg+'.deb'
1443         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1444
1445         try: os.remove(dest.write())
1446         except (IOError,OSError), oe:
1447                 if oe.errno != errno.ENOENT: raise e
1448
1449         try: os.link(af.read(), dest.write())
1450         except (IOError,OSError), oe:
1451                 if oe.errno != errno.EXDEV: raise e
1452                 shutil.copy(af.read(), dest)
1453
1454         if act.ah['deb_'+forwhat] == 'install':
1455                 b.install.append(pkg)
1456
1457         b.registered.add(pkg)
1458
1459  def publish(b):
1460         b._debug('publish')
1461
1462         b._configure_apt(testbed)
1463
1464         script = '''
1465   exec >&2
1466   cd "$1"
1467   apt-ftparchive packages . >Packages
1468   gzip <Packages >Packages.gz
1469   apt-ftparchive release . >Release
1470   rm -f Release.gpg
1471   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1472   gpg --homedir="$2" --batch --export >archive-key.pgp
1473                 '''
1474         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1475         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1476         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1477
1478         b.dir.invalidate(True)
1479         apt_source = b.dir.read(True)
1480
1481         so = TemporaryFile('vlds')
1482         script = '''
1483   exec 3>&1 >&2
1484   apt-key add archive-key.pgp
1485   echo "deb file://'''+apt_source+''' /" >/etc/apt/sources.list.d/autopkgtest.list
1486   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1487     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1488   fi
1489   '''+ b._apt_get() +''' update >&2
1490   cat /var/lib/dpkg/status >&3
1491                 '''
1492         testbed.mungeing_apt()
1493         rc = testbed.execute('apt-key', ['sh','-ec',script],
1494                                 so=so.write(True), cwd=b.dir.write(True),
1495                                 script=script, kind='install')
1496         if rc: bomb('apt setup failed with exit code %d' % rc)
1497
1498         testbed.blamed += b.blamed
1499
1500         b._debug('publish reinstall checking...')
1501         pkgs_reinstall = set()
1502         pkg = None
1503         for l in open(so.read()):
1504                 if l.startswith('Package: '):
1505                         pkg = l[9:].rstrip()
1506                 elif l.startswith('Status: install '):
1507                         if pkg in b.registered:
1508                                 pkgs_reinstall.add(pkg)
1509                                 b._debug(' publish reinstall needs '+pkg)
1510
1511         if pkgs_reinstall:
1512                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1513                 what = 'apt-get-reinstall'
1514                 cmdl = (b._apt_get() + ' --reinstall install '+
1515                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1516                 cmdl = ['sh','-c',cmdl]
1517                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1518                 if rc: badpkg("installation of basic binarries failed,"
1519                                 " exit code %d" % rc)
1520
1521         b._debug('publish install...')
1522         for pkg in b.install:
1523                 what = 'apt-get-install-%s' % pkg
1524                 testbed.blame(pkg)
1525                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1526                 cmdl = ['sh','-c',cmdl]
1527                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1528                 if rc: badpkg("installation of %s failed, exit code %d"
1529                                 % (pkg, rc))
1530
1531         b._debug('publish done')
1532
1533 #---------- processing of sources (building)
1534
1535 def source_rules_command(act,script,what,which,work,cwd,
1536                                 results_lines=0,xargs=[]):
1537         script = [      "exec 3>&1 >&2",
1538                         "set -x"        ] + script
1539         script = '\n'.join(script)
1540         so = TemporaryFile('%s-%s-results' % (what,which))
1541         rc = testbed.execute('%s-%s' % (what,which),
1542                         ['sh','-ec',script]+xargs, script=script,
1543                         so=so.write(True), cwd=cwd, kind='build')
1544         results = open(so.read()).read().rstrip('\n')
1545         if len(results): results = results.split("\n")
1546         else: results = []
1547         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1548         if results_lines is not None and len(results) != results_lines:
1549                 badpkg("got %d lines of results from %s where %d expected"
1550                         % (len(results), which, results_lines))
1551         if results_lines==1: return results[0]
1552         return results
1553
1554 def build_source(act, control_override):
1555         act.blame = 'arg:'+act.af.spec
1556         testbed.blame(act.blame)
1557         testbed.prepare1([])
1558         testbed.needs_reset()
1559
1560         what = act.what
1561         basename = act.af.spec
1562         debiancontrol = None
1563         act.binaries = []
1564
1565         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1566
1567         if act.kind == 'dsc':
1568                 dsc = act.af
1569                 dsc_file = open(dsc.read())
1570                 in_files = False
1571                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1572                 for l in dsc_file:
1573                         l = l.rstrip('\n')
1574                         if l.startswith('Files:'): in_files = True; continue
1575                         elif l.startswith('#'): pass
1576                         elif not l.startswith(' '):
1577                                 in_files = False
1578                                 if l.startswith('Source:'):
1579                                         act.blame = 'dsc:'+l[7:].strip()
1580                                         testbed.blame(act.blame)
1581                         if not in_files: continue
1582
1583                         m = fre.match(l)
1584                         if not m: badpkg(".dsc contains unparseable line"
1585                                         " in Files: `%s'" % l)
1586                         leaf = m.groups(0)[0]
1587                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1588                                         sibling=True)
1589                         subfile.read(True)
1590                 dsc.read(True)
1591
1592         if act.kind == 'ubtree':
1593                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1594                         act.af, 'debian/control')
1595                 dsc = TemporaryFile(what+'-fakedsc')
1596                 dsc_w = open(dsc.write(), 'w')
1597                 for l in open(debiancontrol.read()):
1598                         l = l.rstrip('\n')
1599                         if not len(l): break
1600                         print >>dsc_w, l
1601                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1602                 dsc_w.close()
1603
1604         if act.kind == 'dsc':
1605                 testbed.prepare2([])
1606                 testbed.satisfy_dependencies_string('dpkg-dev',
1607                                                 'install dpkg-dev')
1608
1609         work = TemporaryDir(what+'-build')
1610         act.work = work
1611
1612         tmpdir = work.write(True)+'/tmpdir'
1613         tmpdir_script = [
1614                         'TMPDIR="$1"',
1615                         'rm -rf -- "$TMPDIR"',
1616                         'export TMPDIR',
1617                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1618                 ]
1619
1620         if act.kind == 'ubtree':
1621                 spec = '%s/real-tree' % work.write(True)
1622                 create_command = '''
1623                         rm -rf "$spec"
1624                         mkdir "$spec"
1625                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1626                         '''
1627                 initcwd = act.af.read(True)
1628
1629         if act.kind == 'dsc':
1630                 spec = dsc.read(True)
1631                 create_command = '''
1632                         dpkg-source -x $spec
1633                         '''
1634                 initcwd = work.write(True)
1635
1636         script = [
1637                 'spec="$2"',
1638                 'origpwd=`pwd`',
1639                 'cd '+work.write(True)
1640         ]
1641
1642         if opts.user:
1643                 script += ([ 'chown '+opts.user+' .' ] +
1644                         tmpdir_script +
1645                         [ 'spec="$spec" origpwd="$origpwd" '
1646                                 +opts.user_wrap(create_command) ])
1647         else:
1648                 script += (tmpdir_script +
1649                         [ create_command ])
1650
1651         script += [
1652                         'cd [a-z0-9]*-*/.',
1653                         'pwd >&3',
1654                         'set +e; test -f debian/tests/control; echo $? >&3'
1655                 ]
1656         (result_pwd, control_test_rc) = source_rules_command(
1657                         act,script,what,'extract',work,
1658                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1659
1660         filter = act.ah['dsc_filter']
1661
1662         if control_test_rc == '1': act.missing_tests_control = True
1663
1664         # For optional builds:
1665         #
1666         # We might need to build the package because:
1667         #   - we want its binaries (filter isn't _ and at least one of the
1668         #       deb_... isn't ignore)
1669         #   - the test control file says so
1670         #       (assuming we have any tests)
1671
1672         class NeedBuildException: pass
1673         def build_needed(m):
1674                 debug_b('build needed for %s' % m)
1675                 raise NeedBuildException()
1676
1677         try:
1678                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1679                                       act.ah['deb_fortests'] != 'ignore'):
1680                         build_needed('binaries')
1681
1682                 result_pwd_af = InputDir(what+'-treeforcontrol',
1683                         result_pwd, True)
1684                 stanzas = read_control(act, result_pwd_af, control_override)
1685                 for stanza in stanzas:
1686                         for t in stanza[' tests']:
1687                                 if 'build-needed' in t.restriction_names:
1688                                         build_needed('test %s' % t.tname)
1689
1690                 debug_b('build not needed')
1691                 built = False
1692
1693         except NeedBuildException:
1694
1695                 if act.kind != 'dsc':
1696                         testbed.prepare2([])
1697
1698                 testbed.satisfy_dependencies_string('build-essential',
1699                                 'install build-essential')
1700                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1701
1702                 script = tmpdir_script + [
1703                         'cd "$2"',
1704                         'dpkg-checkbuilddeps',
1705                         opts.user_wrap('debian/rules build'),
1706                         ]
1707                 source_rules_command(act,script,what,'build',work,
1708                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1709
1710                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1711                         badpkg("results dir `%s' is not in expected parent"
1712                                " dir `%s'" % (result_pwd, work.read(True)))
1713
1714                 built = True
1715
1716         act.tests_tree = RelativeInputDir(what+'-tests-tree',
1717                                 work, os.path.basename(result_pwd),
1718                                 True)
1719
1720         if not built:
1721                 act.blamed = []
1722                 return
1723
1724         act.blamed = copy.copy(testbed.blamed)
1725
1726         debug_b('filter=%s' % filter)
1727         if filter != '_':
1728                 script = tmpdir_script + [
1729                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1730                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1731                         'cd ..',
1732                         'echo *.deb >&3',
1733                         ]
1734                 result_debs = source_rules_command(act,script,what,
1735                                 'binary',work,work.write(True),
1736                                 results_lines=1, xargs=['x',tmpdir])
1737                 if result_debs == '*.deb': debs = []
1738                 else: debs = result_debs.split(' ')
1739                 debug_b('debs='+`debs`)
1740                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1741                 for deb in debs:
1742                         m = re.match(deb)
1743                         if not m: badpkg("badly-named binary `%s'" % deb)
1744                         pkg = m.groups()[0]
1745                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1746                         for pat in filter.split(','):
1747                                 debug_b('  pat=%s' % pat)
1748                                 if not fnmatch.fnmatchcase(pkg,pat):
1749                                         debug_b('   no match')
1750                                         continue
1751                                 deb_what = pkg+'_'+what+'.deb'
1752                                 bin = RelativeInputFile(deb_what,work,deb,True)
1753                                 debug_b('  deb_what=%s, bin=%s' %
1754                                         (deb_what, str(bin)))
1755                                 binaries.register(act,pkg,bin,
1756                                         'forbuilds',testbed.blamed)
1757                                 act.binaries.append((pkg,bin))
1758                                 break
1759                 debug_b('all done.')
1760
1761 #---------- main processing loop and main program
1762
1763 def process_actions():
1764         global binaries
1765
1766         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1767         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1768         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1769
1770         debug_a1('starting')
1771         testbed.open()
1772         binaries = Binaries(testbed)
1773
1774         binaries.reset()
1775         control_override = None
1776
1777         debug_a1('builds ...')
1778         for act in opts.actions:
1779                 debug_a2('%s %s' %
1780                         (act.kind, act.what))
1781
1782                 if act.kind == 'control':
1783                         control_override = act.af
1784                 if act.kind == 'deb':
1785                         testbed.blame('arg:'+act.af.spec)
1786                         determine_package(act)
1787                         testbed.blame('deb:'+act.pkg)
1788                         binaries.register(act,act.pkg,act.af,
1789                                 'forbuilds',testbed.blamed)
1790                 if act.kind == 'dsc' or act.kind == 'ubtree':
1791                         build_source(act, control_override)
1792                 if act.kind == 'tree':
1793                         act.binaries = []
1794                 if act.kind.endswith('tree') or act.kind == 'dsc':
1795                         control_override = None
1796                 if act.kind == 'instantiate':
1797                         pass
1798
1799         debug_a1('builds done.')
1800
1801         binaries.reset()
1802         control_override = None
1803
1804         debug_a1('tests ...')
1805         for act in opts.actions:
1806                 debug_a2('test %s %s' % (act.kind, act.what))
1807
1808                 testbed.needs_reset()
1809                 if act.kind == 'control':
1810                         control_override = act.af
1811                 if act.kind == 'deb':
1812                         binaries.register(act,act.pkg,act.af,'fortests',
1813                                 ['deb:'+act.pkg])
1814                 if act.kind == 'dsc' or act.kind == 'ubtree':
1815                         for (pkg,bin) in act.binaries:
1816                                 binaries.register(act,pkg,bin,'fortests',
1817                                         act.blamed)
1818                 if act.kind == 'dsc':
1819                         if act.ah['dsc_tests']:
1820                                 debug_a3('read control ...')
1821                                 stanzas = read_control(act, act.tests_tree,
1822                                                 control_override)
1823                                 testbed.blamed += act.blamed
1824                                 debug_a3('run_tests ...')
1825                                 run_tests(stanzas, act.tests_tree)
1826                         control_override = None
1827                 if act.kind == 'tree' or act.kind == 'ubtree':
1828                         testbed.blame('arg:'+act.af.spec)
1829                         stanzas = read_control(act, act.af, control_override)
1830                         debug_a3('run_tests ...')
1831                         run_tests(stanzas, act.af)
1832                         control_override = None
1833                 if act.kind == 'instantiate':
1834                         testbed.prepare([])
1835         debug_a1('tests done.')
1836
1837 def main():
1838         global testbed
1839         global tmp
1840         try:
1841                 parse_args()
1842         except SystemExit, se:
1843                 os._exit(20)
1844         try:
1845                 setup_trace()
1846                 testbed = Testbed()
1847                 testbed.start()
1848                 finalise_options()
1849                 process_actions()
1850         except:
1851                 ec = print_exception(sys.exc_info(), '')
1852                 cleanup()
1853                 os._exit(ec)
1854         cleanup()
1855         os._exit(errorcode)
1856
1857 main()