runner/adt-run

   1 #!/usr/bin/python2.6
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 try: our_base = os.environ['AUTOPKGTEST_BASE']+'/lib'
  44 except KeyError: our_base = '/usr/share/autopkgtest/python';
  45 sys.path.insert(1, our_base)
  46
  47 from Autopkgtest import *
  48
  49 #---------- global variables
  50
  51 tmpdir = None           # pathstring on host
  52 testbed = None          # Testbed
  53 errorcode = 0           # exit status that we are going to use
  54 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  55 binaries = None         # Binaries (.debs we have registered)
  56 build_essential = ["build-essential"]
  57 paths = []
  58 os.putenv("APT_LISTBUGS_FRONTEND", "none")    # do not consider using apt-listbugs
  59 os.putenv("APT_LISTCHANGES_FRONTEND", "none") # do not consider using apt-listchanges
  60
  61 #---------- output handling
  62 #
  63 #  There are at least the following kinds of output:
  64 #
  65 #   1. stderr output which consists of
  66 #   1a. our errors
  67 #   1b. stuff printed to stderr by the virtualisation server
  68 #   1c. stuff printed to stderr by our short-lived subprocesseses
  69 #           which we don't expect to fail
  70 #
  71 #   2. trace information, which consists of
  72 #   2a. our trace information from calls to debug()
  73 #   2b. progress information and stderr output from
  74 #        general scripts we run on the host
  75 #   2c. progress information and stderr output from things
  76 #        we run on the testbed including builds
  77 #   2d. stderr and stdout output from actual tests
  78 #
  79 #   xxxx
  80 #   3. actual test results (printed to our stdout)
  81 #
  82 # Cloning of 1a and 2a, where necessary, is done by us writing
  83 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  84 # is done by forking off a copy of ourselves to do plumbing,
  85 # which copy we wait for at the appropriate point.
  86
  87 class DummyOpts:
  88  def __init__(do): do.debuglevel = 0
  89
  90 opts = DummyOpts()
  91 trace_stream = None
  92 summary_stream = None
  93
  94 def pstderr(m):
  95         print >>sys.stderr, m
  96         if trace_stream is not None: print >>trace_stream, m
  97
  98 def debug(m, minlevel=0):
  99         if opts.debuglevel < minlevel: return
 100         if opts.quiet and trace_stream is None: return
 101         p = 'adt-run: trace'
 102         if minlevel: p += `minlevel`
 103         p += ': '
 104         for l in m.rstrip('\n').split('\n'):
 105                 s = p + l
 106                 if not opts.quiet: print >>sys.stderr, s
 107                 if trace_stream is not None: print >>trace_stream, s
 108
 109 def debug_file(hp, minlevel=0):
 110         if opts.debuglevel < minlevel: return
 111         def do_copy(stream, what):
 112                 rc = subprocess.call(['cat',hp], stdout=stream)
 113                 if rc: bomb('cat failed copying data from %s'
 114                             ' to %s, exit code %d' % (hp, what, rc))
 115         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 116         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 117
 118 class Errplumb:
 119  def __init__(ep, critical=False):
 120         to_stderr = critical or not opts.quiet
 121         count = to_stderr + (trace_stream is not None)
 122         if count == 0:
 123                 ep.stream = open('/dev/null','w')
 124                 ep._sp = None
 125         elif count == 1:
 126                 if to_stderr: ep.stream = os.dup(2)
 127                 else: ep.stream = trace_stream
 128                 ep._sp = None
 129         else:
 130                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 131                         stdin=subprocess.PIPE, stdout=trace_stream,
 132                         close_fds=True)
 133                 ep.stream = ep._sp.stdin
 134  def wait(ep):
 135         if ep._sp is None: return
 136         if type(ep.stream) == type(2):
 137                 os.close(ep.stream)
 138                 ep.stream = ()
 139         ep._sp.stdin.close()
 140         rc = ep._sp.wait()
 141         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 142         ep._sp = None
 143
 144 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 145         if dbg is not None:
 146                 if isinstance(dbg,tuple): (what,script) = dbg
 147                 else: (what,script) = (dbg,None)
 148                 debug_subprocess(what, cmdl, script=script)
 149         ep = Errplumb(critical)
 150         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 151         output = running.communicate()[0]
 152         rc = running.wait()
 153         ep.wait()
 154         return (rc, output)
 155
 156 def psummary(m):
 157         if summary_stream is not None: print >>summary_stream, m
 158
 159 def preport(m):
 160         print m
 161         sys.stdout.flush()
 162         if trace_stream is not None: print >>trace_stream, m
 163         psummary(m)
 164
 165 def report(tname, result):
 166         preport('%-20s %s' % (tname, result))
 167
 168 #---------- errors we define
 169
 170 class Quit:
 171         def __init__(q,ec,m): q.ec = ec; q.m = m
 172
 173 def bomb(m):
 174         raise Quit(20, "unexpected error: %s" % m)
 175
 176 def badpkg(m):
 177         preport('blame: ' + ' '.join(testbed.blamed))
 178         preport('badpkg: ' + m)
 179         raise Quit(12, "erroneous package: %s" % m)
 180
 181 class Unsupported:
 182  def __init__(u, lno, m):
 183         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 184         else: u.m = m
 185  def report(u, tname):
 186         global errorcode
 187         errorcode != 2
 188         report(tname, 'SKIP %s' % u.m)
 189
 190 #---------- convenience function
 191
 192 def mkdir_okexist(pathname, mode=02755):
 193         try:
 194                 os.mkdir(pathname, mode)
 195         except (IOError,OSError), oe:
 196                 if oe.errno != errno.EEXIST: raise
 197
 198 def rmtree(what, pathname):
 199         debug('/ %s rmtree %s' % (what, pathname), 2)
 200         try: shutil.rmtree(pathname)
 201         except (IOError,OSError), oe:
 202                 if oe.errno != errno.EEXIST: raise
 203
 204 def debug_subprocess(what, cmdl=None, script=None):
 205         o = '$ '+what+':'
 206         if cmdl is not None:
 207                 ol = []
 208                 for x in cmdl:
 209                         if x is script: x = '<SCRIPT>'
 210                         ol.append(x.    replace('\\','\\\\').
 211                                         replace(' ','\\ ')      )
 212                 o += ' '+ ' '.join(ol)
 213         debug(o)
 214         if script is not None and opts.debuglevel >= 1:
 215                 o = ''
 216                 for l in script.rstrip('\n').split('\n'):
 217                         o += '$     '+l+'\n'
 218                 debug(o,1)
 219
 220 def flatten(l):
 221         return reduce((lambda a,b: a + b), l, [])
 222
 223 #---------- fancy automatic file-copying class
 224
 225 class AutoFile:
 226         # p.what
 227         # p.path[tb]    None or path    not None => path known
 228         # p.file[tb]    None or path    not None => file exists
 229         # p.ephem[tb]   boolean         True => destroyed by tb reset
 230         # p.spec        string or None
 231         # p.spec_tbp    True or False, or not set if spec is None
 232         # p.dir         '' or '/'
 233
 234  def __init__(p, what):
 235         p.what = what
 236         p.path = [None,None]
 237         p.file = [None,None]
 238         p.ephem = [False,False]
 239         p.spec = None
 240         p.spec_tbp = None
 241         p.dir = ''
 242
 243  def __repr__(p):
 244         return "<AF@%s>" % p.__str__()
 245  def __str__(p):
 246         def ptbp(tbp):
 247                 if p.path[tbp] is None: return '-'+p.dir
 248                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 249                 else: return p.path[tbp]+p.dir+'!'
 250         out = p.what
 251         if p.spec is not None:
 252                 if not hasattr(p,'spec_tb'): out += '~'
 253                 elif p.spec_tbp: out += '#'
 254                 else: out += '='
 255                 out += p.spec
 256         out += ':'
 257         out += ptbp(False)
 258         out += '|'
 259         out += ptbp(True)
 260         return out
 261
 262  def _wrong(p, how):
 263         xtra = ''
 264         if p.spec is not None:
 265                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 266         raise Exception("internal error: %s (%s)" % (how, str(p)))
 267
 268  def _ensure_path(p, tbp):
 269         if p.path[tbp] is None:
 270                 if '/' in p.what:
 271                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 272                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 273                 if not tbp:
 274                         p.path[tbp] = tmpdir+'/'+p.what
 275                 else:
 276                         p.ephem[tbp] = True
 277                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 278
 279  def write(p, tbp=False):
 280         p._debug('write %s...' % 'HT'[tbp])
 281         p._ensure_path(tbp)
 282
 283         if p.dir and not p.file[tbp]:
 284                 if not tbp:
 285                         p._debug('mkdir H')
 286                         mkdir_okexist(p.path[tbp])
 287                 else:
 288                         cmdl = ['sh','-ec',
 289                                 'test -d "$1" || mkdir -p "$1"',
 290                                 'x', p.path[tbp]]
 291                         tf_what = urllib.quote(p.what).replace('/','%2F')
 292                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 293                         if rc: bomb('failed to create directory %s' %
 294                                 p.path[tbp])
 295
 296         p.file[tbp] = p.path[tbp]
 297         return p.path[tbp]
 298
 299  def read(p, tbp=False):
 300         p._debug('read %s...' % 'HT'[tbp])
 301         p._ensure_path(tbp)
 302
 303         if p.file[tbp] is None:
 304                 if p.file[not tbp] is None:
 305                         p._wrong("requesting read but nonexistent")
 306                 cud = ['copyup','copydown'][tbp]
 307                 src = p.file[not tbp] + p.dir
 308                 dst = p.path[tbp] + p.dir
 309                 testbed.command(cud, (src, dst))
 310                 p.file[tbp] = p.path[tbp]
 311
 312         return p.file[tbp] + p.dir
 313
 314  def invalidate(p, tbp=False):
 315         if p.path[tbp] is not None:
 316                 p._debug('invalidating %s' % 'HT'[tbp])
 317         p.file[tbp] = None
 318         if p.spec_tbp != tbp or p.spec is None:
 319                 p.path[tbp] = None
 320
 321  def invalidate_ephem(p, tbp=False):
 322         if p.ephem[tbp]:
 323                 p.invalidate(tbp)
 324
 325  def _debug(p, m):
 326         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 327
 328  def _constructed(p):
 329         p._debug('constructed: '+str(p))
 330         p._check()
 331         paths.append(p)
 332
 333  def _check(p):
 334         for tbp in [False,True]:
 335          for pf in [p.path, p.file]:
 336                 if pf[tbp] is None: continue
 337                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 338                 if p.dir and pf[tbp].endswith('/'):
 339                         pf[tbp] = pf[tbp].rstrip('/')
 340                         if not pf[tbp]: pf[tbp] = '/'
 341                 if not p.dir and pf[tbp].endswith('/'):
 342                         bomb("directory `%s' specified for "
 343                              "non-directory %s" % (pf[tbp], p.what))
 344
 345  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 346         AutoFile.__init__(p,what)
 347         sh_on = ''; sh_sibl = ''
 348         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 349         if sibling: sh_sibl=' (sibling)'
 350         parent._debug('using as base: %s: %s%s%s' %
 351                         (str(parent), leaf, sh_on, sh_sibl))
 352         if not sibling and not parent.dir:
 353                 parent._wrong('asked for non-sibling relative path of non-dir')
 354         if sibling: trim = os.path.dirname
 355         else: trim = lambda x: x
 356         for tbp in [False,True]:
 357                 p.ephem[tbp] = parent.ephem[tbp]
 358                 if parent.path[tbp] is None: continue
 359                 trimmed = trim(parent.path[tbp])
 360                 if trimmed: trimmed += '/'
 361                 p.path[tbp] = trimmed + leaf
 362                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 363                         p.file[tbp] = p.path[tbp]
 364
 365 class InputFile(AutoFile):
 366  def _init(p, what, spec, spec_tbp=False):
 367         AutoFile.__init__(p, what)
 368         p.spec = spec
 369         p.spec_tbp = spec_tbp
 370         p.path[spec_tbp] = spec
 371         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 372  def __init__(p, what, spec, spec_tbp=False):
 373         p._init(what,spec,spec_tbp)
 374         p._constructed()
 375
 376 class InputDir(InputFile):
 377  def __init__(p, what, spec, spec_tbp=False):
 378         InputFile._init(p,what,spec,spec_tbp)
 379         p.dir = '/'
 380         p._constructed()
 381
 382 class OutputFile(AutoFile):
 383  def _init(p, what, spec, spec_tbp=False):
 384         AutoFile.__init__(p, what)
 385         p.spec = spec
 386         p.spec_tbp = spec_tbp
 387         p.path[spec_tbp] = spec
 388  def __init__(p, what, spec, spec_tbp=False):
 389         p._init(what,spec,spec_tbp)
 390         p._constructed()
 391
 392 class OutputDir(OutputFile):
 393  def __init__(p, what, spec, spec_tbp=False):
 394         OutputFile._init(p,what,spec,spec_tbp)
 395         p.dir = '/'
 396         p._constructed()
 397
 398 class RelativeInputDir(AutoFile):
 399  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 400         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 401         p.dir = '/'
 402         p._constructed()
 403
 404 class RelativeInputFile(AutoFile):
 405  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 406         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 407         p._constructed()
 408
 409 class RelativeOutputFile(AutoFile):
 410  def __init__(p, what, parent, leaf, sibling=False):
 411         p._relative_init(what, parent, leaf, None, False, sibling)
 412         p._constructed()
 413
 414 class TemporaryFile(AutoFile):
 415  def __init__(p, what):
 416         AutoFile.__init__(p, what)
 417         p._constructed()
 418
 419 class TemporaryDir(AutoFile):
 420  def __init__(p, what):
 421         AutoFile.__init__(p, what)
 422         p.dir = '/'
 423         p._constructed()
 424
 425 #---------- parsing and representation of the arguments
 426
 427 class Action:
 428  def __init__(a, kind, af, arghandling, what):
 429         # extra attributes get added during processing
 430         a.kind = kind
 431         a.af = af
 432         a.ah = arghandling
 433         a.what = what
 434         a.missing_tests_control = False
 435  def __repr__(a):
 436         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 437
 438 def parse_args():
 439         global opts, timeouts
 440         global n_non_actions # argh, stupid python scoping rules
 441         usage = "%prog <options> --- <virt-server>..."
 442         parser = OptionParser(usage=usage)
 443         pa = parser.add_option
 444         pe = parser.add_option
 445
 446         arghandling = {
 447                 'dsc_tests': True,
 448                 'dsc_filter': '*',
 449                 'deb_forbuilds': 'auto',
 450                 'deb_fortests': 'auto',
 451                 'tb': False,
 452                 'override_control': None,
 453                 'set_lang': 'C'
 454         }
 455         initial_arghandling = arghandling.copy()
 456         n_non_actions = 0
 457
 458         #----------
 459         # actions (ie, test sets to run, sources to build, binaries to use):
 460
 461         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 462                 global n_non_actions
 463                 parser.largs.append((value,kindpath))
 464                 n_non_actions += not(is_act)
 465
 466         def pa_action(long, metavar, kindpath, help, is_act=True):
 467                 pa('','--'+long, action='callback', callback=cb_action,
 468                         nargs=1, type='string',
 469                         callback_args=(long,kindpath,is_act), help=help)
 470
 471         pa_action('built-tree',         'TREE', '@/',
 472                 help='run tests from build tree TREE')
 473
 474         pa_action('unbuilt-tree',       'TREE', '@//',
 475                 help='run tests from build tree TREE')
 476
 477         pa_action('source',             'DSC', '@.dsc',
 478                 help='build DSC and use its tests and/or'
 479                     ' generated binary packages')
 480
 481         pa_action('binary',             'DEB', '@.deb',
 482                help='use binary package DEB according'
 483                     ' to most recent --binaries-* settings')
 484
 485         def cb_actnoarg(op,optstr,value,parser, largsentry):
 486                 parser.largs.append(largsentry)
 487         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 488                 callback_args=((None, ('instantiate',)),),
 489                 help='instantiate testbed now (during testing phase)'
 490                      ' and install packages'
 491                      ' selected for automatic installation, even'
 492                      ' if this might apparently not be required otherwise')
 493
 494         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 495                help='run tests from control file CONTROL instead,'
 496                     ' (applies to next test suite only)')
 497
 498         #----------
 499         # argument handling settings (what ways to use action
 500         #  arguments, and pathname processing):
 501
 502         def cb_setah(option, opt_str, value, parser, toset,setval):
 503                 if type(setval) == list:
 504                         if not value in setval:
 505                                 parser.error('value for %s option (%s) is not '
 506                                  'one of the permitted values (%s)' %
 507                                  (value, opt_str, setval.join(' ')))
 508                 elif setval is not None:
 509                         value = setval
 510                 for v in toset:
 511                         arghandling[v] = value
 512                 parser.largs.append(arghandling.copy())
 513
 514         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 515                 type = metavar
 516                 if type is not None: type = 'string'
 517                 pa('',long, action='callback', callback=cb_setah,
 518                    callback_args=(affected,effect), **kwargs)
 519
 520         #---- paths: host or testbed:
 521         #
 522         pa_setah('--paths-testbed', ['tb'],True,
 523                 help='subsequent path specifications refer to the testbed')
 524         pa_setah('--paths-host', ['tb'],False,
 525                 help='subsequent path specifications refer to the host')
 526
 527         #---- source processing settings:
 528
 529         pa_setah('--sources-tests', ['dsc_tests'],True,
 530                 help='run tests from builds of subsequent sources')
 531         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 532                 help='do not run tests from builds of subsequent sources')
 533
 534         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 535                 type='string', metavar='PATTERN-LIST',
 536                 help='from subsequent sources, use binaries matching'
 537                      ' PATTERN-LIST (comma-separated glob patterns)'
 538                      ' according to most recent --binaries-* settings')
 539         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 540                 help='from subsequent sources, do not use any binaries')
 541         #---- binary package processing settings:
 542
 543         def pa_setahbins(long,toset,how):
 544          pa_setah(long, toset,['ignore','auto','install'],
 545                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 546                 help=how+' ignore binaries, install them as needed'
 547                         ' for dependencies, or unconditionally install'
 548                         ' them, respectively')
 549         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 550         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 551         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 552
 553         #----------
 554         # general options:
 555
 556         def cb_vserv(op,optstr,value,parser):
 557                 parser.values.vserver = list(parser.rargs)
 558                 del parser.rargs[:]
 559
 560         def cb_path(op,optstr,value,parser, constructor,long,dir):
 561                 name = long.replace('-','_')
 562                 af = constructor(long, value,arghandling['tb'])
 563                 setattr(parser.values, name, af)
 564
 565         def pa_path(long, constructor, help, dir=False):
 566                 pa('','--'+long, action='callback', callback=cb_path,
 567                         callback_args=(constructor,long,dir),
 568                         nargs=1, type='string',
 569                         help=help, metavar='PATH')
 570
 571         pa_path('output-dir', OutputDir, dir=True,
 572                 help='write stderr/out files in PATH')
 573
 574         pa('--leave-lang', dest='set_lang', action='store_false',
 575                 help="leave LANG on testbed set to testbed's default")
 576         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 577                 help='set LANG on testbed to LANGVAL', default='C')
 578
 579         pa('','--tmp-dir',              type='string', dest='tmpdir',
 580                 help='write temporary files to TMPDIR, emptying it'
 581                      ' beforehand and leaving it behind at the end')
 582         pa('','--log-file',             type='string', dest='logfile',
 583                 help='write the log LOGFILE, emptying it beforehand,'
 584                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 585         pa('','--summary-file',         type='string', dest='summary',
 586                 help='write a summary report to SUMMARY,'
 587                      ' emptying it beforehand')
 588
 589         for k in timeouts.keys():
 590                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 591                         metavar='T', help='set %s timeout to T')
 592         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 593                         metavar='FACTOR', default=1.0,
 594                         help='multiply all default timeouts by FACTOR')
 595
 596         pa('','--user',                 type='string', dest='user',
 597                 help='run tests as USER (needs root on testbed)')
 598         pa('','--gain-root',            type='string', dest='gainroot',
 599                 help='prefix debian/rules binary with GAINROOT')
 600         pa('-q', '--quiet', action='store_true', dest='quiet', default=False);
 601         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 602         pa('','--gnupg-home',           type='string', dest='gnupghome',
 603                 default='~/.autopkgtest/gpg',
 604                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 605                         " signing private apt archive);"
 606                         " `fresh' means generate new key each time.")
 607
 608         #----------
 609         # actual meat:
 610
 611         class SpecialOption(optparse.Option): pass
 612         vs_op = SpecialOption('','--VSERVER-DUMMY')
 613         vs_op.action = 'callback'
 614         vs_op.type = None
 615         vs_op.default = None
 616         vs_op.nargs = 0
 617         vs_op.callback = cb_vserv
 618         vs_op.callback_args = ( )
 619         vs_op.callback_kwargs = { }
 620         vs_op.help = 'introduces virtualisation server and args'
 621         vs_op._short_opts = []
 622         vs_op._long_opts = ['---']
 623
 624         pa(vs_op)
 625
 626         (opts,args) = parser.parse_args()
 627         if not hasattr(opts,'vserver'):
 628                 parser.error('you must specifiy --- <virt-server>...')
 629         if n_non_actions >= len(parser.largs):
 630                 parser.error('nothing to do specified')
 631
 632         for k in timeouts.keys():
 633                 t = getattr(opts,'timeout_'+k)
 634                 if t is None: t = timeouts[k] * opts.timeout_factor
 635                 timeouts[k] = int(t)
 636
 637         arghandling = initial_arghandling
 638         opts.actions = []
 639         ix = 0
 640         for act in args:
 641                 if type(act) == dict:
 642                         arghandling = act
 643                         continue
 644                 elif type(act) == tuple:
 645                         pass
 646                 elif type(act) == str:
 647                         act = (act,act)
 648                 else:
 649                         raise Exception("unknown action in list `%s' having"
 650                               " type `%s'" % (act, type(act)))
 651                 (pathstr, kindpath) = act
 652
 653                 constructor = InputFile
 654                 if type(kindpath) is tuple:             kind = kindpath[0]
 655                 elif kindpath.endswith('.deb'):         kind = 'deb'
 656                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 657                 elif kindpath.endswith('//'):
 658                         kind = 'ubtree'
 659                         constructor = InputDir
 660                 elif kindpath.endswith('/'):
 661                         kind = 'tree'
 662                         constructor = InputDir
 663                 else: parser.error("do not know how to handle filename `%s';"
 664                         " specify --source --binary or --build-tree" %
 665                         kindpath)
 666
 667                 what = '%s%s' % (kind,ix); ix += 1
 668
 669                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 670                 else: fwhatx = '-'+kind
 671
 672                 if pathstr is None: af = None
 673                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 674                 opts.actions.append(Action(kind, af, arghandling, what))
 675
 676 def setup_trace():
 677         global trace_stream, tmpdir, summary_stream
 678
 679         if opts.tmpdir is not None:
 680                 rmtree('tmpdir(specified)',opts.tmpdir)
 681                 mkdir_okexist(opts.tmpdir, 0700)
 682                 tmpdir = opts.tmpdir
 683         else:
 684                 assert(tmpdir is None)
 685                 tmpdir = tempfile.mkdtemp()
 686
 687         if opts.logfile is None:
 688                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 689                         opts.logfile = opts.output_dir.spec + '/log'
 690                 elif opts.tmpdir is not None:
 691                         opts.logfile = opts.tmpdir + '/log'
 692         if opts.logfile is not None:
 693                 trace_stream = open(opts.logfile, 'w', 0)
 694         if opts.summary is not None:
 695                 summary_stream = open(opts.summary, 'w', 0)
 696
 697         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 698
 699 def finalise_options():
 700         global opts, tb, build_essential
 701
 702         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 703                 opts.user = ''
 704
 705         if opts.user is None:
 706                 su = 'suggested-normal-user='
 707                 ul = [
 708                         e[len(su):]
 709                         for e in testbed.caps
 710                         if e.startswith(su)
 711                         ]
 712                 if ul:
 713                         opts.user = ul[0]
 714                 else:
 715                         opts.user = ''
 716
 717         if opts.user:
 718                 if 'root-on-testbed' not in testbed.caps:
 719                         pstderr("warning: virtualisation"
 720                                 " system does not offer root on testbed,"
 721                                 " but --user option specified: failure likely")
 722                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 723         else:
 724                 opts.user_wrap = lambda x: x
 725
 726         if opts.gainroot is None:
 727                 opts.gainroot = ''
 728                 if (opts.user or
 729                     'root-on-testbed' not in testbed.caps):
 730                         opts.gainroot = 'fakeroot'
 731                         build_essential += ['fakeroot']
 732
 733         if opts.gnupghome.startswith('~/'):
 734                 try: home = os.environ['HOME']
 735                 except KeyError:
 736                         parser.error("HOME environment variable"
 737                                 " not set, needed for --gnupghome=`%s"
 738                                 % opts.gnupghome)
 739                 opts.gnupghome = home + opts.gnupghome[1:]
 740         elif opts.gnupghome == 'fresh':
 741                 opts.gnupghome = None
 742
 743 #---------- testbed management - the Testbed class
 744
 745 class Testbed:
 746  def __init__(tb):
 747         tb.sp = None
 748         tb.lastsend = None
 749         tb.scratch = None
 750         tb.modified = False
 751         tb.blamed = []
 752         tb._debug('init')
 753         tb._need_reset_apt = False
 754  def _debug(tb, m, minlevel=0):
 755         debug('** '+m, minlevel)
 756  def start(tb):
 757         tb._debug('start')
 758         p = subprocess.PIPE
 759         debug_subprocess('vserver', opts.vserver)
 760         tb._errplumb = Errplumb(True)
 761         tb.sp = subprocess.Popen(opts.vserver,
 762                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 763         tb.expect('ok')
 764         tb.caps = tb.commandr('capabilities')
 765  def stop(tb):
 766         tb._debug('stop')
 767         tb.close()
 768         if tb.sp is None: return
 769         ec = tb.sp.returncode
 770         if ec is None:
 771                 tb.sp.stdout.close()
 772                 tb.send('quit')
 773                 tb.sp.stdin.close()
 774                 ec = tb.sp.wait()
 775         if ec:
 776                 tb.bomb('testbed gave exit status %d after quit' % ec)
 777         tb._errplumb.wait()
 778  def open(tb):
 779         tb._debug('open, scratch=%s' % tb.scratch)
 780         if tb.scratch is not None: return
 781         pl = tb.commandr('open')
 782         tb._opened(pl)
 783  def _opened(tb, pl):
 784         tb.scratch = InputDir('tb-scratch', pl[0], True)
 785         tb.deps_processed = []
 786         for af in paths: af.invalidate_ephem(True)
 787         tb._auxverbscript_make()
 788  def _auxverbscript_make(tb):
 789         pec = tb.commandr('print-auxverb-command')
 790         if len(pec) < 1: tb.bomb('too few results from print-execute-command')
 791         cmdl = map(urllib.unquote, pec[0].split(','))
 792
 793         tb._debug('cmdl = %s' % (`cmdl`))
 794
 795         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 796         print >>open(tb.ec_auxverbscript.write(),'w'), (
 797 '''#!/bin/sh
 798 set -e
 799 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 800         # This is a pretty nasty hack.  Hopefully it can go away
 801         #  eventually.  See #635763.
 802         set -- dpkg --print-architecture
 803 fi
 804 exec '''+shellquote_cmdl(cmdl)+' "$@"'+"\n"
 805                 )
 806         os.chmod(tb.ec_auxverbscript.write(), 0755)
 807
 808  def mungeing_apt(tb):
 809         if not 'revert' in tb.caps:
 810                 tb._need_reset_apt = True
 811  def reset_apt(tb):
 812         if not tb._need_reset_apt: return
 813         what = 'aptget-update-reset'
 814         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 815         rc = tb.execute(what, cmdl, kind='install')
 816         if rc:
 817                 pstderr("\n" "warning: failed to restore"
 818                         " testbed apt cache, exit code %d" % rc)
 819         what = 'aptconf-reset'
 820         cmdl = ['rm','-f','/etc/apt/apt.conf.d/90autopkgtest',
 821                 '/etc/apt/sources.list.d/autopkgtest.list',
 822                 '/etc/apt/preferences.d/90autopkgtest']
 823         rc = tb.execute(what, cmdl, kind='install')
 824         if rc:
 825                 pstderr("\n" "warning: failed to reset changes"
 826                         " made to testbed apt configuration, exit code %d" % rc)
 827         tb._need_reset_apt = False
 828  def close(tb):
 829         tb._debug('close, scratch=%s' % tb.scratch)
 830         if tb.scratch is None: return
 831         tb.scratch = None
 832         if tb.sp is None: return
 833         tb.command('close')
 834  def prepare1(tb, deps_new):
 835         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 836                 (tb.modified, tb.deps_processed, deps_new), 1)
 837         if 'revert' in tb.caps and (tb.modified or
 838             [d for d in tb.deps_processed if d not in deps_new]):
 839                 for af in paths: af.read(False)
 840                 tb._debug('reset **')
 841                 pl = tb.commandr('revert')
 842                 tb._opened(pl)
 843         tb.modified = False
 844  def prepare2(tb, deps_new):
 845         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 846         binaries.publish()
 847         tb._install_deps(deps_new)
 848  def prepare(tb, deps_new):
 849         tb.prepare1(deps_new)
 850         tb.prepare2(deps_new)
 851  def _install_deps(tb, deps_new):
 852         tb._debug(' installing dependencies '+`deps_new`, 1)
 853         tb.deps_processed = deps_new
 854         if not deps_new: return
 855         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 856  def needs_reset(tb):
 857         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 858         tb.modified = True
 859  def blame(tb, m):
 860         tb._debug('blame += %s' % m, 1)
 861         tb.blamed.append(m)
 862  def bomb(tb, m):
 863         tb._debug('bomb %s' % m)
 864         if tb.sp is not None:
 865                 tb.sp.stdout.close()
 866                 tb.sp.stdin.close()
 867                 ec = tb.sp.wait()
 868                 if ec: pstderr('adt-run: testbed failing,'
 869                                 ' exit status %d' % ec)
 870         tb.sp = None
 871         raise Quit(16, 'testbed failed: %s' % m)
 872  def send(tb, string):
 873         tb.sp.stdin
 874         try:
 875                 debug('>> '+string, 2)
 876                 print >>tb.sp.stdin, string
 877                 tb.sp.stdin.flush()
 878                 tb.lastsend = string
 879         except:
 880                 (type, value, dummy) = sys.exc_info()
 881                 tb.bomb('cannot send to testbed: %s' % traceback.
 882                         format_exception_only(type, value))
 883  def expect(tb, keyword, nresults=None):
 884         l = tb.sp.stdout.readline()
 885         if not l: tb.bomb('unexpected eof from the testbed')
 886         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 887         l = l.rstrip('\n')
 888         debug('<< '+l, 2)
 889         ll = l.split()
 890         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 891         if ll[0] != keyword:
 892                 if tb.lastsend is None:
 893                         tb.bomb("got banner `%s', expected `%s...'" %
 894                                 (l, keyword))
 895                 else:
 896                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 897                                 (tb.lastsend, l, keyword))
 898         ll = ll[1:]
 899         if nresults is not None and len(ll) != nresults:
 900                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 901                         " expected %d result parameters" %
 902                         (tb.lastsend, l, len(ll), nresults))
 903         return ll
 904  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 905         # pass args=[None,...] or =(None,...) to avoid more url quoting
 906         if type(cmd) is str: cmd = [cmd]
 907         if len(args) and args[0] is None: args = args[1:]
 908         else: args = map(urllib.quote, args)
 909         al = cmd + args
 910         tb.send(string.join(al))
 911         ll = tb.expect('ok', nresults)
 912         if unquote: ll = map(urllib.unquote, ll)
 913         return ll
 914  def command(tb, cmd, args=()):
 915         tb.commandr(cmd, args, 0)
 916  def commandr1(tb, cmd, args=()):
 917         rl = tb.commandr(cmd, args, 1)
 918         return rl[0]
 919  def execute(tb, what, cmdl,
 920                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 921                 script=False, tmpdir=None, kind='short'):
 922         # Options for script:
 923         #   False - do not call debug_subprocess, no synch. reporting required
 924         #   None or string - call debug_subprocess with that value,
 925         #                       plumb stderr through synchronously if possible
 926         # Options for se:
 927         #   None - usual Errplumb (output is of kind 2c)
 928         #   string - path on testbed (output is of kind 2d)
 929
 930         timeout = timeouts[kind]
 931
 932         if script is not False: debug_subprocess(what, cmdl, script=script)
 933         if cwd is None: cwd = tb.scratch.write(True)
 934
 935         xdump = None
 936         if se is None:
 937                 ep = Errplumb()
 938                 se_catch = TemporaryFile(what+'-xerr')
 939                 se_use = se_catch.write(True)
 940                 if not opts.quiet: xdump = 'debug=2-2'
 941                 elif trace_stream is not None:
 942                         xdump = 'debug=2-%d' % trace_stream.fileno()
 943         else:
 944                 ep = None
 945                 se_catch = None
 946                 se_use = se
 947
 948         cmdl = [None,
 949                 ','.join(map(urllib.quote, cmdl)),
 950                 si, so, se_use, cwd]
 951
 952         if timeout is not None and timeout > 0:
 953                 cmdl.append('timeout=%d' % timeout)
 954
 955         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 956         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 957         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 958         if opts.set_lang is not False:
 959                 cmdl.append('env=LANG=%s' % opts.set_lang)
 960
 961         rc = tb.commandr1('execute', cmdl)
 962         try: rc = int(rc)
 963         except ValueError: bomb("execute for %s gave invalid response `%s'"
 964                                         % (what,rc))
 965
 966         if se_catch is not None:
 967                 debug_file(se_catch.read())
 968         if ep is not None:
 969                 ep.wait()
 970
 971         return rc
 972
 973  def satisfy_dependencies_string(tb, deps, what):
 974         # Must have called Binaries.configure_apt
 975         debug('dependencies: %s: satisfying %s' % (what,deps))
 976         dsc = TemporaryFile('deps.dsc')
 977         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 978         # pbuilder-satisfydepends has a bug where it ignores the
 979         #  Build-Depends if it's the last line in the dsc (#635696)
 980         tb.satisfy_dependencies_dsc(dsc, what)
 981
 982  def satisfy_dependencies_dsc(tb, dsc, what):
 983         # Must have called Binaries.configure_apt
 984         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 985                 '--binary-all', # --check-key
 986                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 987                 '-c',dsc.read()
 988         ]
 989         # The --internal-chrootexec option is really handy but
 990         #  perhaps we are not supposed to use it ?  See also #635697.
 991         debug('dependencies: %s: running %s' % (what,`cmdl`))
 992         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 993         if rc: badpkg('dependency install failed, exit code %d' % rc)
 994
 995 #---------- representation of test control files: Field*, Test, etc.
 996
 997 class FieldBase:
 998  def __init__(f, fname, stz, base, tnames, vl):
 999         assert(vl)
1000         f.stz = stz
1001         f.base = base
1002         f.tnames = tnames
1003         f.vl = vl
1004  def words(f):
1005         def distribute(vle):
1006                 (lno, v) = vle
1007                 r = v.split()
1008                 r = map((lambda w: (lno, w)), r)
1009                 return r
1010         return flatten(map(distribute, f.vl))
1011  def atmostone(f):
1012         if len(vl) == 1:
1013                 (f.lno, f.v) = vl[0]
1014         else:
1015                 raise Unsupported(f.vl[1][0],
1016                         'only one %s field allowed' % fn)
1017         return f.v
1018
1019 class FieldIgnore(FieldBase):
1020  def parse(f): pass
1021
1022 class Restriction:
1023  def __init__(r,rname,base): pass
1024
1025 class Restriction_rw_build_tree(Restriction): pass
1026 class Restriction_build_needed(Restriction): pass
1027 class Restriction_breaks_testbed(Restriction):
1028  def __init__(r, rname, base):
1029         if 'revert-full-system' not in testbed.caps:
1030                 raise Unsupported(-1,
1031         'Test breaks testbed but testbed does not advertise revert-full-system')
1032 class Restriction_needs_root(Restriction):
1033  def __init__(r, rname, base):
1034         if 'root-on-testbed' not in testbed.caps:
1035                 raise Unsupported(-1,
1036                         'Test needs root on testbed which is not available')
1037
1038 class Field_Restrictions(FieldBase):
1039  def parse(f):
1040         for wle in f.words():
1041                 (lno, rname) = wle
1042                 nrname = rname.replace('-','_')
1043                 try: rclass = globals()['Restriction_'+nrname]
1044                 except KeyError: raise Unsupported(lno,
1045                         'unknown restriction %s' % rname)
1046                 r = rclass(nrname, f.base)
1047                 f.base['restriction_names'].append(rname)
1048                 f.base['restrictions'].append(r)
1049
1050 class Field_Features(FieldIgnore):
1051  def parse(f):
1052         for wle in f.words():
1053                 (lno, fname) = wle
1054                 f.base['feature_names'].append(fname)
1055                 nfname = fname.replace('-','_')
1056                 try: fclass = globals()['Feature_'+nfname]
1057                 except KeyError: continue
1058                 ft = fclass(nfname, f.base)
1059                 f.base['features'].append(ft)
1060
1061 class Field_Tests(FieldIgnore): pass
1062
1063 class Field_Depends(FieldBase):
1064  def parse(f):
1065         debug("Field_DependS: %s %s %s %s" % (f.stz, f.base, f.tnames, f.vl), 2)
1066         dl = map(lambda x: x.strip(),
1067                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1068         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1069         for d in dl:
1070                 if re.search(d):
1071                         badpkg("Test Depends field contains dependency"
1072                                " `%s' with invalid characters" % d)
1073         f.base['depends'] = dl
1074
1075 class Field_Tests_directory(FieldBase):
1076  def parse(f):
1077         td = atmostone(f)
1078         if td.startswith('/'): raise Unspported(f.lno,
1079                 'Tests-Directory may not be absolute')
1080         f.base['testsdir'] = td
1081
1082 def run_tests(stanzas, tree):
1083         global errorcode, testbed
1084         if stanzas == ():
1085                 report('*', 'SKIP no tests in this package')
1086                 errorcode |= 8
1087         for stanza in stanzas:
1088                 tests = stanza[' tests']
1089                 if not tests:
1090                         report('*', 'SKIP package has metadata but no tests')
1091                         errorcode |= 8
1092                 for t in tests:
1093                         t.prepare()
1094                         t.run(tree)
1095                         if 'breaks-testbed' in t.restriction_names:
1096                                 testbed.needs_reset()
1097                 testbed.needs_reset()
1098
1099 class Test:
1100  def __init__(t, tname, base, act):
1101         if '/' in tname: raise Unsupported(base[' lno'],
1102                 'test name may not contain / character')
1103         for k in base: setattr(t,k,base[k])
1104         t.tname = tname
1105         t.act = act
1106         t.what = act.what+'t-'+tname
1107         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1108         else: t.path = tname
1109         t._debug('constructed; path=%s' % t.path)
1110         t._debug(' .depends=%s' % t.depends)
1111  def _debug(t, m):
1112         debug('& %s: %s' % (t.what, m))
1113  def report(t, m):
1114         report(t.what, m)
1115  def reportfail(t, m):
1116         global errorcode
1117         errorcode |= 4
1118         report(t.what, 'FAIL ' + m)
1119  def prepare(t):
1120         t._debug('preparing')
1121         dn = []
1122         for d in t.depends:
1123                 t._debug(' processing dependency '+d)
1124                 if not '@' in d:
1125                         t._debug('  literal dependency '+d)
1126                         dn.append(d)
1127                 else:
1128                         for (pkg,bin) in t.act.binaries:
1129                                 dp = d.replace('@',pkg)
1130                                 t._debug('  synthesised dependency '+dp)
1131                                 dn.append(dp)
1132         testbed.prepare(dn)
1133  def run(t, tree):
1134         t._debug('[----------------------------------------')
1135         def stdouterr(oe):
1136                 idstr = t.what + '-' + oe
1137                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1138                         use_dir = opts.output_dir
1139                 else:
1140                         use_dir = testbed.scratch
1141                 return RelativeOutputFile(idstr, use_dir, idstr)
1142
1143         if hasattr(t.act,'work'): t.act.work.write(True)
1144         tree.read(True)
1145
1146         af = RelativeInputFile(t.what, tree, t.path)
1147         so = stdouterr('stdout')
1148         se = stdouterr('stderr')
1149
1150         tf = af.read(True)
1151         tmpdir = None
1152
1153         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1154         if rc: bomb('failed to chmod +x %s' % tf)
1155
1156         if 'needs-root' not in t.restriction_names and opts.user is not None:
1157                 tfl = ['su',opts.user,'-c',tf]
1158                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1159                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1160                 if opts.user:
1161                         script += '; chown %s "$1"' % opts.user
1162                         if 'rw-build-tree' in t.restriction_names:
1163                                 script += '; chown -R %s "$2"' % opts.user
1164                 rc = testbed.execute('mktmpdir-'+t.what,
1165                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1166                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1167                                 % (tmpdir, rc))
1168         else:
1169                 tfl = [tf]
1170
1171         rc = testbed.execute('test-'+t.what, tfl,
1172                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1173                 tmpdir=tmpdir, kind='test')
1174
1175         so_read = so.read()
1176         se_read = se.read()
1177
1178         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1179         stab = os.stat(se_read)
1180         if stab.st_size != 0:
1181                 l = open(se_read).readline()
1182                 l = l.rstrip('\n \t\r')
1183                 if len(l) > 35: l = l[:35] + '...'
1184                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1185                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1186                 debug_file(se_read)
1187         elif rc != 0:
1188                 t.reportfail('non-zero exit status %d' % rc)
1189         else:
1190                 t.report('PASS')
1191
1192         stab = os.stat(so_read)
1193         if stab.st_size != 0:
1194                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1195                 debug_file(so_read)
1196
1197         t._debug('----------------------------------------]')
1198
1199 def read_control(act, tree, control_override):
1200         stanzas = [ ]
1201
1202         if control_override is not None:
1203                 control_af = control_override
1204                 testbed.blame('arg:'+control_override.spec)
1205         else:
1206                 if act.missing_tests_control:
1207                         return ()
1208                 control_af = RelativeInputFile(act.what+'-testcontrol',
1209                         tree, 'debian/tests/control')
1210         try:
1211                 control = open(control_af.read(), 'r')
1212         except (IOError,OSError), oe:
1213                 if oe[0] != errno.ENOENT: raise
1214                 return []
1215
1216         lno = 0
1217         def badctrl(m): testbed.bomb('tests/control line %d: %s' % (lno, m))
1218         stz = { }       # stz[field_name][index] = (lno, value)
1219                         # special field names:
1220                         # stz[' lno'] = number
1221                         # stz[' tests'] = list of Test objects
1222                         # empty dictionary means we're between stanzas
1223         def in_stanza(stz):
1224                 return stz.has_key(' lno')
1225         def end_stanza(stz):
1226                 if not in_stanza(stz): return
1227                 stz[' errs'] = 0
1228                 stanzas.append(stz.copy())
1229                 stz.clear()
1230                 hcurrent = None
1231
1232         initre = regexp.compile('([A-Z][-0-9a-zA-Z]*)\s*\:\s*(.*)$')
1233         while 1:
1234                 l = control.readline()
1235                 if not l: break
1236                 lno += 1
1237                 if not l.endswith('\n'): badctrl('unterminated line')
1238                 if regexp.compile('\s*\#').match(l): continue
1239                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1240                 initmat = initre.match(l)
1241                 if initmat:
1242                         (fname, l) = initmat.groups()
1243                         fname = string.capwords(fname)
1244                         if not in_stanza(stz):
1245                                 stz = { ' lno': lno, ' tests': [] }
1246                         if not stz.has_key(fname): stz[fname] = [ ]
1247                         hcurrent = stz[fname]
1248                 elif regexp.compile('\s').match(l):
1249                         if not hcurrent: badctrl('unexpected continuation')
1250                 else:
1251                         badctrl('syntax error')
1252                 hcurrent.append((lno, l))
1253         end_stanza(stz)
1254
1255         def testbadctrl(stz, lno, m):
1256                 report_badctrl(lno, m)
1257                 stz[' errs'] += 1
1258
1259         for stz in stanzas:
1260                 try:
1261                         try: tnames = stz['Tests']
1262                         except KeyError:
1263                                 tnames = ['*']
1264                                 raise Unsupported(stz[' lno'],
1265                                         'no Tests field')
1266                         tnames = map((lambda lt: lt[1]), tnames)
1267                         tnames = string.join(tnames).split()
1268                         base = {
1269                                 'restriction_names': [],
1270                                 'restrictions': [],
1271                                 'feature_names': [],
1272                                 'features': [],
1273                                 'testsdir': 'debian/tests',
1274                                 'depends' : '@'
1275                         }
1276                         for fname in stz.keys():
1277                                 if fname.startswith(' '): continue
1278                                 vl = stz[fname]
1279                                 try: fclass = globals()['Field_'+
1280                                         fname.replace('-','_')]
1281                                 except KeyError: raise Unsupported(vl[0][0],
1282                                         'unknown metadata field %s' % fname)
1283                                 f = fclass(stz, fname, base, tnames, vl)
1284                                 f.parse()
1285                         for tname in tnames:
1286                                 t = Test(tname, base, act)
1287                                 stz[' tests'].append(t)
1288                 except Unsupported, u:
1289                         for tname in tnames: u.report(tname)
1290                         continue
1291
1292         return stanzas
1293
1294 def print_exception(ei, msgprefix=''):
1295         if msgprefix: pstderr(msgprefix)
1296         (et, q, tb) = ei
1297         if et is Quit:
1298                 pstderr('adt-run: ' + q.m)
1299                 psummary('quitting: '+q.m)
1300                 return q.ec
1301         else:
1302                 pstderr("adt-run: unexpected, exceptional, error:")
1303                 psummary('quitting: unexpected error, consult transcript')
1304                 traceback.print_exc(None, sys.stderr)
1305                 if trace_stream is not None:
1306                         traceback.print_exc(None, trace_stream)
1307                 return 20
1308
1309 def cleanup():
1310         global trace_stream
1311         try:
1312                 if testbed is not None:
1313                         testbed.reset_apt()
1314                         testbed.stop()
1315                 if opts.tmpdir is None and tmpdir is not None:
1316                         rmtree('tmpdir', tmpdir)
1317                 if trace_stream is not None:
1318                         trace_stream.close()
1319                         trace_stream = None
1320         except:
1321                 print_exception(sys.exc_info(),
1322                         '\nadt-run: error cleaning up:\n')
1323                 os._exit(20)
1324
1325 #---------- registration, installation etc. of .deb's: Binaries
1326
1327 def determine_package(act):
1328         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1329         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1330         if rc: badpkg('failed to parse binary package, code %d' % rc)
1331         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1332         act.pkg = None
1333         for l in output.split('\n'):
1334                 m = re.match(l)
1335                 if not m: continue
1336                 if act.pkg: badpkg('two Package: lines in control file')
1337                 act.pkg = m.groups()[0]
1338         if not act.pkg: badpkg('no good Package: line in control file')
1339
1340 class Binaries:
1341  def __init__(b, tb):
1342         b.dir = TemporaryDir('binaries')
1343         b.dir.write()
1344         ok = False
1345
1346         if opts.gnupghome is None:
1347                 opts.gnupghome = tmpdir+'/gnupg'
1348
1349         b._debug('initialising')
1350         try:
1351                 for x in ['pubring','secring']:
1352                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1353                 ok = True
1354         except (IOError,OSError), oe:
1355                 if oe.errno != errno.ENOENT: raise
1356
1357         if ok: b._debug('no key generation needed')
1358         else: b.genkey()
1359
1360  def _debug(b, s):
1361         debug('* '+s)
1362
1363  def genkey(b):
1364         b._debug('preparing for key generation')
1365
1366         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1367         mkdir_okexist(opts.gnupghome, 0700)
1368
1369         script = '''
1370   exec >&2
1371   cd "$1"
1372   cat <<"END" >key-gen-params
1373 Key-Type: DSA
1374 Key-Length: 1024
1375 Key-Usage: sign
1376 Name-Real: autopkgtest per-run key
1377 Name-Comment: do not trust this key
1378 Name-Email: autopkgtest@example.com
1379 END
1380   set -x
1381   gpg --homedir="$1" --batch --gen-key key-gen-params
1382                 '''
1383         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1384         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1385         if rc: bomb('key generation failed, code %d' % rc)
1386
1387  def apt_configs(b):
1388         return {
1389                 "Debug::pkgProblemResolver": "true",
1390                 "APT::Get::force-yes" : "true",
1391                 "APT::Get::Assume-Yes" : "true",
1392                 "quiet" : "true",
1393         }
1394
1395  def _configure_apt(b, tb):
1396         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1397                         True)
1398         f = open(config.write(),'w')
1399         for (k,v) in b.apt_configs().iteritems():
1400                 print >>f, '%s { "%s"; };' % (k, v)
1401         f.close()
1402         config.read(True)
1403
1404         prefs = OutputFile('apt-prefs','/etc/apt/preferences.d/90autopkgtest',
1405                         True)
1406         print >>open(prefs.write(),'w'), '''
1407 Package: *
1408 Pin: origin ""
1409 Pin-Priority: 1002
1410 '''
1411         prefs.read(True)
1412
1413  def _apt_get(b):
1414         ag = ['apt-get','-q']
1415         for kv in b.apt_configs().iteritems():
1416                 ag += ['-o', '%s=%s' % kv]
1417         return ' '.join(ag)
1418
1419  def reset(b):
1420         b._debug('reset')
1421         rmtree('binaries', b.dir.read())
1422         b.dir.invalidate()
1423         b.dir.write()
1424         b.install = []
1425         b.blamed = []
1426         b.registered = set()
1427
1428  def register(b, act, pkg, af, forwhat, blamed):
1429         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1430                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1431
1432         if act.ah['deb_'+forwhat] == 'ignore': return
1433
1434         b.blamed += testbed.blamed
1435
1436         leafname = pkg+'.deb'
1437         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1438
1439         try: os.remove(dest.write())
1440         except (IOError,OSError), oe:
1441                 if oe.errno != errno.ENOENT: raise e
1442
1443         try: os.link(af.read(), dest.write())
1444         except (IOError,OSError), oe:
1445                 if oe.errno != errno.EXDEV: raise e
1446                 shutil.copy(af.read(), dest)
1447
1448         if act.ah['deb_'+forwhat] == 'install':
1449                 b.install.append(pkg)
1450
1451         b.registered.add(pkg)
1452
1453  def publish(b):
1454         b._debug('publish')
1455
1456         b._configure_apt(testbed)
1457
1458         script = '''
1459   exec >&2
1460   cd "$1"
1461   apt-ftparchive packages . >Packages
1462   gzip <Packages >Packages.gz
1463   apt-ftparchive release . >Release
1464   rm -f Release.gpg
1465   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1466   gpg --homedir="$2" --batch --export >archive-key.pgp
1467                 '''
1468         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1469         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1470         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1471
1472         b.dir.invalidate(True)
1473         apt_source = b.dir.read(True)
1474
1475         so = TemporaryFile('vlds')
1476         script = '''
1477   exec 3>&1 >&2
1478   apt-key add archive-key.pgp
1479   echo "deb file://'''+apt_source+''' /" >/etc/apt/sources.list.d/autopkgtest.list
1480   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1481     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1482   fi
1483   '''+ b._apt_get() +''' update >&2
1484   cat /var/lib/dpkg/status >&3
1485                 '''
1486         testbed.mungeing_apt()
1487         rc = testbed.execute('apt-key', ['sh','-ec',script],
1488                                 so=so.write(True), cwd=b.dir.write(True),
1489                                 script=script, kind='install')
1490         if rc: bomb('apt setup failed with exit code %d' % rc)
1491
1492         testbed.blamed += b.blamed
1493
1494         b._debug('publish reinstall checking...')
1495         pkgs_reinstall = set()
1496         pkg = None
1497         for l in open(so.read()):
1498                 if l.startswith('Package: '):
1499                         pkg = l[9:].rstrip()
1500                 elif l.startswith('Status: install '):
1501                         if pkg in b.registered:
1502                                 pkgs_reinstall.add(pkg)
1503                                 b._debug(' publish reinstall needs '+pkg)
1504
1505         if pkgs_reinstall:
1506                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1507                 what = 'apt-get-reinstall'
1508                 cmdl = (b._apt_get() + ' --reinstall install '+
1509                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1510                 cmdl = ['sh','-c',cmdl]
1511                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1512                 if rc: badpkg("installation of basic binarries failed,"
1513                                 " exit code %d" % rc)
1514
1515         b._debug('publish install...')
1516         for pkg in b.install:
1517                 what = 'apt-get-install-%s' % pkg
1518                 testbed.blame(pkg)
1519                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1520                 cmdl = ['sh','-c',cmdl]
1521                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1522                 if rc: badpkg("installation of %s failed, exit code %d"
1523                                 % (pkg, rc))
1524
1525         b._debug('publish done')
1526
1527 #---------- processing of sources (building)
1528
1529 def source_rules_command(act,script,what,which,work,cwd,
1530                                 results_lines=0,xargs=[]):
1531         script = [      "exec 3>&1 >&2",
1532                         "set -x"        ] + script
1533         script = '\n'.join(script)
1534         so = TemporaryFile('%s-%s-results' % (what,which))
1535         rc = testbed.execute('%s-%s' % (what,which),
1536                         ['sh','-ec',script]+xargs, script=script,
1537                         so=so.write(True), cwd=cwd, kind='build')
1538         results = open(so.read()).read().rstrip('\n')
1539         if len(results): results = results.split("\n")
1540         else: results = []
1541         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1542         if results_lines is not None and len(results) != results_lines:
1543                 badpkg("got %d lines of results from %s where %d expected"
1544                         % (len(results), which, results_lines))
1545         if results_lines==1: return results[0]
1546         return results
1547
1548 def build_source(act, control_override):
1549         act.blame = 'arg:'+act.af.spec
1550         testbed.blame(act.blame)
1551         testbed.prepare1([])
1552         testbed.needs_reset()
1553
1554         what = act.what
1555         basename = act.af.spec
1556         debiancontrol = None
1557         act.binaries = []
1558
1559         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1560
1561         if act.kind == 'dsc':
1562                 dsc = act.af
1563                 dsc_file = open(dsc.read())
1564                 in_files = False
1565                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1566                 for l in dsc_file:
1567                         l = l.rstrip('\n')
1568                         if l.startswith('Files:'): in_files = True; continue
1569                         elif l.startswith('#'): pass
1570                         elif not l.startswith(' '):
1571                                 in_files = False
1572                                 if l.startswith('Source:'):
1573                                         act.blame = 'dsc:'+l[7:].strip()
1574                                         testbed.blame(act.blame)
1575                         if not in_files: continue
1576
1577                         m = fre.match(l)
1578                         if not m: badpkg(".dsc contains unparseable line"
1579                                         " in Files: `%s'" % l)
1580                         leaf = m.groups(0)[0]
1581                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1582                                         sibling=True)
1583                         subfile.read(True)
1584                 dsc.read(True)
1585
1586         if act.kind == 'ubtree':
1587                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1588                         act.af, 'debian/control')
1589                 dsc = TemporaryFile(what+'-fakedsc')
1590                 dsc_w = open(dsc.write(), 'w')
1591                 for l in open(debiancontrol.read()):
1592                         l = l.rstrip('\n')
1593                         if not len(l): break
1594                         print >>dsc_w, l
1595                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1596                 dsc_w.close()
1597
1598         if act.kind == 'dsc':
1599                 testbed.prepare2([])
1600                 testbed.satisfy_dependencies_string('dpkg-dev',
1601                                                 'install dpkg-dev')
1602
1603         work = TemporaryDir(what+'-build')
1604         act.work = work
1605
1606         tmpdir = work.write(True)+'/tmpdir'
1607         tmpdir_script = [
1608                         'TMPDIR="$1"',
1609                         'rm -rf -- "$TMPDIR"',
1610                         'export TMPDIR',
1611                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1612                 ]
1613
1614         if act.kind == 'ubtree':
1615                 spec = '%s/real-tree' % work.write(True)
1616                 create_command = '''
1617                         rm -rf "$spec"
1618                         mkdir "$spec"
1619                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1620                         '''
1621                 initcwd = act.af.read(True)
1622
1623         if act.kind == 'dsc':
1624                 spec = dsc.read(True)
1625                 create_command = '''
1626                         dpkg-source -x $spec
1627                         '''
1628                 initcwd = work.write(True)
1629
1630         script = [
1631                 'spec="$2"',
1632                 'origpwd=`pwd`',
1633                 'cd '+work.write(True)
1634         ]
1635
1636         if opts.user:
1637                 script += ([ 'chown '+opts.user+' .' ] +
1638                         tmpdir_script +
1639                         [ 'spec="$spec" origpwd="$origpwd" '
1640                                 +opts.user_wrap(create_command) ])
1641         else:
1642                 script += (tmpdir_script +
1643                         [ create_command ])
1644
1645         script += [
1646                         'cd [a-z0-9]*-*/.',
1647                         'pwd >&3',
1648                         'set +e; test -f debian/tests/control; echo $? >&3'
1649                 ]
1650         (result_pwd, control_test_rc) = source_rules_command(
1651                         act,script,what,'extract',work,
1652                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1653
1654         filter = act.ah['dsc_filter']
1655
1656         if control_test_rc == '1': act.missing_tests_control = True
1657
1658         # For optional builds:
1659         #
1660         # We might need to build the package because:
1661         #   - we want its binaries (filter isn't _ and at least one of the
1662         #       deb_... isn't ignore)
1663         #   - the test control file says so
1664         #       (assuming we have any tests)
1665
1666         class NeedBuildException: pass
1667         def build_needed(m):
1668                 debug_b('build needed for %s' % m)
1669                 raise NeedBuildException()
1670
1671         try:
1672                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1673                                       act.ah['deb_fortests'] != 'ignore'):
1674                         build_needed('binaries')
1675
1676                 result_pwd_af = InputDir(what+'-treeforcontrol',
1677                         result_pwd, True)
1678                 stanzas = read_control(act, result_pwd_af, control_override)
1679                 for stanza in stanzas:
1680                         for t in stanza[' tests']:
1681                                 if 'build-needed' in t.restriction_names:
1682                                         build_needed('test %s' % t.tname)
1683
1684                 debug_b('build not needed')
1685                 built = False
1686
1687         except NeedBuildException:
1688
1689                 if act.kind != 'dsc':
1690                         testbed.prepare2([])
1691
1692                 testbed.satisfy_dependencies_string('build-essential',
1693                                 'install build-essential')
1694                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1695
1696                 script = tmpdir_script + [
1697                         'cd "$2"',
1698                         'dpkg-checkbuilddeps',
1699                         opts.user_wrap('debian/rules build'),
1700                         ]
1701                 source_rules_command(act,script,what,'build',work,
1702                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1703
1704                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1705                         badpkg("results dir `%s' is not in expected parent"
1706                                " dir `%s'" % (result_pwd, work.read(True)))
1707
1708                 built = True
1709
1710         act.tests_tree = RelativeInputDir(what+'-tests-tree',
1711                                 work, os.path.basename(result_pwd),
1712                                 True)
1713
1714         if not built:
1715                 act.blamed = []
1716                 return
1717
1718         act.blamed = copy.copy(testbed.blamed)
1719
1720         debug_b('filter=%s' % filter)
1721         if filter != '_':
1722                 script = tmpdir_script + [
1723                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1724                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1725                         'cd ..',
1726                         'echo *.deb >&3',
1727                         ]
1728                 result_debs = source_rules_command(act,script,what,
1729                                 'binary',work,work.write(True),
1730                                 results_lines=1, xargs=['x',tmpdir])
1731                 if result_debs == '*.deb': debs = []
1732                 else: debs = result_debs.split(' ')
1733                 debug_b('debs='+`debs`)
1734                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1735                 for deb in debs:
1736                         m = re.match(deb)
1737                         if not m: badpkg("badly-named binary `%s'" % deb)
1738                         pkg = m.groups()[0]
1739                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1740                         for pat in filter.split(','):
1741                                 debug_b('  pat=%s' % pat)
1742                                 if not fnmatch.fnmatchcase(pkg,pat):
1743                                         debug_b('   no match')
1744                                         continue
1745                                 deb_what = pkg+'_'+what+'.deb'
1746                                 bin = RelativeInputFile(deb_what,work,deb,True)
1747                                 debug_b('  deb_what=%s, bin=%s' %
1748                                         (deb_what, str(bin)))
1749                                 binaries.register(act,pkg,bin,
1750                                         'forbuilds',testbed.blamed)
1751                                 act.binaries.append((pkg,bin))
1752                                 break
1753                 debug_b('all done.')
1754
1755 #---------- main processing loop and main program
1756
1757 def process_actions():
1758         global binaries
1759
1760         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1761         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1762         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1763
1764         debug_a1('starting')
1765         testbed.open()
1766         binaries = Binaries(testbed)
1767
1768         binaries.reset()
1769         control_override = None
1770
1771         debug_a1('builds ...')
1772         for act in opts.actions:
1773                 debug_a2('%s %s' %
1774                         (act.kind, act.what))
1775
1776                 if act.kind == 'control':
1777                         control_override = act.af
1778                 if act.kind == 'deb':
1779                         testbed.blame('arg:'+act.af.spec)
1780                         determine_package(act)
1781                         testbed.blame('deb:'+act.pkg)
1782                         binaries.register(act,act.pkg,act.af,
1783                                 'forbuilds',testbed.blamed)
1784                 if act.kind == 'dsc' or act.kind == 'ubtree':
1785                         build_source(act, control_override)
1786                 if act.kind == 'tree':
1787                         act.binaries = []
1788                 if act.kind.endswith('tree') or act.kind == 'dsc':
1789                         control_override = None
1790                 if act.kind == 'instantiate':
1791                         pass
1792
1793         debug_a1('builds done.')
1794
1795         binaries.reset()
1796         control_override = None
1797
1798         debug_a1('tests ...')
1799         for act in opts.actions:
1800                 debug_a2('test %s %s' % (act.kind, act.what))
1801
1802                 testbed.needs_reset()
1803                 if act.kind == 'control':
1804                         control_override = act.af
1805                 if act.kind == 'deb':
1806                         binaries.register(act,act.pkg,act.af,'fortests',
1807                                 ['deb:'+act.pkg])
1808                 if act.kind == 'dsc' or act.kind == 'ubtree':
1809                         for (pkg,bin) in act.binaries:
1810                                 binaries.register(act,pkg,bin,'fortests',
1811                                         act.blamed)
1812                 if act.kind == 'dsc':
1813                         if act.ah['dsc_tests']:
1814                                 debug_a3('read control ...')
1815                                 stanzas = read_control(act, act.tests_tree,
1816                                                 control_override)
1817                                 testbed.blamed += act.blamed
1818                                 debug_a3('run_tests ...')
1819                                 run_tests(stanzas, act.tests_tree)
1820                         control_override = None
1821                 if act.kind == 'tree' or act.kind == 'ubtree':
1822                         testbed.blame('arg:'+act.af.spec)
1823                         stanzas = read_control(act, act.af, control_override)
1824                         debug_a3('run_tests ...')
1825                         run_tests(stanzas, act.af)
1826                         control_override = None
1827                 if act.kind == 'instantiate':
1828                         testbed.prepare([])
1829         debug_a1('tests done.')
1830
1831 def main():
1832         global testbed
1833         global tmpdir
1834         try:
1835                 parse_args()
1836         except SystemExit, se:
1837                 os._exit(20)
1838         try:
1839                 setup_trace()
1840                 testbed = Testbed()
1841                 testbed.start()
1842                 finalise_options()
1843                 process_actions()
1844         except:
1845                 ec = print_exception(sys.exc_info(), '')
1846                 cleanup()
1847                 os._exit(ec)
1848         cleanup()
1849         os._exit(errorcode)
1850
1851 main()