runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':10, 'install':900, 'test':600, 'build':3000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         try: shutil.rmtree(pathname)
 192         except (IOError,OSError), oe:
 193                 if oe.errno != errno.EEXIST: raise
 194
 195 def debug_subprocess(what, cmdl=None, script=None):
 196         o = '$ '+what+':'
 197         if cmdl is not None:
 198                 ol = []
 199                 for x in cmdl:
 200                         if x is script: x = '<SCRIPT>'
 201                         ol.append(x.    replace('\\','\\\\').
 202                                         replace(' ','\\ ')      )
 203                 o += ' '+ ' '.join(ol)
 204         debug(o)
 205         if script is not None and opts.debuglevel >= 1:
 206                 o = ''
 207                 for l in script.rstrip('\n').split('\n'):
 208                         o += '$     '+l+'\n'
 209                 debug(o,1)
 210
 211 def flatten(l):
 212         return reduce((lambda a,b: a + b), l, [])
 213
 214 #---------- fancy automatic file-copying class
 215
 216 class AutoFile:
 217         # p.what
 218         # p.path[tb]    None or path    not None => path known
 219         # p.file[tb]    None or path    not None => file exists
 220         # p.spec        string or None
 221         # p.spec_tbp    True or False, or not set if spec is None
 222         # p.dir         '' or '/'
 223
 224  def __init__(p, what):
 225         p.what = what
 226         p.path = [None,None]
 227         p.file = [None,None]
 228         p.spec = None
 229         p.dir = ''
 230
 231  def __repr__(p):
 232         return "<AF@%s>" % p.__str__()
 233  def __str__(p):
 234         def ptbp(tbp):
 235                 if p.path[tbp] is None: return '-'+p.dir
 236                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 237                 else: return p.path[tbp]+p.dir+'!'
 238         out = p.what
 239         if p.spec is not None:
 240                 if not hasattr(p,'spec_tb'): out += '~'
 241                 elif p.spec_tbp: out += '#'
 242                 else: out += '='
 243                 out += p.spec
 244         out += ':'
 245         out += ptbp(False)
 246         out += '|'
 247         out += ptbp(True)
 248         return out
 249
 250  def _wrong(p, how):
 251         xtra = ''
 252         if p.spec is not None:
 253                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 254         raise ("internal error: %s (%s)" % (how, str(p)))
 255
 256  def _ensure_path(p, tbp):
 257         if p.path[tbp] is None:
 258                 if '/' in p.what:
 259                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 260                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 261                 if not tbp:
 262                         p.path[tbp] = tmpdir+'/'+p.what
 263                 else:
 264                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 265
 266  def write(p, tbp=False):
 267         p._debug('write %s...' % 'HT'[tbp])
 268         p._ensure_path(tbp)
 269
 270         if p.dir and not p.file[tbp]:
 271                 if not tbp:
 272                         p._debug('mkdir H')
 273                         mkdir_okexist(p.path[tbp])
 274                 else:
 275                         cmdl = ['sh','-ec',
 276                                 'test -d "$1" || mkdir -p "$1"',
 277                                 'x', p.path[tbp]]
 278                         tf_what = urllib.quote(p.what).replace('/','%2F')
 279                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 280                         if rc: bomb('failed to create directory %s' %
 281                                 p.path[tbp])
 282
 283         p.file[tbp] = p.path[tbp]
 284         return p.path[tbp]
 285
 286  def read(p, tbp=False):
 287         p._debug('read %s...' % 'HT'[tbp])
 288         p._ensure_path(tbp)
 289
 290         if p.file[tbp] is None:
 291                 if p.file[not tbp] is None:
 292                         p._wrong("requesting read but nonexistent")
 293                 cud = ['copyup','copydown'][tbp]
 294                 src = p.file[not tbp] + p.dir
 295                 dst = p.path[tbp] + p.dir
 296                 testbed.command(cud, (src, dst))
 297                 p.file[tbp] = p.path[tbp]
 298
 299         return p.file[tbp] + p.dir
 300
 301  def invalidate(p, tbp=False):
 302         p.file[tbp] = None
 303         p._debug('invalidated %s' % 'HT'[tbp])
 304
 305  def _debug(p, m):
 306         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 307
 308  def _constructed(p):
 309         p._debug('constructed: '+str(p))
 310         p._check()
 311
 312  def _check(p):
 313         for tbp in [False,True]:
 314          for pf in [p.path, p.file]:
 315                 if pf[tbp] is None: continue
 316                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 317                 if p.dir and pf[tbp].endswith('/'):
 318                         pf[tbp] = pf[tbp].rstrip('/')
 319                         if not pf[tbp]: pf[tbp] = '/'
 320                 if not p.dir and pf[tbp].endswith('/'):
 321                         bomb("directory `%s' specified for "
 322                              "non-directory %s" % (pf[tbp], p.what))
 323
 324  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 325         AutoFile.__init__(p,what)
 326         sh_on = ''; sh_sibl = ''
 327         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 328         if sibling: sh_sibl=' (sibling)'
 329         parent._debug('using as base: %s: %s%s%s' %
 330                         (str(parent), leaf, sh_on, sh_sibl))
 331         if not sibling and not parent.dir:
 332                 parent._wrong('asked for non-sibling relative path of non-dir')
 333         if sibling: trim = os.path.dirname
 334         else: trim = lambda x: x
 335         for tbp in [False,True]:
 336                 if parent.path[tbp] is None: continue
 337                 trimmed = trim(parent.path[tbp])
 338                 if trimmed: trimmed += '/'
 339                 p.path[tbp] = trimmed + leaf
 340                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 341                         p.file[tbp] = p.path[tbp]
 342
 343 class InputFile(AutoFile):
 344  def _init(p, what, spec, spec_tbp=False):
 345         AutoFile.__init__(p, what)
 346         p.spec = spec
 347         p.spec_tbp = spec_tbp
 348         p.path[spec_tbp] = spec
 349         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 350  def __init__(p, what, spec, spec_tbp=False):
 351         p._init(what,spec,spec_tbp)
 352         p._constructed()
 353
 354 class InputDir(InputFile):
 355  def __init__(p, what, spec, spec_tbp=False):
 356         InputFile._init(p,what,spec,spec_tbp)
 357         p.dir = '/'
 358         p._constructed()
 359
 360 class OutputFile(AutoFile):
 361  def _init(p, what, spec, spec_tbp=False):
 362         AutoFile.__init__(p, what)
 363         p.spec = spec
 364         p.spec_tbp = spec_tbp
 365         p.path[spec_tbp] = spec
 366  def __init__(p, what, spec, spec_tbp=False):
 367         p._init(what,spec,spec_tbp)
 368         p._constructed()
 369
 370 class OutputDir(OutputFile):
 371  def __init__(p, what, spec, spec_tbp=False):
 372         OutputFile._init(p,what,spec,spec_tbp)
 373         p.dir = '/'
 374         p._constructed()
 375
 376 class RelativeInputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 378         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 379         p._constructed()
 380
 381 class RelativeOutputFile(AutoFile):
 382  def __init__(p, what, parent, leaf, sibling=False):
 383         p._relative_init(what, parent, leaf, None, False, sibling)
 384         p._constructed()
 385
 386 class TemporaryFile(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p._constructed()
 390
 391 class TemporaryDir(AutoFile):
 392  def __init__(p, what):
 393         AutoFile.__init__(p, what)
 394         p.dir = '/'
 395         p._constructed()
 396
 397 #---------- parsing and representation of the arguments
 398
 399 class Action:
 400  def __init__(a, kind, af, arghandling, what):
 401         # extra attributes get added during processing
 402         a.kind = kind
 403         a.af = af
 404         a.ah = arghandling
 405         a.what = what
 406         a.missing_tests_control = False
 407  def __repr__(a):
 408         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 409
 410 def parse_args():
 411         global opts
 412         global n_non_actions # argh, stupid python scoping rules
 413         usage = "%prog <options> --- <virt-server>..."
 414         parser = OptionParser(usage=usage)
 415         pa = parser.add_option
 416         pe = parser.add_option
 417
 418         arghandling = {
 419                 'dsc_tests': True,
 420                 'dsc_filter': '*',
 421                 'deb_forbuilds': 'auto',
 422                 'deb_fortests': 'auto',
 423                 'tb': False,
 424                 'override_control': None,
 425                 'set_lang': 'C'
 426         }
 427         initial_arghandling = arghandling.copy()
 428         n_non_actions = 0
 429
 430         #----------
 431         # actions (ie, test sets to run, sources to build, binaries to use):
 432
 433         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 434                 global n_non_actions
 435                 parser.largs.append((value,kindpath))
 436                 n_non_actions += not(is_act)
 437
 438         def pa_action(long, metavar, kindpath, help, is_act=True):
 439                 pa('','--'+long, action='callback', callback=cb_action,
 440                         nargs=1, type='string',
 441                         callback_args=(long,kindpath,is_act), help=help)
 442
 443         pa_action('built-tree',         'TREE', '@/',
 444                 help='run tests from build tree TREE')
 445
 446         pa_action('unbuilt-tree',       'TREE', '@//',
 447                 help='run tests from build tree TREE')
 448
 449         pa_action('source',             'DSC', '@.dsc',
 450                 help='build DSC and use its tests and/or'
 451                     ' generated binary packages')
 452
 453         pa_action('binary',             'DEB', '@.deb',
 454                help='use binary package DEB according'
 455                     ' to most recent --binaries-* settings')
 456
 457         def cb_actnoarg(op,optstr,value,parser, largsentry):
 458                 parser.largs.append(largsentry)
 459         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 460                 callback_args=((None, ('instantiate',)),),
 461                 help='instantiate testbed now (during testing phase)'
 462                      ' and install packages'
 463                      ' selected for automatic installation, even'
 464                      ' if this might apparently not be required otherwise')
 465
 466         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 467                help='run tests from control file CONTROL instead,'
 468                     ' (applies to next test suite only)')
 469
 470         #----------
 471         # argument handling settings (what ways to use action
 472         #  arguments, and pathname processing):
 473
 474         def cb_setah(option, opt_str, value, parser, toset,setval):
 475                 if type(setval) == list:
 476                         if not value in setval:
 477                                 parser.error('value for %s option (%s) is not '
 478                                  'one of the permitted values (%s)' %
 479                                  (value, opt_str, setval.join(' ')))
 480                 elif setval is not None:
 481                         value = setval
 482                 for v in toset:
 483                         arghandling[v] = value
 484                 parser.largs.append(arghandling.copy())
 485
 486         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 487                 type = metavar
 488                 if type is not None: type = 'string'
 489                 pa('',long, action='callback', callback=cb_setah,
 490                    callback_args=(affected,effect), **kwargs)
 491
 492         #---- paths: host or testbed:
 493         #
 494         pa_setah('--paths-testbed', ['tb'],True,
 495                 help='subsequent path specifications refer to the testbed')
 496         pa_setah('--paths-host', ['tb'],False,
 497                 help='subsequent path specifications refer to the host')
 498
 499         #---- source processing settings:
 500
 501         pa_setah('--sources-tests', ['dsc_tests'],True,
 502                 help='run tests from builds of subsequent sources')
 503         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 504                 help='do not run tests from builds of subsequent sources')
 505
 506         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 507                 type='string', metavar='PATTERN-LIST',
 508                 help='from subsequent sources, use binaries matching'
 509                      ' PATTERN-LIST (comma-separated glob patterns)'
 510                      ' according to most recent --binaries-* settings')
 511         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 512                 help='from subsequent sources, do not use any binaries')
 513         #---- binary package processing settings:
 514
 515         def pa_setahbins(long,toset,how):
 516          pa_setah(long, toset,['ignore','auto','install'],
 517                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 518                 help=how+' ignore binaries, install them as needed'
 519                         ' for dependencies, or unconditionally install'
 520                         ' them, respectively')
 521         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 522         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 523         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 524
 525         #----------
 526         # general options:
 527
 528         def cb_vserv(op,optstr,value,parser):
 529                 parser.values.vserver = list(parser.rargs)
 530                 del parser.rargs[:]
 531
 532         def cb_path(op,optstr,value,parser, constructor,long,dir):
 533                 name = long.replace('-','_')
 534                 af = constructor(long, value,arghandling['tb'])
 535                 setattr(parser.values, name, af)
 536
 537         def pa_path(long, constructor, help, dir=False):
 538                 pa('','--'+long, action='callback', callback=cb_path,
 539                         callback_args=(constructor,long,dir),
 540                         nargs=1, type='string',
 541                         help=help, metavar='PATH')
 542
 543         pa_path('output-dir', OutputDir, dir=True,
 544                 help='write stderr/out files in PATH')
 545
 546         pa('--leave-lang', dest='set_lang', action='store_false',
 547                 help="leave LANG on testbed set to testbed's default")
 548         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 549                 help='set LANG on testbed to LANGVAL', default='C')
 550
 551         pa('','--tmp-dir',              type='string', dest='tmpdir',
 552                 help='write temporary files to TMPDIR, emptying it'
 553                      ' beforehand and leaving it behind at the end')
 554         pa('','--log-file',             type='string', dest='logfile',
 555                 help='write the log LOGFILE, emptying it beforehand,'
 556                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 557         pa('','--summary-file',         type='string', dest='summary',
 558                 help='write a summary report to SUMMARY,'
 559                      ' emptying it beforehand')
 560
 561         pa('','--user',                 type='string', dest='user',
 562                 help='run tests as USER (needs root on testbed)')
 563         pa('','--gain-root',            type='string', dest='gainroot',
 564                 help='prefix debian/rules binary with GAINROOT')
 565         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 566         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 567         pa('','--gnupg-home',           type='string', dest='gnupghome',
 568                 default='~/.autopkgtest/gpg',
 569                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 570                         " signing private apt archive);"
 571                         " `fresh' means generate new key each time.")
 572
 573         #----------
 574         # actual meat:
 575
 576         class SpecialOption(optparse.Option): pass
 577         vs_op = SpecialOption('','--VSERVER-DUMMY')
 578         vs_op.action = 'callback'
 579         vs_op.type = None
 580         vs_op.default = None
 581         vs_op.nargs = 0
 582         vs_op.callback = cb_vserv
 583         vs_op.callback_args = ( )
 584         vs_op.callback_kwargs = { }
 585         vs_op.help = 'introduces virtualisation server and args'
 586         vs_op._short_opts = []
 587         vs_op._long_opts = ['---']
 588
 589         pa(vs_op)
 590
 591         (opts,args) = parser.parse_args()
 592         if not hasattr(opts,'vserver'):
 593                 parser.error('you must specifiy --- <virt-server>...')
 594         if n_non_actions >= len(parser.largs):
 595                 parser.error('nothing to do specified')
 596
 597         arghandling = initial_arghandling
 598         opts.actions = []
 599         ix = 0
 600         for act in args:
 601                 if type(act) == dict:
 602                         arghandling = act
 603                         continue
 604                 elif type(act) == tuple:
 605                         pass
 606                 elif type(act) == str:
 607                         act = (act,act)
 608                 else:
 609                         raise ("unknown action in list `%s' having"
 610                               " type `%s'" % (act, type(act)))
 611                 (pathstr, kindpath) = act
 612
 613                 constructor = InputFile
 614                 if type(kindpath) is tuple:             kind = kindpath[0]
 615                 elif kindpath.endswith('.deb'):         kind = 'deb'
 616                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 617                 elif kindpath.endswith('//'):
 618                         kind = 'ubtree'
 619                         constructor = InputDir
 620                 elif kindpath.endswith('/'):
 621                         kind = 'tree'
 622                         constructor = InputDir
 623                 else: parser.error("do not know how to handle filename `%s';"
 624                         " specify --source --binary or --build-tree" %
 625                         kindpath)
 626
 627                 what = '%s%s' % (kind,ix); ix += 1
 628
 629                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 630                 else: fwhatx = '-'+kind
 631
 632                 if pathstr is None: af = None
 633                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 634                 opts.actions.append(Action(kind, af, arghandling, what))
 635
 636 def setup_trace():
 637         global trace_stream, tmpdir, summary_stream
 638
 639         if opts.tmpdir is not None:
 640                 rmtree('tmpdir(specified)',opts.tmpdir)
 641                 mkdir_okexist(opts.tmpdir, 0700)
 642                 tmpdir = opts.tmpdir
 643         else:
 644                 assert(tmpdir is None)
 645                 tmpdir = tempfile.mkdtemp()
 646
 647         if opts.logfile is None:
 648                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 649                         opts.logfile = opts.output_dir.spec + '/log'
 650                 elif opts.tmpdir is not None:
 651                         opts.logfile = opts.tmpdir + '/log'
 652         if opts.logfile is not None:
 653                 trace_stream = open(opts.logfile, 'w', 0)
 654         if opts.summary is not None:
 655                 summary_stream = open(opts.summary, 'w', 0)
 656
 657         debug('options: '+`opts`, 1)
 658
 659 def finalise_options():
 660         global opts, tb, build_essential
 661
 662         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 663                 opts.user = ''
 664
 665         if opts.user is None:
 666                 su = 'suggested-normal-user='
 667                 ul = [
 668                         e[len(su):]
 669                         for e in testbed.caps
 670                         if e.startswith(su)
 671                         ]
 672                 if ul:
 673                         opts.user = ul[0]
 674                 else:
 675                         opts.user = ''
 676
 677         if opts.user:
 678                 if 'root-on-testbed' not in testbed.caps:
 679                         pstderr("warning: virtualisation"
 680                                 " system does not offer root on testbed,"
 681                                 " but --user option specified: failure likely")
 682                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 683         else:
 684                 opts.user_wrap = lambda x: x
 685
 686         if opts.gainroot is None:
 687                 opts.gainroot = ''
 688                 if (opts.user or
 689                     'root-on-testbed' not in testbed.caps):
 690                         opts.gainroot = 'fakeroot'
 691                         build_essential += ['fakeroot']
 692
 693         if opts.gnupghome.startswith('~/'):
 694                 try: home = os.environ['HOME']
 695                 except KeyError:
 696                         parser.error("HOME environment variable"
 697                                 " not set, needed for --gnupghome=`%s"
 698                                 % opts.gnupghome)
 699                 opts.gnupghome = home + opts.gnupghome[1:]
 700         elif opts.gnupghome == 'fresh':
 701                 opts.gnupghome = None
 702
 703 #---------- testbed management - the Testbed class
 704
 705 class Testbed:
 706  def __init__(tb):
 707         tb.sp = None
 708         tb.lastsend = None
 709         tb.scratch = None
 710         tb.modified = False
 711         tb.blamed = []
 712         tb._ephemeral = []
 713         tb._debug('init')
 714         tb._need_reset_apt = False
 715  def _debug(tb, m, minlevel=0):
 716         debug('** '+m, minlevel)
 717  def start(tb):
 718         tb._debug('start')
 719         p = subprocess.PIPE
 720         debug_subprocess('vserver', opts.vserver)
 721         tb._errplumb = Errplumb(True)
 722         tb.sp = subprocess.Popen(opts.vserver,
 723                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 724         tb.expect('ok')
 725         tb.caps = tb.commandr('capabilities')
 726  def stop(tb):
 727         tb._debug('stop')
 728         tb.close()
 729         if tb.sp is None: return
 730         ec = tb.sp.returncode
 731         if ec is None:
 732                 tb.sp.stdout.close()
 733                 tb.send('quit')
 734                 tb.sp.stdin.close()
 735                 ec = tb.sp.wait()
 736         if ec:
 737                 tb.bomb('testbed gave exit status %d after quit' % ec)
 738         tb._errplumb.wait()
 739  def open(tb):
 740         tb._debug('open, scratch=%s' % tb.scratch)
 741         if tb.scratch is not None: return
 742         pl = tb.commandr('open')
 743         tb.scratch = InputDir('tb-scratch', pl[0], True)
 744         tb.deps_processed = []
 745  def mungeing_apt(tb):
 746         if not 'revert' in tb.caps:
 747                 tb._need_reset_apt = True
 748  def reset_apt(tb):
 749         if not tb._need_reset_apt: return
 750         what = 'aptget-update-reset'
 751         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 752         rc = tb.execute(what, cmdl, kind='install')
 753         if rc:
 754                 pstderr("\n" "warning: failed to restore"
 755                         " testbed apt cache, exit code %d" % rc)
 756         tb._need_reset_apt = False
 757  def close(tb):
 758         tb._debug('close, scratch=%s' % tb.scratch)
 759         if tb.scratch is None: return
 760         tb.scratch = None
 761         if tb.sp is None: return
 762         tb.command('close')
 763  def prepare1(tb, deps_new):
 764         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 765                 (tb.modified, tb.deps_processed, deps_new), 1)
 766         if 'revert' in tb.caps and (tb.modified or
 767             [d for d in tb.deps_processed if d not in deps_new]):
 768                 for af in tb._ephemeral: af.read(False)
 769                 tb._debug('reset **')
 770                 tb.command('revert')
 771                 tb.blamed = []
 772                 for af in tb._ephemeral: af.invalidate(True)
 773         tb.modified = False
 774  def prepare2(tb, deps_new):
 775         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 776         binaries.publish()
 777         tb._install_deps(deps_new)
 778  def prepare(tb, deps_new):
 779         tb.prepare1(deps_new)
 780         tb.prepare2(deps_new)
 781  def register_ephemeral(tb, af):
 782         tb._ephemeral.append(af)
 783  def _install_deps(tb, deps_new):
 784         tb._debug(' installing dependencies '+`deps_new`, 1)
 785         tb.deps_processed = deps_new
 786         if not deps_new: return
 787         dstr = ', '.join(deps_new)
 788         script = binaries.apt_pkg_gdebi_script(
 789                 dstr, [[
 790                 'from GDebi.DebPackage import DebPackage',
 791                 'd = DebPackage(cache)',
 792                 'res = d.satisfyDependsStr(arg)',
 793                 ]])
 794         cmdl = ['python','-c',script]
 795         what = 'install-deps'
 796         rc = testbed.execute(what+'-debinstall', cmdl, script=script,
 797                                 kind='install')
 798         if rc: badpkg('dependency install failed, exit code %d' % rc)
 799  def needs_reset(tb):
 800         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 801         tb.modified = True
 802  def blame(tb, m):
 803         tb._debug('blame += %s' % m, 1)
 804         tb.blamed.append(m)
 805  def bomb(tb, m):
 806         tb._debug('bomb %s' % m)
 807         if tb.sp is not None:
 808                 tb.sp.stdout.close()
 809                 tb.sp.stdin.close()
 810                 ec = tb.sp.wait()
 811                 if ec: pstderr('adt-run: testbed failing,'
 812                                 ' exit status %d' % ec)
 813         tb.sp = None
 814         raise Quit(16, 'testbed failed: %s' % m)
 815  def send(tb, string):
 816         tb.sp.stdin
 817         try:
 818                 debug('>> '+string, 2)
 819                 print >>tb.sp.stdin, string
 820                 tb.sp.stdin.flush()
 821                 tb.lastsend = string
 822         except:
 823                 (type, value, dummy) = sys.exc_info()
 824                 tb.bomb('cannot send to testbed: %s' % traceback.
 825                         format_exception_only(type, value))
 826  def expect(tb, keyword, nresults=None):
 827         l = tb.sp.stdout.readline()
 828         if not l: tb.bomb('unexpected eof from the testbed')
 829         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 830         l = l.rstrip('\n')
 831         debug('<< '+l, 2)
 832         ll = l.split()
 833         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 834         if ll[0] != keyword:
 835                 if tb.lastsend is None:
 836                         tb.bomb("got banner `%s', expected `%s...'" %
 837                                 (l, keyword))
 838                 else:
 839                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 840                                 (tb.lastsend, l, keyword))
 841         ll = ll[1:]
 842         if nresults is not None and len(ll) != nresults:
 843                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 844                         " expected %d result parameters" %
 845                         (string, l, len(ll), nresults))
 846         return ll
 847  def commandr(tb, cmd, args=(), nresults=None):
 848         # pass args=[None,...] or =(None,...) to avoid more url quoting
 849         if type(cmd) is str: cmd = [cmd]
 850         if len(args) and args[0] is None: args = args[1:]
 851         else: args = map(urllib.quote, args)
 852         al = cmd + args
 853         tb.send(string.join(al))
 854         ll = tb.expect('ok', nresults)
 855         rl = map(urllib.unquote, ll)
 856         return rl
 857  def command(tb, cmd, args=()):
 858         tb.commandr(cmd, args, 0)
 859  def commandr1(tb, cmd, args=()):
 860         rl = tb.commandr(cmd, args, 1)
 861         return rl[0]
 862  def execute(tb, what, cmdl,
 863                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 864                 script=False, tmpdir=None, kind='short'):
 865         # Options for script:
 866         #   False - do not call debug_subprocess, no synch. reporting required
 867         #   None or string - call debug_subprocess with that value,
 868         #                       plumb stderr through synchronously if possible
 869         # Options for se:
 870         #   None - usual Errplumb (output is of kind 2c)
 871         #   string - path on testbed (output is of kind 2d)
 872
 873         timeout = timeouts[kind]
 874
 875         if script is not False: debug_subprocess(what, cmdl, script=script)
 876         if cwd is None: cwd = tb.scratch.write(True)
 877
 878         xdump = None
 879         if se is None:
 880                 ep = Errplumb()
 881                 se_catch = TemporaryFile(what+'-xerr')
 882                 se_use = se_catch.write(True)
 883                 if not opts.quiet: xdump = 'debug=2-2'
 884                 elif trace_stream is not None:
 885                         xdump = 'debug=2-%d' % trace_stream.fileno()
 886         else:
 887                 ep = None
 888                 se_catch = None
 889                 se_use = se
 890
 891         cmdl = [None,
 892                 ','.join(map(urllib.quote, cmdl)),
 893                 si, so, se_use, cwd]
 894
 895         if timeout is not None and timeout > 0:
 896                 cmdl.append('timeout=%d' % timeout)
 897
 898         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 899         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 900         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 901         if opts.set_lang is not False:
 902                 cmdl.append('env=LANG=%s' % opts.set_lang)
 903
 904         rc = tb.commandr1('execute', cmdl)
 905         try: rc = int(rc)
 906         except ValueError: bomb("execute for %s gave invalid response `%s'"
 907                                         % (what,rc))
 908
 909         if se_catch is not None:
 910                 debug_file(se_catch.read())
 911         if ep is not None:
 912                 ep.wait()
 913
 914         return rc
 915
 916 #---------- representation of test control files: Field*, Test, etc.
 917
 918 class FieldBase:
 919  def __init__(f, fname, stz, base, tnames, vl):
 920         assert(vl)
 921         f.stz = stz
 922         f.base = base
 923         f.tnames = tnames
 924         f.vl = vl
 925  def words(f):
 926         def distribute(vle):
 927                 (lno, v) = vle
 928                 r = v.split()
 929                 r = map((lambda w: (lno, w)), r)
 930                 return r
 931         return flatten(map(distribute, f.vl))
 932  def atmostone(f):
 933         if len(vl) == 1:
 934                 (f.lno, f.v) = vl[0]
 935         else:
 936                 raise Unsupported(f.vl[1][0],
 937                         'only one %s field allowed' % fn)
 938         return f.v
 939
 940 class FieldIgnore(FieldBase):
 941  def parse(f): pass
 942
 943 class Restriction:
 944  def __init__(r,rname,base): pass
 945
 946 class Restriction_rw_build_tree(Restriction): pass
 947 class Restriction_breaks_testbed(Restriction):
 948  def __init__(r, rname, base):
 949         if 'revert' not in testbed.caps:
 950                 raise Unsupported(f.lno,
 951                         'Test breaks testbed but testbed cannot revert')
 952 class Restriction_needs_root(Restriction):
 953  def __init__(r, rname, base):
 954         if 'root-on-testbed' not in testbed.caps:
 955                 raise Unsupported(f.lno,
 956                         'Test needs root on testbed which is not available')
 957
 958 class Field_Restrictions(FieldBase):
 959  def parse(f):
 960         for wle in f.words():
 961                 (lno, rname) = wle
 962                 nrname = rname.replace('-','_')
 963                 try: rclass = globals()['Restriction_'+nrname]
 964                 except KeyError: raise Unsupported(lno,
 965                         'unknown restriction %s' % rname)
 966                 r = rclass(nrname, f.base)
 967                 f.base['restriction_names'].append(rname)
 968                 f.base['restrictions'].append(r)
 969
 970 class Field_Features(FieldIgnore):
 971  def parse(f):
 972         for wle in f.words():
 973                 (lno, fname) = wle
 974                 f.base['feature_names'].append(fname)
 975                 nfname = fname.replace('-','_')
 976                 try: fclass = globals()['Feature_'+nfname]
 977                 except KeyError: continue
 978                 ft = fclass(nfname, f.base)
 979                 f.base['features'].append(ft)
 980
 981 class Field_Tests(FieldIgnore): pass
 982
 983 class Field_Depends(FieldBase):
 984  def parse(f):
 985         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 986         dl = map(lambda x: x.strip(),
 987                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 988         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
 989         for d in dl:
 990                 if re.search(d):
 991                         badpkg("Test Depends field contains dependency"
 992                                " `%s' with invalid characters" % d)
 993         f.base['depends'] = dl
 994
 995 class Field_Tests_directory(FieldBase):
 996  def parse(f):
 997         td = atmostone(f)
 998         if td.startswith('/'): raise Unspported(f.lno,
 999                 'Tests-Directory may not be absolute')
1000         f.base['testsdir'] = td
1001
1002 def run_tests(stanzas, tree):
1003         global errorcode, testbed
1004         if stanzas == ():
1005                 report('*', 'SKIP no tests in this package')
1006                 errorcode |= 8
1007         for stanza in stanzas:
1008                 tests = stanza[' tests']
1009                 if not tests:
1010                         report('*', 'SKIP package has metadata but no tests')
1011                         errorcode |= 8
1012                 for t in tests:
1013                         t.prepare()
1014                         t.run(tree)
1015                         if 'breaks-testbed' in t.restriction_names:
1016                                 testbed.needs_reset()
1017                 testbed.needs_reset()
1018
1019 class Test:
1020  def __init__(t, tname, base, act):
1021         if '/' in tname: raise Unsupported(base[' lno'],
1022                 'test name may not contain / character')
1023         for k in base: setattr(t,k,base[k])
1024         t.tname = tname
1025         t.act = act
1026         t.what = act.what+'t-'+tname
1027         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1028         else: t.path = tname
1029         t._debug('constructed; path=%s' % t.path)
1030         t._debug(' .depends=%s' % t.depends)
1031  def _debug(t, m):
1032         debug('& %s: %s' % (t.what, m))
1033  def report(t, m):
1034         report(t.what, m)
1035  def reportfail(t, m):
1036         global errorcode
1037         errorcode |= 4
1038         report(t.what, 'FAIL ' + m)
1039  def prepare(t):
1040         t._debug('preparing')
1041         dn = []
1042         for d in t.depends:
1043                 t._debug(' processing dependency '+d)
1044                 if not '@' in d:
1045                         t._debug('  literal dependency '+d)
1046                         dn.append(d)
1047                 else:
1048                         for (pkg,bin) in t.act.binaries:
1049                                 d = d.replace('@',pkg)
1050                                 t._debug('  synthesised dependency '+d)
1051                                 dn.append(d)
1052         testbed.prepare(dn)
1053  def run(t, tree):
1054         t._debug('[----------------------------------------')
1055         def stdouterr(oe):
1056                 idstr = t.what + '-' + oe
1057                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1058                         use_dir = opts.output_dir
1059                 else:
1060                         use_dir = testbed.scratch
1061                 return RelativeOutputFile(idstr, use_dir, idstr)
1062
1063         if hasattr(t.act,'work'): t.act.work.write(True)
1064         tree.read(True)
1065
1066         af = RelativeInputFile(t.what, tree, t.path)
1067         so = stdouterr('stdout')
1068         se = stdouterr('stderr')
1069
1070         tf = af.read(True)
1071         tmpdir = None
1072
1073         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1074         if rc: bomb('failed to chmod +x %s' % tf)
1075
1076         if 'needs-root' not in t.restriction_names and opts.user is not None:
1077                 tfl = ['su',opts.user,'-c',tf]
1078                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1079                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1080                 if opts.user: script += '; chown %s "$1"' % opts.user
1081                 if 'rw-build-tree' in t.restriction_names:
1082                         script += '; chown -R %s "$2"' % opts.user
1083                 rc = testbed.execute('mktmpdir-'+t.what,
1084                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1085                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1086                                 % (tmpdir, rc))
1087         else:
1088                 tfl = [tf]
1089
1090         rc = testbed.execute('test-'+t.what, tfl,
1091                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1092                 tmpdir=tmpdir, kind='test')
1093
1094         so_read = so.read()
1095         se_read = se.read()
1096
1097         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1098         stab = os.stat(se_read)
1099         if stab.st_size != 0:
1100                 l = open(se_read).readline()
1101                 l = l.rstrip('\n \t\r')
1102                 if len(l) > 35: l = l[:35] + '...'
1103                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1104                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1105                 debug_file(se_read)
1106         elif rc != 0:
1107                 t.reportfail('non-zero exit status %d' % rc)
1108         else:
1109                 t.report('PASS')
1110
1111         stab = os.stat(so_read)
1112         if stab.st_size != 0:
1113                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1114                 debug_file(so_read)
1115
1116         t._debug('----------------------------------------]')
1117
1118 def read_control(act, tree, control_override):
1119         stanzas = [ ]
1120
1121         if control_override is not None:
1122                 control_af = control_override
1123                 testbed.blame('arg:'+control_override.spec)
1124         else:
1125                 if act.missing_tests_control:
1126                         return ()
1127                 control_af = RelativeInputFile(act.what+'-testcontrol',
1128                         tree, 'debian/tests/control')
1129         try:
1130                 control = open(control_af.read(), 'r')
1131         except (IOError,OSError), oe:
1132                 if oe[0] != errno.ENOENT: raise
1133                 return []
1134
1135         lno = 0
1136         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1137         stz = None      # stz[field_name][index] = (lno, value)
1138                         # special field names:
1139                         # stz[' lno'] = number
1140                         # stz[' tests'] = list of Test objects
1141         def end_stanza(stz):
1142                 if stz is None: return
1143                 stz[' errs'] = 0
1144                 stanzas.append(stz)
1145                 stz = None
1146                 hcurrent = None
1147
1148         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1149         while 1:
1150                 l = control.readline()
1151                 if not l: break
1152                 lno += 1
1153                 if not l.endswith('\n'): badctrl('unterminated line')
1154                 if regexp.compile('\s*\#').match(l): continue
1155                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1156                 initmat = initre.match(l)
1157                 if initmat:
1158                         (fname, l) = initmat.groups()
1159                         fname = string.capwords(fname)
1160                         if stz is None:
1161                                 stz = { ' lno': lno, ' tests': [] }
1162                         if not stz.has_key(fname): stz[fname] = [ ]
1163                         hcurrent = stz[fname]
1164                 elif regexp.compile('\s').match(l):
1165                         if not hcurrent: badctrl('unexpected continuation')
1166                 else:
1167                         badctrl('syntax error')
1168                 hcurrent.append((lno, l))
1169         end_stanza(stz)
1170
1171         def testbadctrl(stz, lno, m):
1172                 report_badctrl(lno, m)
1173                 stz[' errs'] += 1
1174
1175         for stz in stanzas:
1176                 try:
1177                         try: tnames = stz['Tests']
1178                         except KeyError:
1179                                 tnames = ['*']
1180                                 raise Unsupported(stz[' lno'],
1181                                         'no Tests field')
1182                         tnames = map((lambda lt: lt[1]), tnames)
1183                         tnames = string.join(tnames).split()
1184                         base = {
1185                                 'restriction_names': [],
1186                                 'restrictions': [],
1187                                 'feature_names': [],
1188                                 'features': [],
1189                                 'testsdir': 'debian/tests',
1190                                 'depends' : '@'
1191                         }
1192                         for fname in stz.keys():
1193                                 if fname.startswith(' '): continue
1194                                 vl = stz[fname]
1195                                 try: fclass = globals()['Field_'+
1196                                         fname.replace('-','_')]
1197                                 except KeyError: raise Unsupported(vl[0][0],
1198                                         'unknown metadata field %s' % fname)
1199                                 f = fclass(stz, fname, base, tnames, vl)
1200                                 f.parse()
1201                         for tname in tnames:
1202                                 t = Test(tname, base, act)
1203                                 stz[' tests'].append(t)
1204                 except Unsupported, u:
1205                         for tname in tnames: u.report(tname)
1206                         continue
1207
1208         return stanzas
1209
1210 def print_exception(ei, msgprefix=''):
1211         if msgprefix: pstderr(msgprefix)
1212         (et, q, tb) = ei
1213         if et is Quit:
1214                 pstderr('adt-run: ' + q.m)
1215                 psummary('quitting: '+q.m)
1216                 return q.ec
1217         else:
1218                 pstderr("adt-run: unexpected, exceptional, error:")
1219                 psummary('quitting: unexpected error, consult transcript')
1220                 traceback.print_exc(None, sys.stderr)
1221                 if trace_stream is not None:
1222                         traceback.print_exc(None, trace_stream)
1223                 return 20
1224
1225 def cleanup():
1226         global trace_stream
1227         try:
1228                 if testbed is not None:
1229                         testbed.reset_apt()
1230                         testbed.stop()
1231                 if opts.tmpdir is None and tmpdir is not None:
1232                         rmtree('tmpdir', tmpdir)
1233                 if trace_stream is not None:
1234                         trace_stream.close()
1235                         trace_stream = None
1236         except:
1237                 print_exception(sys.exc_info(),
1238                         '\nadt-run: error cleaning up:\n')
1239                 os._exit(20)
1240
1241 #---------- registration, installation etc. of .deb's: Binaries
1242
1243 def determine_package(act):
1244         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1245         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1246         if rc: badpkg('failed to parse binary package, code %d' % rc)
1247         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1248         act.pkg = None
1249         for l in output.split('\n'):
1250                 m = re.match(l)
1251                 if not m: continue
1252                 if act.pkg: badpkg('two Package: lines in control file')
1253                 act.pkg = m.groups()[0]
1254         if not act.pkg: badpkg('no good Package: line in control file')
1255
1256 class Binaries:
1257  def __init__(b):
1258         b.dir = TemporaryDir('binaries')
1259         b.dir.write()
1260         ok = False
1261
1262         if opts.gnupghome is None:
1263                 opts.gnupghome = tmpdir+'/gnupg'
1264
1265         b._debug('initialising')
1266         try:
1267                 for x in ['pubring','secring']:
1268                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1269                 ok = True
1270         except (IOError,OSError), oe:
1271                 if oe.errno != errno.ENOENT: raise
1272
1273         if ok: b._debug('no key generation needed')
1274         else: b.genkey()
1275
1276  def _debug(b, s):
1277         debug('* '+s)
1278
1279  def genkey(b):
1280         b._debug('preparing for key generation')
1281
1282         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1283         mkdir_okexist(opts.gnupghome, 0700)
1284
1285         script = '''
1286   exec >&2
1287   cd "$1"
1288   cat <<"END" >key-gen-params
1289 Key-Type: DSA
1290 Key-Length: 1024
1291 Key-Usage: sign
1292 Name-Real: autopkgtest per-run key
1293 Name-Comment: do not trust this key
1294 Name-Email: autopkgtest@example.com
1295 END
1296   set -x
1297   gpg --homedir="$1" --batch --gen-key key-gen-params
1298                 '''
1299         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1300         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1301         if rc: bomb('key generation failed, code %d' % rc)
1302
1303  def apt_configs(b):
1304         return {
1305                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1306         }
1307
1308  def apt_pkg_gdebi_script(b, arg, middle):
1309         script = [
1310                 'import apt_pkg',
1311                 'import urllib',
1312                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1313                 ]
1314         for (k,v) in b.apt_configs().iteritems():
1315                 v = urllib.quote(v)
1316                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1317                                 % (k, v))
1318         script += [
1319                 'from GDebi.Cache import Cache',
1320                 'cache = Cache()',
1321                 ]
1322         for m in middle:
1323                 script += m + [
1324                 'print res',
1325                 'print d.missingDeps',
1326                 'print d.requiredChanges',
1327                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1328                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1329                           'd._failureString)',
1330                 'cache.commit()',
1331                 ''
1332                 ]
1333         return '\n'.join(script)
1334  def apt_get(b):
1335         ag = ['apt-get','-qy']
1336         for kv in b.apt_configs().iteritems():
1337                 ag += ['-o', '%s=%s' % kv]
1338         return ' '.join(ag)
1339
1340  def reset(b):
1341         b._debug('reset')
1342         rmtree('binaries', b.dir.read())
1343         b.dir.invalidate()
1344         b.dir.write()
1345         b.install = []
1346         b.blamed = []
1347         b.registered = set()
1348
1349  def register(b, act, pkg, af, forwhat, blamed):
1350         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1351                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1352
1353         if act.ah['deb_'+forwhat] == 'ignore': return
1354
1355         b.blamed += testbed.blamed
1356
1357         leafname = pkg+'.deb'
1358         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1359
1360         try: os.remove(dest.write())
1361         except (IOError,OSError), oe:
1362                 if oe.errno != errno.ENOENT: raise e
1363
1364         try: os.link(af.read(), dest.write())
1365         except (IOError,OSError), oe:
1366                 if oe.errno != errno.EXDEV: raise e
1367                 shutil.copy(af.read(), dest)
1368
1369         if act.ah['deb_'+forwhat] == 'install':
1370                 b.install.append(pkg)
1371
1372         b.registered.add(pkg)
1373
1374  def publish(b):
1375         b._debug('publish')
1376
1377         script = '''
1378   exec >&2
1379   cd "$1"
1380   apt-ftparchive packages . >Packages
1381   gzip <Packages >Packages.gz
1382   apt-ftparchive release . >Release
1383   rm -f Release.gpg
1384   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1385   gpg --homedir="$2" --batch --export >archive-key.pgp
1386                 '''
1387         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1388         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1389         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1390
1391         b.dir.invalidate(True)
1392         apt_source = b.dir.read(True)
1393
1394         so = TemporaryFile('vlds')
1395         script = '''
1396   exec 3>&1 >&2
1397   apt-key add archive-key.pgp
1398   echo "deb file://'''+apt_source+''' /" >sources.list
1399   cat /etc/apt/sources.list >>sources.list
1400   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1401     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1402   fi
1403   '''+ b.apt_get() +''' update >&2
1404   cat /var/lib/dpkg/status >&3
1405                 '''
1406         testbed.mungeing_apt()
1407         rc = testbed.execute('apt-key', ['sh','-ec',script],
1408                                 so=so.write(True), cwd=b.dir.write(True),
1409                                 script=script, kind='install')
1410         if rc: bomb('apt setup failed with exit code %d' % rc)
1411
1412         testbed.blamed += b.blamed
1413
1414         b._debug('publish reinstall checking...')
1415         pkgs_reinstall = set()
1416         pkg = None
1417         for l in open(so.read()):
1418                 if l.startswith('Package: '):
1419                         pkg = l[9:].rstrip()
1420                 elif l.startswith('Status: install '):
1421                         if pkg in b.registered:
1422                                 pkgs_reinstall.add(pkg)
1423                                 b._debug(' publish reinstall needs '+pkg)
1424
1425         if pkgs_reinstall:
1426                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1427                 what = 'apt-get-reinstall'
1428                 cmdl = (b.apt_get() + ' --reinstall install '+
1429                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1430                 cmdl = ['sh','-c',cmdl]
1431                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1432                 if rc: badpkg("installation of basic binarries failed,"
1433                                 " exit code %d" % rc)
1434
1435         b._debug('publish install...')
1436         for pkg in b.install:
1437                 what = 'apt-get-install-%s' % pkg
1438                 testbed.blame(pkg)
1439                 cmdl = b.apt_get() + ' install ' + pkg + ' >&2'
1440                 cmdl = ['sh','-c',cmdl]
1441                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1442                 if rc: badpkg("installation of %s failed, exit code %d"
1443                                 % (pkg, rc))
1444
1445         b._debug('publish done')
1446
1447 #---------- processing of sources (building)
1448
1449 def source_rules_command(act,script,what,which,work,cwd,
1450                                 results_lines=0,xargs=[]):
1451         script = [      "exec 3>&1 >&2",
1452                         "set -x"        ] + script
1453         script = '\n'.join(script)
1454         so = TemporaryFile('%s-%s-results' % (what,which))
1455         rc = testbed.execute('%s-%s' % (what,which),
1456                         ['sh','-ec',script]+xargs, script=script,
1457                         so=so.write(True), cwd=cwd, kind='build')
1458         results = open(so.read()).read().rstrip('\n')
1459         if len(results): results = results.split("\n")
1460         else: results = []
1461         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1462         if results_lines is not None and len(results) != results_lines:
1463                 badpkg("got %d lines of results from %s where %d expected"
1464                         % (len(results), which, results_lines))
1465         if results_lines==1: return results[0]
1466         return results
1467
1468 def build_source(act, control_override):
1469         act.blame = 'arg:'+act.af.spec
1470         testbed.blame(act.blame)
1471         testbed.prepare1([])
1472         testbed.needs_reset()
1473
1474         what = act.what
1475         basename = act.af.spec
1476         debiancontrol = None
1477         act.binaries = []
1478
1479         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1480
1481         if act.kind == 'dsc':
1482                 dsc = act.af
1483                 dsc_file = open(dsc.read())
1484                 in_files = False
1485                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1486                 for l in dsc_file:
1487                         l = l.rstrip('\n')
1488                         if l.startswith('Files:'): in_files = True; continue
1489                         elif l.startswith('#'): pass
1490                         elif not l.startswith(' '):
1491                                 in_files = False
1492                                 if l.startswith('Source:'):
1493                                         act.blame = 'dsc:'+l[7:].strip()
1494                                         testbed.blame(act.blame)
1495                         if not in_files: continue
1496
1497                         m = fre.match(l)
1498                         if not m: badpkg(".dsc contains unparseable line"
1499                                         " in Files: `%s'" % l)
1500                         leaf = m.groups(0)[0]
1501                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1502                                         sibling=True)
1503                         subfile.read(True)
1504                 dsc.read(True)
1505
1506         if act.kind == 'ubtree':
1507                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1508                         act.af, 'debian/control')
1509                 dsc = TemporaryFile(what+'-fakedsc')
1510                 dsc_w = open(dsc.write(), 'w')
1511                 for l in open(debiancontrol.read()):
1512                         l = l.rstrip('\n')
1513                         if not len(l): break
1514                         print >>dsc_w, l
1515                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1516                 dsc_w.close()
1517
1518         if act.kind == 'dsc':
1519                 testbed.prepare2([])
1520                 script = binaries.apt_pkg_gdebi_script('', [[
1521                                 'from GDebi.DebPackage import DebPackage',
1522                                 'd = DebPackage(cache)',
1523                                 'res = d.satisfyDependsStr("dpkg-dev")',
1524                         ]])
1525                 cmdl = ['python','-c',script]
1526                 whatp = what+'-dpkgsource'
1527                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1528                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1529
1530         work = TemporaryDir(what+'-build')
1531         act.work = work
1532
1533         tmpdir = work.write(True)+'/tmpdir'
1534         tmpdir_script = [
1535                         'TMPDIR="$1"',
1536                         'rm -rf -- "$TMPDIR"',
1537                         'export TMPDIR',
1538                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1539                 ]
1540
1541         if act.kind == 'ubtree':
1542                 spec = '%s/real-tree' % work.write(True)
1543                 create_command = '''
1544                         rm -rf "$spec"
1545                         mkdir "$spec"
1546                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1547                         '''
1548                 initcwd = act.af.read(True)
1549
1550         if act.kind == 'dsc':
1551                 spec = dsc.read(True)
1552                 create_command = '''
1553                         dpkg-source -x $spec
1554                         '''
1555                 initcwd = work.write(True)
1556
1557         script = [
1558                 'spec="$2"',
1559                 'origpwd=`pwd`',
1560                 'cd '+work.write(True)
1561         ]
1562
1563         if opts.user:
1564                 script += ([ 'chown '+opts.user+' .' ] +
1565                         tmpdir_script +
1566                         [ 'spec="$spec" origpwd="$origpwd" '
1567                                 +opts.user_wrap(create_command) ])
1568         else:
1569                 script += (tmpdir_script +
1570                         [ create_command ])
1571
1572         script += [
1573                         'cd [a-z0-9]*-*/.',
1574                         'pwd >&3',
1575                         'set +e; test -f debian/tests/control; echo $? >&3'
1576                 ]
1577         (result_pwd, control_test_rc) = source_rules_command(
1578                         act,script,what,'extract',work,
1579                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1580
1581         filter = act.ah['dsc_filter']
1582
1583         if control_test_rc == '1': act.missing_tests_control = True
1584
1585         # For optional builds:
1586         #
1587         # We might need to build the package because:
1588         #   - we want its binaries (filter isn't _ and at least one of the
1589         #       deb_... isn't ignore)
1590         #   - the test control file says so
1591         #       (assuming we have any tests)
1592
1593         class NeedBuildException: pass
1594         def build_needed(m):
1595                 debug_b('build needed for %s' % m)
1596                 raise NeedBuildException()
1597
1598         try:
1599                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1600                                       act.ah['deb_fortests'] != 'ignore'):
1601                         build_needed('binaries')
1602
1603                 result_pwd_af = InputDir(what+'-treeforcontrol',
1604                         result_pwd, True)
1605                 stanzas = read_control(act, result_pwd_af, control_override)
1606                 for stanza in stanzas:
1607                         for t in stanza[' tests']:
1608                                 if 'no-build-needed' not in t.feature_names:
1609                                         build_needed('test %s' % t.tname)
1610                                 for d in t.depends:
1611                                         if '@' in d:
1612                                                 build_needed('test %s '
1613                                                  'dependency %s' % (t.tname,d))
1614
1615                 debug_b('build not needed')
1616                 built = False
1617
1618         except NeedBuildException:
1619
1620                 if act.kind != 'dsc':
1621                         testbed.prepare2([])
1622
1623                 script = binaries.apt_pkg_gdebi_script(
1624                         dsc.read(True), [[
1625                         'from GDebi.DscSrcPackage import DscSrcPackage',
1626                         'd = DscSrcPackage(cache, arg)',
1627                         'res = d.checkDeb()',
1628                          ],[
1629                         'from GDebi.DebPackage import DebPackage',
1630                         'd = DebPackage(cache)',
1631                         'res = d.satisfyDependsStr("'+
1632                                         ','.join(build_essential)+
1633                                 '")',
1634                         ]])
1635
1636                 cmdl = ['python','-c',script]
1637                 whatp = what+'-builddeps'
1638                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1639                 if rc: badpkg('build-depends install failed,'
1640                               ' exit code %d' % rc)
1641
1642                 script = tmpdir_script + [
1643                         'cd "$2"',
1644                         'dpkg-checkbuilddeps',
1645                         opts.user_wrap('debian/rules build'),
1646                         ]
1647                 source_rules_command(act,script,what,'build',work,
1648                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1649
1650                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1651                         badpkg("results dir `%s' is not in expected parent"
1652                                " dir `%s'" % (result_pwd, work.read(True)))
1653
1654                 built = True
1655
1656         act.tests_tree = InputDir(what+'-tests-tree',
1657                                 work.read(True)+os.path.basename(result_pwd),
1658                                 True)
1659         if act.ah['dsc_tests']:
1660                 testbed.register_ephemeral(act.work)
1661                 testbed.register_ephemeral(act.tests_tree)
1662
1663         if not built:
1664                 act.blamed = []
1665                 return
1666
1667         act.blamed = copy.copy(testbed.blamed)
1668
1669         debug_b('filter=%s' % filter)
1670         if filter != '_':
1671                 script = tmpdir_script + [
1672                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1673                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1674                         'cd ..',
1675                         'echo *.deb >&3',
1676                         ]
1677                 result_debs = source_rules_command(act,script,what,
1678                                 'binary',work,work.write(True),
1679                                 results_lines=1, xargs=['x',tmpdir])
1680                 if result_debs == '*.deb': debs = []
1681                 else: debs = result_debs.split(' ')
1682                 debug_b('debs='+`debs`)
1683                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1684                 for deb in debs:
1685                         m = re.match(deb)
1686                         if not m: badpkg("badly-named binary `%s'" % deb)
1687                         pkg = m.groups()[0]
1688                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1689                         for pat in filter.split(','):
1690                                 debug_b('  pat=%s' % pat)
1691                                 if not fnmatch.fnmatchcase(pkg,pat):
1692                                         debug_b('   no match')
1693                                         continue
1694                                 deb_what = pkg+'_'+what+'.deb'
1695                                 bin = RelativeInputFile(deb_what,work,deb,True)
1696                                 debug_b('  deb_what=%s, bin=%s' %
1697                                         (deb_what, str(bin)))
1698                                 binaries.register(act,pkg,bin,
1699                                         'forbuilds',testbed.blamed)
1700                                 act.binaries.append((pkg,bin))
1701                                 break
1702                 debug_b('all done.')
1703
1704 #---------- main processing loop and main program
1705
1706 def process_actions():
1707         global binaries
1708
1709         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1710         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1711         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1712
1713         debug_a1('starting')
1714         testbed.open()
1715         binaries = Binaries()
1716
1717         for act in opts.actions:
1718                 if act.af is not None and not act.af.spec_tbp:
1719                         testbed.register_ephemeral(act.af)
1720
1721         binaries.reset()
1722         control_override = None
1723
1724         debug_a1('builds ...')
1725         for act in opts.actions:
1726                 debug_a2('%s %s' %
1727                         (act.kind, act.what))
1728
1729                 if act.kind == 'control':
1730                         control_override = act.af
1731                 if act.kind == 'deb':
1732                         testbed.blame('arg:'+act.af.spec)
1733                         determine_package(act)
1734                         testbed.blame('deb:'+act.pkg)
1735                         binaries.register(act,act.pkg,act.af,
1736                                 'forbuilds',testbed.blamed)
1737                 if act.kind == 'dsc' or act.kind == 'ubtree':
1738                         build_source(act, control_override)
1739                 if act.kind == 'tree':
1740                         act.binaries = []
1741                 if act.kind.endswith('tree') or act.kind == 'dsc':
1742                         control_override = None
1743                 if act.kind == 'instantiate':
1744                         pass
1745
1746         debug_a1('builds done.')
1747
1748         binaries.reset()
1749         control_override = None
1750
1751         debug_a1('tests ...')
1752         for act in opts.actions:
1753                 debug_a2('test %s %s' % (act.kind, act.what))
1754
1755                 testbed.needs_reset()
1756                 if act.kind == 'control':
1757                         control_override = act.af
1758                 if act.kind == 'deb':
1759                         binaries.register(act,act.pkg,act.af,'fortests',
1760                                 ['deb:'+act.pkg])
1761                 if act.kind == 'dsc' or act.kind == 'ubtree':
1762                         for (pkg,bin) in act.binaries:
1763                                 binaries.register(act,pkg,bin,'fortests',
1764                                         act.blamed)
1765                 if act.kind == 'dsc':
1766                         if act.ah['dsc_tests']:
1767                                 debug_a3('read control ...')
1768                                 stanzas = read_control(act, act.tests_tree,
1769                                                 control_override)
1770                                 testbed.blamed += act.blamed
1771                                 debug_a3('run_tests ...')
1772                                 run_tests(stanzas, act.tests_tree)
1773                         control_override = None
1774                 if act.kind == 'tree' or act.kind == 'ubtree':
1775                         testbed.blame('arg:'+act.af.spec)
1776                         stanzas = read_control(act, act.af, control_override)
1777                         debug_a3('run_tests ...')
1778                         run_tests(stanzas, act.af)
1779                         control_override = None
1780                 if act.kind == 'instantiate':
1781                         testbed.prepare([])
1782         debug_a1('tests done.')
1783
1784 def main():
1785         global testbed
1786         global tmpdir
1787         try:
1788                 parse_args()
1789         except SystemExit, se:
1790                 os._exit(20)
1791         try:
1792                 setup_trace()
1793                 testbed = Testbed()
1794                 testbed.start()
1795                 finalise_options()
1796                 process_actions()
1797         except:
1798                 ec = print_exception(sys.exc_info(), '')
1799                 cleanup()
1800                 os._exit(ec)
1801         cleanup()
1802         os._exit(errorcode)
1803
1804 main()