runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         try: shutil.rmtree(pathname)
 192         except (IOError,OSError), oe:
 193                 if oe.errno != errno.EEXIST: raise
 194
 195 def debug_subprocess(what, cmdl=None, script=None):
 196         o = '$ '+what+':'
 197         if cmdl is not None:
 198                 ol = []
 199                 for x in cmdl:
 200                         if x is script: x = '<SCRIPT>'
 201                         ol.append(x.    replace('\\','\\\\').
 202                                         replace(' ','\\ ')      )
 203                 o += ' '+ ' '.join(ol)
 204         debug(o)
 205         if script is not None and opts.debuglevel >= 1:
 206                 o = ''
 207                 for l in script.rstrip('\n').split('\n'):
 208                         o += '$     '+l+'\n'
 209                 debug(o,1)
 210
 211 def flatten(l):
 212         return reduce((lambda a,b: a + b), l, [])
 213
 214 #---------- fancy automatic file-copying class
 215
 216 class AutoFile:
 217         # p.what
 218         # p.path[tb]    None or path    not None => path known
 219         # p.file[tb]    None or path    not None => file exists
 220         # p.spec        string or None
 221         # p.spec_tbp    True or False, or not set if spec is None
 222         # p.dir         '' or '/'
 223
 224  def __init__(p, what):
 225         p.what = what
 226         p.path = [None,None]
 227         p.file = [None,None]
 228         p.spec = None
 229         p.dir = ''
 230
 231  def __repr__(p):
 232         return "<AF@%s>" % p.__str__()
 233  def __str__(p):
 234         def ptbp(tbp):
 235                 if p.path[tbp] is None: return '-'+p.dir
 236                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 237                 else: return p.path[tbp]+p.dir+'!'
 238         out = p.what
 239         if p.spec is not None:
 240                 if not hasattr(p,'spec_tb'): out += '~'
 241                 elif p.spec_tbp: out += '#'
 242                 else: out += '='
 243                 out += p.spec
 244         out += ':'
 245         out += ptbp(False)
 246         out += '|'
 247         out += ptbp(True)
 248         return out
 249
 250  def _wrong(p, how):
 251         xtra = ''
 252         if p.spec is not None:
 253                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 254         raise ("internal error: %s (%s)" % (how, str(p)))
 255
 256  def _ensure_path(p, tbp):
 257         if p.path[tbp] is None:
 258                 if '/' in p.what:
 259                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 260                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 261                 if not tbp:
 262                         p.path[tbp] = tmpdir+'/'+p.what
 263                 else:
 264                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 265
 266  def write(p, tbp=False):
 267         p._debug('write %s...' % 'HT'[tbp])
 268         p._ensure_path(tbp)
 269
 270         if p.dir and not p.file[tbp]:
 271                 if not tbp:
 272                         p._debug('mkdir H')
 273                         mkdir_okexist(p.path[tbp])
 274                 else:
 275                         cmdl = ['sh','-ec',
 276                                 'test -d "$1" || mkdir -p "$1"',
 277                                 'x', p.path[tbp]]
 278                         tf_what = urllib.quote(p.what).replace('/','%2F')
 279                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 280                         if rc: bomb('failed to create directory %s' %
 281                                 p.path[tbp])
 282
 283         p.file[tbp] = p.path[tbp]
 284         return p.path[tbp]
 285
 286  def read(p, tbp=False):
 287         p._debug('read %s...' % 'HT'[tbp])
 288         p._ensure_path(tbp)
 289
 290         if p.file[tbp] is None:
 291                 if p.file[not tbp] is None:
 292                         p._wrong("requesting read but nonexistent")
 293                 cud = ['copyup','copydown'][tbp]
 294                 src = p.file[not tbp] + p.dir
 295                 dst = p.path[tbp] + p.dir
 296                 testbed.command(cud, (src, dst))
 297                 p.file[tbp] = p.path[tbp]
 298
 299         return p.file[tbp] + p.dir
 300
 301  def invalidate(p, tbp=False):
 302         p.file[tbp] = None
 303         p._debug('invalidated %s' % 'HT'[tbp])
 304
 305  def _debug(p, m):
 306         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 307
 308  def _constructed(p):
 309         p._debug('constructed: '+str(p))
 310         p._check()
 311
 312  def _check(p):
 313         for tbp in [False,True]:
 314          for pf in [p.path, p.file]:
 315                 if pf[tbp] is None: continue
 316                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 317                 if p.dir and pf[tbp].endswith('/'):
 318                         pf[tbp] = pf[tbp].rstrip('/')
 319                         if not pf[tbp]: pf[tbp] = '/'
 320                 if not p.dir and pf[tbp].endswith('/'):
 321                         bomb("directory `%s' specified for "
 322                              "non-directory %s" % (pf[tbp], p.what))
 323
 324  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 325         AutoFile.__init__(p,what)
 326         sh_on = ''; sh_sibl = ''
 327         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 328         if sibling: sh_sibl=' (sibling)'
 329         parent._debug('using as base: %s: %s%s%s' %
 330                         (str(parent), leaf, sh_on, sh_sibl))
 331         if not sibling and not parent.dir:
 332                 parent._wrong('asked for non-sibling relative path of non-dir')
 333         if sibling: trim = os.path.dirname
 334         else: trim = lambda x: x
 335         for tbp in [False,True]:
 336                 if parent.path[tbp] is None: continue
 337                 trimmed = trim(parent.path[tbp])
 338                 if trimmed: trimmed += '/'
 339                 p.path[tbp] = trimmed + leaf
 340                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 341                         p.file[tbp] = p.path[tbp]
 342
 343 class InputFile(AutoFile):
 344  def _init(p, what, spec, spec_tbp=False):
 345         AutoFile.__init__(p, what)
 346         p.spec = spec
 347         p.spec_tbp = spec_tbp
 348         p.path[spec_tbp] = spec
 349         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 350  def __init__(p, what, spec, spec_tbp=False):
 351         p._init(what,spec,spec_tbp)
 352         p._constructed()
 353
 354 class InputDir(InputFile):
 355  def __init__(p, what, spec, spec_tbp=False):
 356         InputFile._init(p,what,spec,spec_tbp)
 357         p.dir = '/'
 358         p._constructed()
 359
 360 class OutputFile(AutoFile):
 361  def _init(p, what, spec, spec_tbp=False):
 362         AutoFile.__init__(p, what)
 363         p.spec = spec
 364         p.spec_tbp = spec_tbp
 365         p.path[spec_tbp] = spec
 366  def __init__(p, what, spec, spec_tbp=False):
 367         p._init(what,spec,spec_tbp)
 368         p._constructed()
 369
 370 class OutputDir(OutputFile):
 371  def __init__(p, what, spec, spec_tbp=False):
 372         OutputFile._init(p,what,spec,spec_tbp)
 373         p.dir = '/'
 374         p._constructed()
 375
 376 class RelativeInputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 378         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 379         p._constructed()
 380
 381 class RelativeOutputFile(AutoFile):
 382  def __init__(p, what, parent, leaf, sibling=False):
 383         p._relative_init(what, parent, leaf, None, False, sibling)
 384         p._constructed()
 385
 386 class TemporaryFile(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p._constructed()
 390
 391 class TemporaryDir(AutoFile):
 392  def __init__(p, what):
 393         AutoFile.__init__(p, what)
 394         p.dir = '/'
 395         p._constructed()
 396
 397 #---------- parsing and representation of the arguments
 398
 399 class Action:
 400  def __init__(a, kind, af, arghandling, what):
 401         # extra attributes get added during processing
 402         a.kind = kind
 403         a.af = af
 404         a.ah = arghandling
 405         a.what = what
 406         a.missing_tests_control = False
 407  def __repr__(a):
 408         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 409
 410 def parse_args():
 411         global opts, timeouts
 412         global n_non_actions # argh, stupid python scoping rules
 413         usage = "%prog <options> --- <virt-server>..."
 414         parser = OptionParser(usage=usage)
 415         pa = parser.add_option
 416         pe = parser.add_option
 417
 418         arghandling = {
 419                 'dsc_tests': True,
 420                 'dsc_filter': '*',
 421                 'deb_forbuilds': 'auto',
 422                 'deb_fortests': 'auto',
 423                 'tb': False,
 424                 'override_control': None,
 425                 'set_lang': 'C'
 426         }
 427         initial_arghandling = arghandling.copy()
 428         n_non_actions = 0
 429
 430         #----------
 431         # actions (ie, test sets to run, sources to build, binaries to use):
 432
 433         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 434                 global n_non_actions
 435                 parser.largs.append((value,kindpath))
 436                 n_non_actions += not(is_act)
 437
 438         def pa_action(long, metavar, kindpath, help, is_act=True):
 439                 pa('','--'+long, action='callback', callback=cb_action,
 440                         nargs=1, type='string',
 441                         callback_args=(long,kindpath,is_act), help=help)
 442
 443         pa_action('built-tree',         'TREE', '@/',
 444                 help='run tests from build tree TREE')
 445
 446         pa_action('unbuilt-tree',       'TREE', '@//',
 447                 help='run tests from build tree TREE')
 448
 449         pa_action('source',             'DSC', '@.dsc',
 450                 help='build DSC and use its tests and/or'
 451                     ' generated binary packages')
 452
 453         pa_action('binary',             'DEB', '@.deb',
 454                help='use binary package DEB according'
 455                     ' to most recent --binaries-* settings')
 456
 457         def cb_actnoarg(op,optstr,value,parser, largsentry):
 458                 parser.largs.append(largsentry)
 459         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 460                 callback_args=((None, ('instantiate',)),),
 461                 help='instantiate testbed now (during testing phase)'
 462                      ' and install packages'
 463                      ' selected for automatic installation, even'
 464                      ' if this might apparently not be required otherwise')
 465
 466         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 467                help='run tests from control file CONTROL instead,'
 468                     ' (applies to next test suite only)')
 469
 470         #----------
 471         # argument handling settings (what ways to use action
 472         #  arguments, and pathname processing):
 473
 474         def cb_setah(option, opt_str, value, parser, toset,setval):
 475                 if type(setval) == list:
 476                         if not value in setval:
 477                                 parser.error('value for %s option (%s) is not '
 478                                  'one of the permitted values (%s)' %
 479                                  (value, opt_str, setval.join(' ')))
 480                 elif setval is not None:
 481                         value = setval
 482                 for v in toset:
 483                         arghandling[v] = value
 484                 parser.largs.append(arghandling.copy())
 485
 486         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 487                 type = metavar
 488                 if type is not None: type = 'string'
 489                 pa('',long, action='callback', callback=cb_setah,
 490                    callback_args=(affected,effect), **kwargs)
 491
 492         #---- paths: host or testbed:
 493         #
 494         pa_setah('--paths-testbed', ['tb'],True,
 495                 help='subsequent path specifications refer to the testbed')
 496         pa_setah('--paths-host', ['tb'],False,
 497                 help='subsequent path specifications refer to the host')
 498
 499         #---- source processing settings:
 500
 501         pa_setah('--sources-tests', ['dsc_tests'],True,
 502                 help='run tests from builds of subsequent sources')
 503         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 504                 help='do not run tests from builds of subsequent sources')
 505
 506         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 507                 type='string', metavar='PATTERN-LIST',
 508                 help='from subsequent sources, use binaries matching'
 509                      ' PATTERN-LIST (comma-separated glob patterns)'
 510                      ' according to most recent --binaries-* settings')
 511         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 512                 help='from subsequent sources, do not use any binaries')
 513         #---- binary package processing settings:
 514
 515         def pa_setahbins(long,toset,how):
 516          pa_setah(long, toset,['ignore','auto','install'],
 517                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 518                 help=how+' ignore binaries, install them as needed'
 519                         ' for dependencies, or unconditionally install'
 520                         ' them, respectively')
 521         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 522         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 523         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 524
 525         #----------
 526         # general options:
 527
 528         def cb_vserv(op,optstr,value,parser):
 529                 parser.values.vserver = list(parser.rargs)
 530                 del parser.rargs[:]
 531
 532         def cb_path(op,optstr,value,parser, constructor,long,dir):
 533                 name = long.replace('-','_')
 534                 af = constructor(long, value,arghandling['tb'])
 535                 setattr(parser.values, name, af)
 536
 537         def pa_path(long, constructor, help, dir=False):
 538                 pa('','--'+long, action='callback', callback=cb_path,
 539                         callback_args=(constructor,long,dir),
 540                         nargs=1, type='string',
 541                         help=help, metavar='PATH')
 542
 543         pa_path('output-dir', OutputDir, dir=True,
 544                 help='write stderr/out files in PATH')
 545
 546         pa('--leave-lang', dest='set_lang', action='store_false',
 547                 help="leave LANG on testbed set to testbed's default")
 548         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 549                 help='set LANG on testbed to LANGVAL', default='C')
 550
 551         pa('','--tmp-dir',              type='string', dest='tmpdir',
 552                 help='write temporary files to TMPDIR, emptying it'
 553                      ' beforehand and leaving it behind at the end')
 554         pa('','--log-file',             type='string', dest='logfile',
 555                 help='write the log LOGFILE, emptying it beforehand,'
 556                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 557         pa('','--summary-file',         type='string', dest='summary',
 558                 help='write a summary report to SUMMARY,'
 559                      ' emptying it beforehand')
 560
 561         for k in timeouts.keys():
 562                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 563                         metavar='T', help='set %s timeout to T')
 564         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 565                         metavar='FACTOR', default=1.0,
 566                         help='multiply all default timeouts by FACTOR')
 567
 568         pa('','--user',                 type='string', dest='user',
 569                 help='run tests as USER (needs root on testbed)')
 570         pa('','--gain-root',            type='string', dest='gainroot',
 571                 help='prefix debian/rules binary with GAINROOT')
 572         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 573         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 574         pa('','--gnupg-home',           type='string', dest='gnupghome',
 575                 default='~/.autopkgtest/gpg',
 576                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 577                         " signing private apt archive);"
 578                         " `fresh' means generate new key each time.")
 579
 580         #----------
 581         # actual meat:
 582
 583         class SpecialOption(optparse.Option): pass
 584         vs_op = SpecialOption('','--VSERVER-DUMMY')
 585         vs_op.action = 'callback'
 586         vs_op.type = None
 587         vs_op.default = None
 588         vs_op.nargs = 0
 589         vs_op.callback = cb_vserv
 590         vs_op.callback_args = ( )
 591         vs_op.callback_kwargs = { }
 592         vs_op.help = 'introduces virtualisation server and args'
 593         vs_op._short_opts = []
 594         vs_op._long_opts = ['---']
 595
 596         pa(vs_op)
 597
 598         (opts,args) = parser.parse_args()
 599         if not hasattr(opts,'vserver'):
 600                 parser.error('you must specifiy --- <virt-server>...')
 601         if n_non_actions >= len(parser.largs):
 602                 parser.error('nothing to do specified')
 603
 604         for k in timeouts.keys():
 605                 t = getattr(opts,'timeout_'+k)
 606                 if t is None: t = timeouts[k] * opts.timeout_factor
 607                 timeouts[k] = int(t)
 608
 609         arghandling = initial_arghandling
 610         opts.actions = []
 611         ix = 0
 612         for act in args:
 613                 if type(act) == dict:
 614                         arghandling = act
 615                         continue
 616                 elif type(act) == tuple:
 617                         pass
 618                 elif type(act) == str:
 619                         act = (act,act)
 620                 else:
 621                         raise ("unknown action in list `%s' having"
 622                               " type `%s'" % (act, type(act)))
 623                 (pathstr, kindpath) = act
 624
 625                 constructor = InputFile
 626                 if type(kindpath) is tuple:             kind = kindpath[0]
 627                 elif kindpath.endswith('.deb'):         kind = 'deb'
 628                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 629                 elif kindpath.endswith('//'):
 630                         kind = 'ubtree'
 631                         constructor = InputDir
 632                 elif kindpath.endswith('/'):
 633                         kind = 'tree'
 634                         constructor = InputDir
 635                 else: parser.error("do not know how to handle filename `%s';"
 636                         " specify --source --binary or --build-tree" %
 637                         kindpath)
 638
 639                 what = '%s%s' % (kind,ix); ix += 1
 640
 641                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 642                 else: fwhatx = '-'+kind
 643
 644                 if pathstr is None: af = None
 645                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 646                 opts.actions.append(Action(kind, af, arghandling, what))
 647
 648 def setup_trace():
 649         global trace_stream, tmpdir, summary_stream
 650
 651         if opts.tmpdir is not None:
 652                 rmtree('tmpdir(specified)',opts.tmpdir)
 653                 mkdir_okexist(opts.tmpdir, 0700)
 654                 tmpdir = opts.tmpdir
 655         else:
 656                 assert(tmpdir is None)
 657                 tmpdir = tempfile.mkdtemp()
 658
 659         if opts.logfile is None:
 660                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 661                         opts.logfile = opts.output_dir.spec + '/log'
 662                 elif opts.tmpdir is not None:
 663                         opts.logfile = opts.tmpdir + '/log'
 664         if opts.logfile is not None:
 665                 trace_stream = open(opts.logfile, 'w', 0)
 666         if opts.summary is not None:
 667                 summary_stream = open(opts.summary, 'w', 0)
 668
 669         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 670
 671 def finalise_options():
 672         global opts, tb, build_essential
 673
 674         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 675                 opts.user = ''
 676
 677         if opts.user is None:
 678                 su = 'suggested-normal-user='
 679                 ul = [
 680                         e[len(su):]
 681                         for e in testbed.caps
 682                         if e.startswith(su)
 683                         ]
 684                 if ul:
 685                         opts.user = ul[0]
 686                 else:
 687                         opts.user = ''
 688
 689         if opts.user:
 690                 if 'root-on-testbed' not in testbed.caps:
 691                         pstderr("warning: virtualisation"
 692                                 " system does not offer root on testbed,"
 693                                 " but --user option specified: failure likely")
 694                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 695         else:
 696                 opts.user_wrap = lambda x: x
 697
 698         if opts.gainroot is None:
 699                 opts.gainroot = ''
 700                 if (opts.user or
 701                     'root-on-testbed' not in testbed.caps):
 702                         opts.gainroot = 'fakeroot'
 703                         build_essential += ['fakeroot']
 704
 705         if opts.gnupghome.startswith('~/'):
 706                 try: home = os.environ['HOME']
 707                 except KeyError:
 708                         parser.error("HOME environment variable"
 709                                 " not set, needed for --gnupghome=`%s"
 710                                 % opts.gnupghome)
 711                 opts.gnupghome = home + opts.gnupghome[1:]
 712         elif opts.gnupghome == 'fresh':
 713                 opts.gnupghome = None
 714
 715 #---------- testbed management - the Testbed class
 716
 717 class Testbed:
 718  def __init__(tb):
 719         tb.sp = None
 720         tb.lastsend = None
 721         tb.scratch = None
 722         tb.modified = False
 723         tb.blamed = []
 724         tb._ephemeral = []
 725         tb._debug('init')
 726         tb._need_reset_apt = False
 727  def _debug(tb, m, minlevel=0):
 728         debug('** '+m, minlevel)
 729  def start(tb):
 730         tb._debug('start')
 731         p = subprocess.PIPE
 732         debug_subprocess('vserver', opts.vserver)
 733         tb._errplumb = Errplumb(True)
 734         tb.sp = subprocess.Popen(opts.vserver,
 735                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 736         tb.expect('ok')
 737         tb.caps = tb.commandr('capabilities')
 738  def stop(tb):
 739         tb._debug('stop')
 740         tb.close()
 741         if tb.sp is None: return
 742         ec = tb.sp.returncode
 743         if ec is None:
 744                 tb.sp.stdout.close()
 745                 tb.send('quit')
 746                 tb.sp.stdin.close()
 747                 ec = tb.sp.wait()
 748         if ec:
 749                 tb.bomb('testbed gave exit status %d after quit' % ec)
 750         tb._errplumb.wait()
 751  def open(tb):
 752         tb._debug('open, scratch=%s' % tb.scratch)
 753         if tb.scratch is not None: return
 754         pl = tb.commandr('open')
 755         tb.scratch = InputDir('tb-scratch', pl[0], True)
 756         tb.deps_processed = []
 757  def mungeing_apt(tb):
 758         if not 'revert' in tb.caps:
 759                 tb._need_reset_apt = True
 760  def reset_apt(tb):
 761         if not tb._need_reset_apt: return
 762         what = 'aptget-update-reset'
 763         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 764         rc = tb.execute(what, cmdl, kind='install')
 765         if rc:
 766                 pstderr("\n" "warning: failed to restore"
 767                         " testbed apt cache, exit code %d" % rc)
 768         tb._need_reset_apt = False
 769  def close(tb):
 770         tb._debug('close, scratch=%s' % tb.scratch)
 771         if tb.scratch is None: return
 772         tb.scratch = None
 773         if tb.sp is None: return
 774         tb.command('close')
 775  def prepare1(tb, deps_new):
 776         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 777                 (tb.modified, tb.deps_processed, deps_new), 1)
 778         if 'revert' in tb.caps and (tb.modified or
 779             [d for d in tb.deps_processed if d not in deps_new]):
 780                 for af in tb._ephemeral: af.read(False)
 781                 tb._debug('reset **')
 782                 tb.command('revert')
 783                 tb.blamed = []
 784                 for af in tb._ephemeral: af.invalidate(True)
 785         tb.modified = False
 786  def prepare2(tb, deps_new):
 787         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 788         binaries.publish()
 789         tb._install_deps(deps_new)
 790  def prepare(tb, deps_new):
 791         tb.prepare1(deps_new)
 792         tb.prepare2(deps_new)
 793  def register_ephemeral(tb, af):
 794         tb._ephemeral.append(af)
 795  def _install_deps(tb, deps_new):
 796         tb._debug(' installing dependencies '+`deps_new`, 1)
 797         tb.deps_processed = deps_new
 798         if not deps_new: return
 799         dstr = ', '.join(deps_new)
 800         script = binaries.apt_pkg_gdebi_script(
 801                 dstr, [[
 802                 'from GDebi.DebPackage import DebPackage',
 803                 'd = DebPackage(cache)',
 804                 'res = d.satisfyDependsStr(arg)',
 805                 ]])
 806         cmdl = ['python','-c',script]
 807         what = 'install-deps'
 808         rc = testbed.execute(what+'-debinstall', cmdl, script=script,
 809                                 kind='install')
 810         if rc: badpkg('dependency install failed, exit code %d' % rc)
 811  def needs_reset(tb):
 812         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 813         tb.modified = True
 814  def blame(tb, m):
 815         tb._debug('blame += %s' % m, 1)
 816         tb.blamed.append(m)
 817  def bomb(tb, m):
 818         tb._debug('bomb %s' % m)
 819         if tb.sp is not None:
 820                 tb.sp.stdout.close()
 821                 tb.sp.stdin.close()
 822                 ec = tb.sp.wait()
 823                 if ec: pstderr('adt-run: testbed failing,'
 824                                 ' exit status %d' % ec)
 825         tb.sp = None
 826         raise Quit(16, 'testbed failed: %s' % m)
 827  def send(tb, string):
 828         tb.sp.stdin
 829         try:
 830                 debug('>> '+string, 2)
 831                 print >>tb.sp.stdin, string
 832                 tb.sp.stdin.flush()
 833                 tb.lastsend = string
 834         except:
 835                 (type, value, dummy) = sys.exc_info()
 836                 tb.bomb('cannot send to testbed: %s' % traceback.
 837                         format_exception_only(type, value))
 838  def expect(tb, keyword, nresults=None):
 839         l = tb.sp.stdout.readline()
 840         if not l: tb.bomb('unexpected eof from the testbed')
 841         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 842         l = l.rstrip('\n')
 843         debug('<< '+l, 2)
 844         ll = l.split()
 845         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 846         if ll[0] != keyword:
 847                 if tb.lastsend is None:
 848                         tb.bomb("got banner `%s', expected `%s...'" %
 849                                 (l, keyword))
 850                 else:
 851                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 852                                 (tb.lastsend, l, keyword))
 853         ll = ll[1:]
 854         if nresults is not None and len(ll) != nresults:
 855                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 856                         " expected %d result parameters" %
 857                         (string, l, len(ll), nresults))
 858         return ll
 859  def commandr(tb, cmd, args=(), nresults=None):
 860         # pass args=[None,...] or =(None,...) to avoid more url quoting
 861         if type(cmd) is str: cmd = [cmd]
 862         if len(args) and args[0] is None: args = args[1:]
 863         else: args = map(urllib.quote, args)
 864         al = cmd + args
 865         tb.send(string.join(al))
 866         ll = tb.expect('ok', nresults)
 867         rl = map(urllib.unquote, ll)
 868         return rl
 869  def command(tb, cmd, args=()):
 870         tb.commandr(cmd, args, 0)
 871  def commandr1(tb, cmd, args=()):
 872         rl = tb.commandr(cmd, args, 1)
 873         return rl[0]
 874  def execute(tb, what, cmdl,
 875                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 876                 script=False, tmpdir=None, kind='short'):
 877         # Options for script:
 878         #   False - do not call debug_subprocess, no synch. reporting required
 879         #   None or string - call debug_subprocess with that value,
 880         #                       plumb stderr through synchronously if possible
 881         # Options for se:
 882         #   None - usual Errplumb (output is of kind 2c)
 883         #   string - path on testbed (output is of kind 2d)
 884
 885         timeout = timeouts[kind]
 886
 887         if script is not False: debug_subprocess(what, cmdl, script=script)
 888         if cwd is None: cwd = tb.scratch.write(True)
 889
 890         xdump = None
 891         if se is None:
 892                 ep = Errplumb()
 893                 se_catch = TemporaryFile(what+'-xerr')
 894                 se_use = se_catch.write(True)
 895                 if not opts.quiet: xdump = 'debug=2-2'
 896                 elif trace_stream is not None:
 897                         xdump = 'debug=2-%d' % trace_stream.fileno()
 898         else:
 899                 ep = None
 900                 se_catch = None
 901                 se_use = se
 902
 903         cmdl = [None,
 904                 ','.join(map(urllib.quote, cmdl)),
 905                 si, so, se_use, cwd]
 906
 907         if timeout is not None and timeout > 0:
 908                 cmdl.append('timeout=%d' % timeout)
 909
 910         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 911         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 912         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 913         if opts.set_lang is not False:
 914                 cmdl.append('env=LANG=%s' % opts.set_lang)
 915
 916         rc = tb.commandr1('execute', cmdl)
 917         try: rc = int(rc)
 918         except ValueError: bomb("execute for %s gave invalid response `%s'"
 919                                         % (what,rc))
 920
 921         if se_catch is not None:
 922                 debug_file(se_catch.read())
 923         if ep is not None:
 924                 ep.wait()
 925
 926         return rc
 927
 928 #---------- representation of test control files: Field*, Test, etc.
 929
 930 class FieldBase:
 931  def __init__(f, fname, stz, base, tnames, vl):
 932         assert(vl)
 933         f.stz = stz
 934         f.base = base
 935         f.tnames = tnames
 936         f.vl = vl
 937  def words(f):
 938         def distribute(vle):
 939                 (lno, v) = vle
 940                 r = v.split()
 941                 r = map((lambda w: (lno, w)), r)
 942                 return r
 943         return flatten(map(distribute, f.vl))
 944  def atmostone(f):
 945         if len(vl) == 1:
 946                 (f.lno, f.v) = vl[0]
 947         else:
 948                 raise Unsupported(f.vl[1][0],
 949                         'only one %s field allowed' % fn)
 950         return f.v
 951
 952 class FieldIgnore(FieldBase):
 953  def parse(f): pass
 954
 955 class Restriction:
 956  def __init__(r,rname,base): pass
 957
 958 class Restriction_rw_build_tree(Restriction): pass
 959 class Restriction_breaks_testbed(Restriction):
 960  def __init__(r, rname, base):
 961         if 'revert' not in testbed.caps:
 962                 raise Unsupported(f.lno,
 963                         'Test breaks testbed but testbed cannot revert')
 964 class Restriction_needs_root(Restriction):
 965  def __init__(r, rname, base):
 966         if 'root-on-testbed' not in testbed.caps:
 967                 raise Unsupported(f.lno,
 968                         'Test needs root on testbed which is not available')
 969
 970 class Field_Restrictions(FieldBase):
 971  def parse(f):
 972         for wle in f.words():
 973                 (lno, rname) = wle
 974                 nrname = rname.replace('-','_')
 975                 try: rclass = globals()['Restriction_'+nrname]
 976                 except KeyError: raise Unsupported(lno,
 977                         'unknown restriction %s' % rname)
 978                 r = rclass(nrname, f.base)
 979                 f.base['restriction_names'].append(rname)
 980                 f.base['restrictions'].append(r)
 981
 982 class Field_Features(FieldIgnore):
 983  def parse(f):
 984         for wle in f.words():
 985                 (lno, fname) = wle
 986                 f.base['feature_names'].append(fname)
 987                 nfname = fname.replace('-','_')
 988                 try: fclass = globals()['Feature_'+nfname]
 989                 except KeyError: continue
 990                 ft = fclass(nfname, f.base)
 991                 f.base['features'].append(ft)
 992
 993 class Field_Tests(FieldIgnore): pass
 994
 995 class Field_Depends(FieldBase):
 996  def parse(f):
 997         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 998         dl = map(lambda x: x.strip(),
 999                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1000         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1001         for d in dl:
1002                 if re.search(d):
1003                         badpkg("Test Depends field contains dependency"
1004                                " `%s' with invalid characters" % d)
1005         f.base['depends'] = dl
1006
1007 class Field_Tests_directory(FieldBase):
1008  def parse(f):
1009         td = atmostone(f)
1010         if td.startswith('/'): raise Unspported(f.lno,
1011                 'Tests-Directory may not be absolute')
1012         f.base['testsdir'] = td
1013
1014 def run_tests(stanzas, tree):
1015         global errorcode, testbed
1016         if stanzas == ():
1017                 report('*', 'SKIP no tests in this package')
1018                 errorcode |= 8
1019         for stanza in stanzas:
1020                 tests = stanza[' tests']
1021                 if not tests:
1022                         report('*', 'SKIP package has metadata but no tests')
1023                         errorcode |= 8
1024                 for t in tests:
1025                         t.prepare()
1026                         t.run(tree)
1027                         if 'breaks-testbed' in t.restriction_names:
1028                                 testbed.needs_reset()
1029                 testbed.needs_reset()
1030
1031 class Test:
1032  def __init__(t, tname, base, act):
1033         if '/' in tname: raise Unsupported(base[' lno'],
1034                 'test name may not contain / character')
1035         for k in base: setattr(t,k,base[k])
1036         t.tname = tname
1037         t.act = act
1038         t.what = act.what+'t-'+tname
1039         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1040         else: t.path = tname
1041         t._debug('constructed; path=%s' % t.path)
1042         t._debug(' .depends=%s' % t.depends)
1043  def _debug(t, m):
1044         debug('& %s: %s' % (t.what, m))
1045  def report(t, m):
1046         report(t.what, m)
1047  def reportfail(t, m):
1048         global errorcode
1049         errorcode |= 4
1050         report(t.what, 'FAIL ' + m)
1051  def prepare(t):
1052         t._debug('preparing')
1053         dn = []
1054         for d in t.depends:
1055                 t._debug(' processing dependency '+d)
1056                 if not '@' in d:
1057                         t._debug('  literal dependency '+d)
1058                         dn.append(d)
1059                 else:
1060                         for (pkg,bin) in t.act.binaries:
1061                                 d = d.replace('@',pkg)
1062                                 t._debug('  synthesised dependency '+d)
1063                                 dn.append(d)
1064         testbed.prepare(dn)
1065  def run(t, tree):
1066         t._debug('[----------------------------------------')
1067         def stdouterr(oe):
1068                 idstr = t.what + '-' + oe
1069                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1070                         use_dir = opts.output_dir
1071                 else:
1072                         use_dir = testbed.scratch
1073                 return RelativeOutputFile(idstr, use_dir, idstr)
1074
1075         if hasattr(t.act,'work'): t.act.work.write(True)
1076         tree.read(True)
1077
1078         af = RelativeInputFile(t.what, tree, t.path)
1079         so = stdouterr('stdout')
1080         se = stdouterr('stderr')
1081
1082         tf = af.read(True)
1083         tmpdir = None
1084
1085         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1086         if rc: bomb('failed to chmod +x %s' % tf)
1087
1088         if 'needs-root' not in t.restriction_names and opts.user is not None:
1089                 tfl = ['su',opts.user,'-c',tf]
1090                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1091                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1092                 if opts.user: script += '; chown %s "$1"' % opts.user
1093                 if 'rw-build-tree' in t.restriction_names:
1094                         script += '; chown -R %s "$2"' % opts.user
1095                 rc = testbed.execute('mktmpdir-'+t.what,
1096                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1097                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1098                                 % (tmpdir, rc))
1099         else:
1100                 tfl = [tf]
1101
1102         rc = testbed.execute('test-'+t.what, tfl,
1103                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1104                 tmpdir=tmpdir, kind='test')
1105
1106         so_read = so.read()
1107         se_read = se.read()
1108
1109         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1110         stab = os.stat(se_read)
1111         if stab.st_size != 0:
1112                 l = open(se_read).readline()
1113                 l = l.rstrip('\n \t\r')
1114                 if len(l) > 35: l = l[:35] + '...'
1115                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1116                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1117                 debug_file(se_read)
1118         elif rc != 0:
1119                 t.reportfail('non-zero exit status %d' % rc)
1120         else:
1121                 t.report('PASS')
1122
1123         stab = os.stat(so_read)
1124         if stab.st_size != 0:
1125                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1126                 debug_file(so_read)
1127
1128         t._debug('----------------------------------------]')
1129
1130 def read_control(act, tree, control_override):
1131         stanzas = [ ]
1132
1133         if control_override is not None:
1134                 control_af = control_override
1135                 testbed.blame('arg:'+control_override.spec)
1136         else:
1137                 if act.missing_tests_control:
1138                         return ()
1139                 control_af = RelativeInputFile(act.what+'-testcontrol',
1140                         tree, 'debian/tests/control')
1141         try:
1142                 control = open(control_af.read(), 'r')
1143         except (IOError,OSError), oe:
1144                 if oe[0] != errno.ENOENT: raise
1145                 return []
1146
1147         lno = 0
1148         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1149         stz = None      # stz[field_name][index] = (lno, value)
1150                         # special field names:
1151                         # stz[' lno'] = number
1152                         # stz[' tests'] = list of Test objects
1153         def end_stanza(stz):
1154                 if stz is None: return
1155                 stz[' errs'] = 0
1156                 stanzas.append(stz)
1157                 stz = None
1158                 hcurrent = None
1159
1160         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1161         while 1:
1162                 l = control.readline()
1163                 if not l: break
1164                 lno += 1
1165                 if not l.endswith('\n'): badctrl('unterminated line')
1166                 if regexp.compile('\s*\#').match(l): continue
1167                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1168                 initmat = initre.match(l)
1169                 if initmat:
1170                         (fname, l) = initmat.groups()
1171                         fname = string.capwords(fname)
1172                         if stz is None:
1173                                 stz = { ' lno': lno, ' tests': [] }
1174                         if not stz.has_key(fname): stz[fname] = [ ]
1175                         hcurrent = stz[fname]
1176                 elif regexp.compile('\s').match(l):
1177                         if not hcurrent: badctrl('unexpected continuation')
1178                 else:
1179                         badctrl('syntax error')
1180                 hcurrent.append((lno, l))
1181         end_stanza(stz)
1182
1183         def testbadctrl(stz, lno, m):
1184                 report_badctrl(lno, m)
1185                 stz[' errs'] += 1
1186
1187         for stz in stanzas:
1188                 try:
1189                         try: tnames = stz['Tests']
1190                         except KeyError:
1191                                 tnames = ['*']
1192                                 raise Unsupported(stz[' lno'],
1193                                         'no Tests field')
1194                         tnames = map((lambda lt: lt[1]), tnames)
1195                         tnames = string.join(tnames).split()
1196                         base = {
1197                                 'restriction_names': [],
1198                                 'restrictions': [],
1199                                 'feature_names': [],
1200                                 'features': [],
1201                                 'testsdir': 'debian/tests',
1202                                 'depends' : '@'
1203                         }
1204                         for fname in stz.keys():
1205                                 if fname.startswith(' '): continue
1206                                 vl = stz[fname]
1207                                 try: fclass = globals()['Field_'+
1208                                         fname.replace('-','_')]
1209                                 except KeyError: raise Unsupported(vl[0][0],
1210                                         'unknown metadata field %s' % fname)
1211                                 f = fclass(stz, fname, base, tnames, vl)
1212                                 f.parse()
1213                         for tname in tnames:
1214                                 t = Test(tname, base, act)
1215                                 stz[' tests'].append(t)
1216                 except Unsupported, u:
1217                         for tname in tnames: u.report(tname)
1218                         continue
1219
1220         return stanzas
1221
1222 def print_exception(ei, msgprefix=''):
1223         if msgprefix: pstderr(msgprefix)
1224         (et, q, tb) = ei
1225         if et is Quit:
1226                 pstderr('adt-run: ' + q.m)
1227                 psummary('quitting: '+q.m)
1228                 return q.ec
1229         else:
1230                 pstderr("adt-run: unexpected, exceptional, error:")
1231                 psummary('quitting: unexpected error, consult transcript')
1232                 traceback.print_exc(None, sys.stderr)
1233                 if trace_stream is not None:
1234                         traceback.print_exc(None, trace_stream)
1235                 return 20
1236
1237 def cleanup():
1238         global trace_stream
1239         try:
1240                 if testbed is not None:
1241                         testbed.reset_apt()
1242                         testbed.stop()
1243                 if opts.tmpdir is None and tmpdir is not None:
1244                         rmtree('tmpdir', tmpdir)
1245                 if trace_stream is not None:
1246                         trace_stream.close()
1247                         trace_stream = None
1248         except:
1249                 print_exception(sys.exc_info(),
1250                         '\nadt-run: error cleaning up:\n')
1251                 os._exit(20)
1252
1253 #---------- registration, installation etc. of .deb's: Binaries
1254
1255 def determine_package(act):
1256         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1257         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1258         if rc: badpkg('failed to parse binary package, code %d' % rc)
1259         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1260         act.pkg = None
1261         for l in output.split('\n'):
1262                 m = re.match(l)
1263                 if not m: continue
1264                 if act.pkg: badpkg('two Package: lines in control file')
1265                 act.pkg = m.groups()[0]
1266         if not act.pkg: badpkg('no good Package: line in control file')
1267
1268 class Binaries:
1269  def __init__(b):
1270         b.dir = TemporaryDir('binaries')
1271         b.dir.write()
1272         ok = False
1273
1274         if opts.gnupghome is None:
1275                 opts.gnupghome = tmpdir+'/gnupg'
1276
1277         b._debug('initialising')
1278         try:
1279                 for x in ['pubring','secring']:
1280                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1281                 ok = True
1282         except (IOError,OSError), oe:
1283                 if oe.errno != errno.ENOENT: raise
1284
1285         if ok: b._debug('no key generation needed')
1286         else: b.genkey()
1287
1288  def _debug(b, s):
1289         debug('* '+s)
1290
1291  def genkey(b):
1292         b._debug('preparing for key generation')
1293
1294         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1295         mkdir_okexist(opts.gnupghome, 0700)
1296
1297         script = '''
1298   exec >&2
1299   cd "$1"
1300   cat <<"END" >key-gen-params
1301 Key-Type: DSA
1302 Key-Length: 1024
1303 Key-Usage: sign
1304 Name-Real: autopkgtest per-run key
1305 Name-Comment: do not trust this key
1306 Name-Email: autopkgtest@example.com
1307 END
1308   set -x
1309   gpg --homedir="$1" --batch --gen-key key-gen-params
1310                 '''
1311         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1312         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1313         if rc: bomb('key generation failed, code %d' % rc)
1314
1315  def apt_configs(b):
1316         return {
1317                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1318                 "Debug::pkgProblemResolver": "true",
1319         }
1320
1321  def apt_pkg_gdebi_script(b, arg, middle):
1322         script = [
1323                 'import apt_pkg',
1324                 'import urllib',
1325                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1326                 ]
1327         for (k,v) in b.apt_configs().iteritems():
1328                 v = urllib.quote(v)
1329                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1330                                 % (k, v))
1331         script += [
1332                 'from GDebi.Cache import Cache',
1333                 'cache = Cache()',
1334                 ]
1335         for m in middle:
1336                 script += m + [
1337                 'print res',
1338                 'print d.missingDeps',
1339                 'print d.requiredChanges',
1340                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1341                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1342                           'd._failureString)',
1343                 'cache.commit()',
1344                 ''
1345                 ]
1346         return '\n'.join(script)
1347  def apt_get(b):
1348         ag = ['apt-get','-qy']
1349         for kv in b.apt_configs().iteritems():
1350                 ag += ['-o', '%s=%s' % kv]
1351         return ' '.join(ag)
1352
1353  def reset(b):
1354         b._debug('reset')
1355         rmtree('binaries', b.dir.read())
1356         b.dir.invalidate()
1357         b.dir.write()
1358         b.install = []
1359         b.blamed = []
1360         b.registered = set()
1361
1362  def register(b, act, pkg, af, forwhat, blamed):
1363         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1364                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1365
1366         if act.ah['deb_'+forwhat] == 'ignore': return
1367
1368         b.blamed += testbed.blamed
1369
1370         leafname = pkg+'.deb'
1371         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1372
1373         try: os.remove(dest.write())
1374         except (IOError,OSError), oe:
1375                 if oe.errno != errno.ENOENT: raise e
1376
1377         try: os.link(af.read(), dest.write())
1378         except (IOError,OSError), oe:
1379                 if oe.errno != errno.EXDEV: raise e
1380                 shutil.copy(af.read(), dest)
1381
1382         if act.ah['deb_'+forwhat] == 'install':
1383                 b.install.append(pkg)
1384
1385         b.registered.add(pkg)
1386
1387  def publish(b):
1388         b._debug('publish')
1389
1390         script = '''
1391   exec >&2
1392   cd "$1"
1393   apt-ftparchive packages . >Packages
1394   gzip <Packages >Packages.gz
1395   apt-ftparchive release . >Release
1396   rm -f Release.gpg
1397   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1398   gpg --homedir="$2" --batch --export >archive-key.pgp
1399                 '''
1400         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1401         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1402         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1403
1404         b.dir.invalidate(True)
1405         apt_source = b.dir.read(True)
1406
1407         so = TemporaryFile('vlds')
1408         script = '''
1409   exec 3>&1 >&2
1410   apt-key add archive-key.pgp
1411   echo "deb file://'''+apt_source+''' /" >sources.list
1412   cat /etc/apt/sources.list >>sources.list
1413   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1414     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1415   fi
1416   '''+ b.apt_get() +''' update >&2
1417   cat /var/lib/dpkg/status >&3
1418                 '''
1419         testbed.mungeing_apt()
1420         rc = testbed.execute('apt-key', ['sh','-ec',script],
1421                                 so=so.write(True), cwd=b.dir.write(True),
1422                                 script=script, kind='install')
1423         if rc: bomb('apt setup failed with exit code %d' % rc)
1424
1425         testbed.blamed += b.blamed
1426
1427         b._debug('publish reinstall checking...')
1428         pkgs_reinstall = set()
1429         pkg = None
1430         for l in open(so.read()):
1431                 if l.startswith('Package: '):
1432                         pkg = l[9:].rstrip()
1433                 elif l.startswith('Status: install '):
1434                         if pkg in b.registered:
1435                                 pkgs_reinstall.add(pkg)
1436                                 b._debug(' publish reinstall needs '+pkg)
1437
1438         if pkgs_reinstall:
1439                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1440                 what = 'apt-get-reinstall'
1441                 cmdl = (b.apt_get() + ' --reinstall install '+
1442                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1443                 cmdl = ['sh','-c',cmdl]
1444                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1445                 if rc: badpkg("installation of basic binarries failed,"
1446                                 " exit code %d" % rc)
1447
1448         b._debug('publish install...')
1449         for pkg in b.install:
1450                 what = 'apt-get-install-%s' % pkg
1451                 testbed.blame(pkg)
1452                 cmdl = b.apt_get() + ' install ' + pkg + ' >&2'
1453                 cmdl = ['sh','-c',cmdl]
1454                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1455                 if rc: badpkg("installation of %s failed, exit code %d"
1456                                 % (pkg, rc))
1457
1458         b._debug('publish done')
1459
1460 #---------- processing of sources (building)
1461
1462 def source_rules_command(act,script,what,which,work,cwd,
1463                                 results_lines=0,xargs=[]):
1464         script = [      "exec 3>&1 >&2",
1465                         "set -x"        ] + script
1466         script = '\n'.join(script)
1467         so = TemporaryFile('%s-%s-results' % (what,which))
1468         rc = testbed.execute('%s-%s' % (what,which),
1469                         ['sh','-ec',script]+xargs, script=script,
1470                         so=so.write(True), cwd=cwd, kind='build')
1471         results = open(so.read()).read().rstrip('\n')
1472         if len(results): results = results.split("\n")
1473         else: results = []
1474         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1475         if results_lines is not None and len(results) != results_lines:
1476                 badpkg("got %d lines of results from %s where %d expected"
1477                         % (len(results), which, results_lines))
1478         if results_lines==1: return results[0]
1479         return results
1480
1481 def build_source(act, control_override):
1482         act.blame = 'arg:'+act.af.spec
1483         testbed.blame(act.blame)
1484         testbed.prepare1([])
1485         testbed.needs_reset()
1486
1487         what = act.what
1488         basename = act.af.spec
1489         debiancontrol = None
1490         act.binaries = []
1491
1492         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1493
1494         if act.kind == 'dsc':
1495                 dsc = act.af
1496                 dsc_file = open(dsc.read())
1497                 in_files = False
1498                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1499                 for l in dsc_file:
1500                         l = l.rstrip('\n')
1501                         if l.startswith('Files:'): in_files = True; continue
1502                         elif l.startswith('#'): pass
1503                         elif not l.startswith(' '):
1504                                 in_files = False
1505                                 if l.startswith('Source:'):
1506                                         act.blame = 'dsc:'+l[7:].strip()
1507                                         testbed.blame(act.blame)
1508                         if not in_files: continue
1509
1510                         m = fre.match(l)
1511                         if not m: badpkg(".dsc contains unparseable line"
1512                                         " in Files: `%s'" % l)
1513                         leaf = m.groups(0)[0]
1514                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1515                                         sibling=True)
1516                         subfile.read(True)
1517                 dsc.read(True)
1518
1519         if act.kind == 'ubtree':
1520                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1521                         act.af, 'debian/control')
1522                 dsc = TemporaryFile(what+'-fakedsc')
1523                 dsc_w = open(dsc.write(), 'w')
1524                 for l in open(debiancontrol.read()):
1525                         l = l.rstrip('\n')
1526                         if not len(l): break
1527                         print >>dsc_w, l
1528                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1529                 dsc_w.close()
1530
1531         if act.kind == 'dsc':
1532                 testbed.prepare2([])
1533                 script = binaries.apt_pkg_gdebi_script('', [[
1534                                 'from GDebi.DebPackage import DebPackage',
1535                                 'd = DebPackage(cache)',
1536                                 'res = d.satisfyDependsStr("dpkg-dev")',
1537                         ]])
1538                 cmdl = ['python','-c',script]
1539                 whatp = what+'-dpkgsource'
1540                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1541                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1542
1543         work = TemporaryDir(what+'-build')
1544         act.work = work
1545
1546         tmpdir = work.write(True)+'/tmpdir'
1547         tmpdir_script = [
1548                         'TMPDIR="$1"',
1549                         'rm -rf -- "$TMPDIR"',
1550                         'export TMPDIR',
1551                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1552                 ]
1553
1554         if act.kind == 'ubtree':
1555                 spec = '%s/real-tree' % work.write(True)
1556                 create_command = '''
1557                         rm -rf "$spec"
1558                         mkdir "$spec"
1559                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1560                         '''
1561                 initcwd = act.af.read(True)
1562
1563         if act.kind == 'dsc':
1564                 spec = dsc.read(True)
1565                 create_command = '''
1566                         dpkg-source -x $spec
1567                         '''
1568                 initcwd = work.write(True)
1569
1570         script = [
1571                 'spec="$2"',
1572                 'origpwd=`pwd`',
1573                 'cd '+work.write(True)
1574         ]
1575
1576         if opts.user:
1577                 script += ([ 'chown '+opts.user+' .' ] +
1578                         tmpdir_script +
1579                         [ 'spec="$spec" origpwd="$origpwd" '
1580                                 +opts.user_wrap(create_command) ])
1581         else:
1582                 script += (tmpdir_script +
1583                         [ create_command ])
1584
1585         script += [
1586                         'cd [a-z0-9]*-*/.',
1587                         'pwd >&3',
1588                         'set +e; test -f debian/tests/control; echo $? >&3'
1589                 ]
1590         (result_pwd, control_test_rc) = source_rules_command(
1591                         act,script,what,'extract',work,
1592                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1593
1594         filter = act.ah['dsc_filter']
1595
1596         if control_test_rc == '1': act.missing_tests_control = True
1597
1598         # For optional builds:
1599         #
1600         # We might need to build the package because:
1601         #   - we want its binaries (filter isn't _ and at least one of the
1602         #       deb_... isn't ignore)
1603         #   - the test control file says so
1604         #       (assuming we have any tests)
1605
1606         class NeedBuildException: pass
1607         def build_needed(m):
1608                 debug_b('build needed for %s' % m)
1609                 raise NeedBuildException()
1610
1611         try:
1612                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1613                                       act.ah['deb_fortests'] != 'ignore'):
1614                         build_needed('binaries')
1615
1616                 result_pwd_af = InputDir(what+'-treeforcontrol',
1617                         result_pwd, True)
1618                 stanzas = read_control(act, result_pwd_af, control_override)
1619                 for stanza in stanzas:
1620                         for t in stanza[' tests']:
1621                                 if 'no-build-needed' not in t.feature_names:
1622                                         build_needed('test %s' % t.tname)
1623                                 for d in t.depends:
1624                                         if '@' in d:
1625                                                 build_needed('test %s '
1626                                                  'dependency %s' % (t.tname,d))
1627
1628                 debug_b('build not needed')
1629                 built = False
1630
1631         except NeedBuildException:
1632
1633                 if act.kind != 'dsc':
1634                         testbed.prepare2([])
1635
1636                 script = binaries.apt_pkg_gdebi_script(
1637                         dsc.read(True), [[
1638                         'from GDebi.DscSrcPackage import DscSrcPackage',
1639                         'd = DscSrcPackage(cache, arg)',
1640                         'res = d.checkDeb()',
1641                          ],[
1642                         'from GDebi.DebPackage import DebPackage',
1643                         'd = DebPackage(cache)',
1644                         'res = d.satisfyDependsStr("'+
1645                                         ','.join(build_essential)+
1646                                 '")',
1647                         ]])
1648
1649                 cmdl = ['python','-c',script]
1650                 whatp = what+'-builddeps'
1651                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1652                 if rc: badpkg('build-depends install failed,'
1653                               ' exit code %d' % rc)
1654
1655                 script = tmpdir_script + [
1656                         'cd "$2"',
1657                         'dpkg-checkbuilddeps',
1658                         opts.user_wrap('debian/rules build'),
1659                         ]
1660                 source_rules_command(act,script,what,'build',work,
1661                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1662
1663                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1664                         badpkg("results dir `%s' is not in expected parent"
1665                                " dir `%s'" % (result_pwd, work.read(True)))
1666
1667                 built = True
1668
1669         act.tests_tree = InputDir(what+'-tests-tree',
1670                                 work.read(True)+os.path.basename(result_pwd),
1671                                 True)
1672         if act.ah['dsc_tests']:
1673                 testbed.register_ephemeral(act.work)
1674                 testbed.register_ephemeral(act.tests_tree)
1675
1676         if not built:
1677                 act.blamed = []
1678                 return
1679
1680         act.blamed = copy.copy(testbed.blamed)
1681
1682         debug_b('filter=%s' % filter)
1683         if filter != '_':
1684                 script = tmpdir_script + [
1685                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1686                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1687                         'cd ..',
1688                         'echo *.deb >&3',
1689                         ]
1690                 result_debs = source_rules_command(act,script,what,
1691                                 'binary',work,work.write(True),
1692                                 results_lines=1, xargs=['x',tmpdir])
1693                 if result_debs == '*.deb': debs = []
1694                 else: debs = result_debs.split(' ')
1695                 debug_b('debs='+`debs`)
1696                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1697                 for deb in debs:
1698                         m = re.match(deb)
1699                         if not m: badpkg("badly-named binary `%s'" % deb)
1700                         pkg = m.groups()[0]
1701                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1702                         for pat in filter.split(','):
1703                                 debug_b('  pat=%s' % pat)
1704                                 if not fnmatch.fnmatchcase(pkg,pat):
1705                                         debug_b('   no match')
1706                                         continue
1707                                 deb_what = pkg+'_'+what+'.deb'
1708                                 bin = RelativeInputFile(deb_what,work,deb,True)
1709                                 debug_b('  deb_what=%s, bin=%s' %
1710                                         (deb_what, str(bin)))
1711                                 binaries.register(act,pkg,bin,
1712                                         'forbuilds',testbed.blamed)
1713                                 act.binaries.append((pkg,bin))
1714                                 break
1715                 debug_b('all done.')
1716
1717 #---------- main processing loop and main program
1718
1719 def process_actions():
1720         global binaries
1721
1722         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1723         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1724         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1725
1726         debug_a1('starting')
1727         testbed.open()
1728         binaries = Binaries()
1729
1730         for act in opts.actions:
1731                 if act.af is not None and not act.af.spec_tbp:
1732                         testbed.register_ephemeral(act.af)
1733
1734         binaries.reset()
1735         control_override = None
1736
1737         debug_a1('builds ...')
1738         for act in opts.actions:
1739                 debug_a2('%s %s' %
1740                         (act.kind, act.what))
1741
1742                 if act.kind == 'control':
1743                         control_override = act.af
1744                 if act.kind == 'deb':
1745                         testbed.blame('arg:'+act.af.spec)
1746                         determine_package(act)
1747                         testbed.blame('deb:'+act.pkg)
1748                         binaries.register(act,act.pkg,act.af,
1749                                 'forbuilds',testbed.blamed)
1750                 if act.kind == 'dsc' or act.kind == 'ubtree':
1751                         build_source(act, control_override)
1752                 if act.kind == 'tree':
1753                         act.binaries = []
1754                 if act.kind.endswith('tree') or act.kind == 'dsc':
1755                         control_override = None
1756                 if act.kind == 'instantiate':
1757                         pass
1758
1759         debug_a1('builds done.')
1760
1761         binaries.reset()
1762         control_override = None
1763
1764         debug_a1('tests ...')
1765         for act in opts.actions:
1766                 debug_a2('test %s %s' % (act.kind, act.what))
1767
1768                 testbed.needs_reset()
1769                 if act.kind == 'control':
1770                         control_override = act.af
1771                 if act.kind == 'deb':
1772                         binaries.register(act,act.pkg,act.af,'fortests',
1773                                 ['deb:'+act.pkg])
1774                 if act.kind == 'dsc' or act.kind == 'ubtree':
1775                         for (pkg,bin) in act.binaries:
1776                                 binaries.register(act,pkg,bin,'fortests',
1777                                         act.blamed)
1778                 if act.kind == 'dsc':
1779                         if act.ah['dsc_tests']:
1780                                 debug_a3('read control ...')
1781                                 stanzas = read_control(act, act.tests_tree,
1782                                                 control_override)
1783                                 testbed.blamed += act.blamed
1784                                 debug_a3('run_tests ...')
1785                                 run_tests(stanzas, act.tests_tree)
1786                         control_override = None
1787                 if act.kind == 'tree' or act.kind == 'ubtree':
1788                         testbed.blame('arg:'+act.af.spec)
1789                         stanzas = read_control(act, act.af, control_override)
1790                         debug_a3('run_tests ...')
1791                         run_tests(stanzas, act.af)
1792                         control_override = None
1793                 if act.kind == 'instantiate':
1794                         testbed.prepare([])
1795         debug_a1('tests done.')
1796
1797 def main():
1798         global testbed
1799         global tmpdir
1800         try:
1801                 parse_args()
1802         except SystemExit, se:
1803                 os._exit(20)
1804         try:
1805                 setup_trace()
1806                 testbed = Testbed()
1807                 testbed.start()
1808                 finalise_options()
1809                 process_actions()
1810         except:
1811                 ec = print_exception(sys.exc_info(), '')
1812                 cleanup()
1813                 os._exit(ec)
1814         cleanup()
1815         os._exit(errorcode)
1816
1817 main()