runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         try: shutil.rmtree(pathname)
 192         except (IOError,OSError), oe:
 193                 if oe.errno != errno.EEXIST: raise
 194
 195 def debug_subprocess(what, cmdl=None, script=None):
 196         o = '$ '+what+':'
 197         if cmdl is not None:
 198                 ol = []
 199                 for x in cmdl:
 200                         if x is script: x = '<SCRIPT>'
 201                         ol.append(x.    replace('\\','\\\\').
 202                                         replace(' ','\\ ')      )
 203                 o += ' '+ ' '.join(ol)
 204         debug(o)
 205         if script is not None and opts.debuglevel >= 1:
 206                 o = ''
 207                 for l in script.rstrip('\n').split('\n'):
 208                         o += '$     '+l+'\n'
 209                 debug(o,1)
 210
 211 def flatten(l):
 212         return reduce((lambda a,b: a + b), l, [])
 213
 214 #---------- fancy automatic file-copying class
 215
 216 class AutoFile:
 217         # p.what
 218         # p.path[tb]    None or path    not None => path known
 219         # p.file[tb]    None or path    not None => file exists
 220         # p.spec        string or None
 221         # p.spec_tbp    True or False, or not set if spec is None
 222         # p.dir         '' or '/'
 223
 224  def __init__(p, what):
 225         p.what = what
 226         p.path = [None,None]
 227         p.file = [None,None]
 228         p.spec = None
 229         p.dir = ''
 230
 231  def __repr__(p):
 232         return "<AF@%s>" % p.__str__()
 233  def __str__(p):
 234         def ptbp(tbp):
 235                 if p.path[tbp] is None: return '-'+p.dir
 236                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 237                 else: return p.path[tbp]+p.dir+'!'
 238         out = p.what
 239         if p.spec is not None:
 240                 if not hasattr(p,'spec_tb'): out += '~'
 241                 elif p.spec_tbp: out += '#'
 242                 else: out += '='
 243                 out += p.spec
 244         out += ':'
 245         out += ptbp(False)
 246         out += '|'
 247         out += ptbp(True)
 248         return out
 249
 250  def _wrong(p, how):
 251         xtra = ''
 252         if p.spec is not None:
 253                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 254         raise ("internal error: %s (%s)" % (how, str(p)))
 255
 256  def _ensure_path(p, tbp):
 257         if p.path[tbp] is None:
 258                 if '/' in p.what:
 259                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 260                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 261                 if not tbp:
 262                         p.path[tbp] = tmpdir+'/'+p.what
 263                 else:
 264                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 265
 266  def write(p, tbp=False):
 267         p._debug('write %s...' % 'HT'[tbp])
 268         p._ensure_path(tbp)
 269
 270         if p.dir and not p.file[tbp]:
 271                 if not tbp:
 272                         p._debug('mkdir H')
 273                         mkdir_okexist(p.path[tbp])
 274                 else:
 275                         cmdl = ['sh','-ec',
 276                                 'test -d "$1" || mkdir -p "$1"',
 277                                 'x', p.path[tbp]]
 278                         tf_what = urllib.quote(p.what).replace('/','%2F')
 279                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 280                         if rc: bomb('failed to create directory %s' %
 281                                 p.path[tbp])
 282
 283         p.file[tbp] = p.path[tbp]
 284         return p.path[tbp]
 285
 286  def read(p, tbp=False):
 287         p._debug('read %s...' % 'HT'[tbp])
 288         p._ensure_path(tbp)
 289
 290         if p.file[tbp] is None:
 291                 if p.file[not tbp] is None:
 292                         p._wrong("requesting read but nonexistent")
 293                 cud = ['copyup','copydown'][tbp]
 294                 src = p.file[not tbp] + p.dir
 295                 dst = p.path[tbp] + p.dir
 296                 testbed.command(cud, (src, dst))
 297                 p.file[tbp] = p.path[tbp]
 298
 299         return p.file[tbp] + p.dir
 300
 301  def invalidate(p, tbp=False):
 302         p.file[tbp] = None
 303         p._debug('invalidated %s' % 'HT'[tbp])
 304
 305  def _debug(p, m):
 306         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 307
 308  def _constructed(p):
 309         p._debug('constructed: '+str(p))
 310         p._check()
 311
 312  def _check(p):
 313         for tbp in [False,True]:
 314          for pf in [p.path, p.file]:
 315                 if pf[tbp] is None: continue
 316                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 317                 if p.dir and pf[tbp].endswith('/'):
 318                         pf[tbp] = pf[tbp].rstrip('/')
 319                         if not pf[tbp]: pf[tbp] = '/'
 320                 if not p.dir and pf[tbp].endswith('/'):
 321                         bomb("directory `%s' specified for "
 322                              "non-directory %s" % (pf[tbp], p.what))
 323
 324  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 325         AutoFile.__init__(p,what)
 326         sh_on = ''; sh_sibl = ''
 327         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 328         if sibling: sh_sibl=' (sibling)'
 329         parent._debug('using as base: %s: %s%s%s' %
 330                         (str(parent), leaf, sh_on, sh_sibl))
 331         if not sibling and not parent.dir:
 332                 parent._wrong('asked for non-sibling relative path of non-dir')
 333         if sibling: trim = os.path.dirname
 334         else: trim = lambda x: x
 335         for tbp in [False,True]:
 336                 if parent.path[tbp] is None: continue
 337                 trimmed = trim(parent.path[tbp])
 338                 if trimmed: trimmed += '/'
 339                 p.path[tbp] = trimmed + leaf
 340                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 341                         p.file[tbp] = p.path[tbp]
 342
 343 class InputFile(AutoFile):
 344  def _init(p, what, spec, spec_tbp=False):
 345         AutoFile.__init__(p, what)
 346         p.spec = spec
 347         p.spec_tbp = spec_tbp
 348         p.path[spec_tbp] = spec
 349         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 350  def __init__(p, what, spec, spec_tbp=False):
 351         p._init(what,spec,spec_tbp)
 352         p._constructed()
 353
 354 class InputDir(InputFile):
 355  def __init__(p, what, spec, spec_tbp=False):
 356         InputFile._init(p,what,spec,spec_tbp)
 357         p.dir = '/'
 358         p._constructed()
 359
 360 class OutputFile(AutoFile):
 361  def _init(p, what, spec, spec_tbp=False):
 362         AutoFile.__init__(p, what)
 363         p.spec = spec
 364         p.spec_tbp = spec_tbp
 365         p.path[spec_tbp] = spec
 366  def __init__(p, what, spec, spec_tbp=False):
 367         p._init(what,spec,spec_tbp)
 368         p._constructed()
 369
 370 class OutputDir(OutputFile):
 371  def __init__(p, what, spec, spec_tbp=False):
 372         OutputFile._init(p,what,spec,spec_tbp)
 373         p.dir = '/'
 374         p._constructed()
 375
 376 class RelativeInputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 378         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 379         p._constructed()
 380
 381 class RelativeOutputFile(AutoFile):
 382  def __init__(p, what, parent, leaf, sibling=False):
 383         p._relative_init(what, parent, leaf, None, False, sibling)
 384         p._constructed()
 385
 386 class TemporaryFile(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p._constructed()
 390
 391 class TemporaryDir(AutoFile):
 392  def __init__(p, what):
 393         AutoFile.__init__(p, what)
 394         p.dir = '/'
 395         p._constructed()
 396
 397 #---------- parsing and representation of the arguments
 398
 399 class Action:
 400  def __init__(a, kind, af, arghandling, what):
 401         # extra attributes get added during processing
 402         a.kind = kind
 403         a.af = af
 404         a.ah = arghandling
 405         a.what = what
 406         a.missing_tests_control = False
 407  def __repr__(a):
 408         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 409
 410 def parse_args():
 411         global opts, timeouts
 412         global n_non_actions # argh, stupid python scoping rules
 413         usage = "%prog <options> --- <virt-server>..."
 414         parser = OptionParser(usage=usage)
 415         pa = parser.add_option
 416         pe = parser.add_option
 417
 418         arghandling = {
 419                 'dsc_tests': True,
 420                 'dsc_filter': '*',
 421                 'deb_forbuilds': 'auto',
 422                 'deb_fortests': 'auto',
 423                 'tb': False,
 424                 'override_control': None,
 425                 'set_lang': 'C'
 426         }
 427         initial_arghandling = arghandling.copy()
 428         n_non_actions = 0
 429
 430         #----------
 431         # actions (ie, test sets to run, sources to build, binaries to use):
 432
 433         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 434                 global n_non_actions
 435                 parser.largs.append((value,kindpath))
 436                 n_non_actions += not(is_act)
 437
 438         def pa_action(long, metavar, kindpath, help, is_act=True):
 439                 pa('','--'+long, action='callback', callback=cb_action,
 440                         nargs=1, type='string',
 441                         callback_args=(long,kindpath,is_act), help=help)
 442
 443         pa_action('built-tree',         'TREE', '@/',
 444                 help='run tests from build tree TREE')
 445
 446         pa_action('unbuilt-tree',       'TREE', '@//',
 447                 help='run tests from build tree TREE')
 448
 449         pa_action('source',             'DSC', '@.dsc',
 450                 help='build DSC and use its tests and/or'
 451                     ' generated binary packages')
 452
 453         pa_action('binary',             'DEB', '@.deb',
 454                help='use binary package DEB according'
 455                     ' to most recent --binaries-* settings')
 456
 457         def cb_actnoarg(op,optstr,value,parser, largsentry):
 458                 parser.largs.append(largsentry)
 459         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 460                 callback_args=((None, ('instantiate',)),),
 461                 help='instantiate testbed now (during testing phase)'
 462                      ' and install packages'
 463                      ' selected for automatic installation, even'
 464                      ' if this might apparently not be required otherwise')
 465
 466         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 467                help='run tests from control file CONTROL instead,'
 468                     ' (applies to next test suite only)')
 469
 470         #----------
 471         # argument handling settings (what ways to use action
 472         #  arguments, and pathname processing):
 473
 474         def cb_setah(option, opt_str, value, parser, toset,setval):
 475                 if type(setval) == list:
 476                         if not value in setval:
 477                                 parser.error('value for %s option (%s) is not '
 478                                  'one of the permitted values (%s)' %
 479                                  (value, opt_str, setval.join(' ')))
 480                 elif setval is not None:
 481                         value = setval
 482                 for v in toset:
 483                         arghandling[v] = value
 484                 parser.largs.append(arghandling.copy())
 485
 486         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 487                 type = metavar
 488                 if type is not None: type = 'string'
 489                 pa('',long, action='callback', callback=cb_setah,
 490                    callback_args=(affected,effect), **kwargs)
 491
 492         #---- paths: host or testbed:
 493         #
 494         pa_setah('--paths-testbed', ['tb'],True,
 495                 help='subsequent path specifications refer to the testbed')
 496         pa_setah('--paths-host', ['tb'],False,
 497                 help='subsequent path specifications refer to the host')
 498
 499         #---- source processing settings:
 500
 501         pa_setah('--sources-tests', ['dsc_tests'],True,
 502                 help='run tests from builds of subsequent sources')
 503         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 504                 help='do not run tests from builds of subsequent sources')
 505
 506         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 507                 type='string', metavar='PATTERN-LIST',
 508                 help='from subsequent sources, use binaries matching'
 509                      ' PATTERN-LIST (comma-separated glob patterns)'
 510                      ' according to most recent --binaries-* settings')
 511         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 512                 help='from subsequent sources, do not use any binaries')
 513         #---- binary package processing settings:
 514
 515         def pa_setahbins(long,toset,how):
 516          pa_setah(long, toset,['ignore','auto','install'],
 517                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 518                 help=how+' ignore binaries, install them as needed'
 519                         ' for dependencies, or unconditionally install'
 520                         ' them, respectively')
 521         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 522         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 523         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 524
 525         #----------
 526         # general options:
 527
 528         def cb_vserv(op,optstr,value,parser):
 529                 parser.values.vserver = list(parser.rargs)
 530                 del parser.rargs[:]
 531
 532         def cb_path(op,optstr,value,parser, constructor,long,dir):
 533                 name = long.replace('-','_')
 534                 af = constructor(long, value,arghandling['tb'])
 535                 setattr(parser.values, name, af)
 536
 537         def pa_path(long, constructor, help, dir=False):
 538                 pa('','--'+long, action='callback', callback=cb_path,
 539                         callback_args=(constructor,long,dir),
 540                         nargs=1, type='string',
 541                         help=help, metavar='PATH')
 542
 543         pa_path('output-dir', OutputDir, dir=True,
 544                 help='write stderr/out files in PATH')
 545
 546         pa('--leave-lang', dest='set_lang', action='store_false',
 547                 help="leave LANG on testbed set to testbed's default")
 548         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 549                 help='set LANG on testbed to LANGVAL', default='C')
 550
 551         pa('','--tmp-dir',              type='string', dest='tmpdir',
 552                 help='write temporary files to TMPDIR, emptying it'
 553                      ' beforehand and leaving it behind at the end')
 554         pa('','--log-file',             type='string', dest='logfile',
 555                 help='write the log LOGFILE, emptying it beforehand,'
 556                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 557         pa('','--summary-file',         type='string', dest='summary',
 558                 help='write a summary report to SUMMARY,'
 559                      ' emptying it beforehand')
 560
 561         for k in timeouts.keys():
 562                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 563                         metavar='T', help='set %s timeout to T')
 564         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 565                         metavar='FACTOR', default=1.0,
 566                         help='multiply all default timeouts by FACTOR')
 567
 568         pa('','--user',                 type='string', dest='user',
 569                 help='run tests as USER (needs root on testbed)')
 570         pa('','--gain-root',            type='string', dest='gainroot',
 571                 help='prefix debian/rules binary with GAINROOT')
 572         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 573         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 574         pa('','--gnupg-home',           type='string', dest='gnupghome',
 575                 default='~/.autopkgtest/gpg',
 576                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 577                         " signing private apt archive);"
 578                         " `fresh' means generate new key each time.")
 579
 580         #----------
 581         # actual meat:
 582
 583         class SpecialOption(optparse.Option): pass
 584         vs_op = SpecialOption('','--VSERVER-DUMMY')
 585         vs_op.action = 'callback'
 586         vs_op.type = None
 587         vs_op.default = None
 588         vs_op.nargs = 0
 589         vs_op.callback = cb_vserv
 590         vs_op.callback_args = ( )
 591         vs_op.callback_kwargs = { }
 592         vs_op.help = 'introduces virtualisation server and args'
 593         vs_op._short_opts = []
 594         vs_op._long_opts = ['---']
 595
 596         pa(vs_op)
 597
 598         (opts,args) = parser.parse_args()
 599         if not hasattr(opts,'vserver'):
 600                 parser.error('you must specifiy --- <virt-server>...')
 601         if n_non_actions >= len(parser.largs):
 602                 parser.error('nothing to do specified')
 603
 604         for k in timeouts.keys():
 605                 t = getattr(opts,'timeout_'+k)
 606                 if t is None: t = timeouts[k] * opts.timeout_factor
 607                 timeouts[k] = int(t)
 608
 609         arghandling = initial_arghandling
 610         opts.actions = []
 611         ix = 0
 612         for act in args:
 613                 if type(act) == dict:
 614                         arghandling = act
 615                         continue
 616                 elif type(act) == tuple:
 617                         pass
 618                 elif type(act) == str:
 619                         act = (act,act)
 620                 else:
 621                         raise ("unknown action in list `%s' having"
 622                               " type `%s'" % (act, type(act)))
 623                 (pathstr, kindpath) = act
 624
 625                 constructor = InputFile
 626                 if type(kindpath) is tuple:             kind = kindpath[0]
 627                 elif kindpath.endswith('.deb'):         kind = 'deb'
 628                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 629                 elif kindpath.endswith('//'):
 630                         kind = 'ubtree'
 631                         constructor = InputDir
 632                 elif kindpath.endswith('/'):
 633                         kind = 'tree'
 634                         constructor = InputDir
 635                 else: parser.error("do not know how to handle filename `%s';"
 636                         " specify --source --binary or --build-tree" %
 637                         kindpath)
 638
 639                 what = '%s%s' % (kind,ix); ix += 1
 640
 641                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 642                 else: fwhatx = '-'+kind
 643
 644                 if pathstr is None: af = None
 645                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 646                 opts.actions.append(Action(kind, af, arghandling, what))
 647
 648 def setup_trace():
 649         global trace_stream, tmpdir, summary_stream
 650
 651         if opts.tmpdir is not None:
 652                 rmtree('tmpdir(specified)',opts.tmpdir)
 653                 mkdir_okexist(opts.tmpdir, 0700)
 654                 tmpdir = opts.tmpdir
 655         else:
 656                 assert(tmpdir is None)
 657                 tmpdir = tempfile.mkdtemp()
 658
 659         if opts.logfile is None:
 660                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 661                         opts.logfile = opts.output_dir.spec + '/log'
 662                 elif opts.tmpdir is not None:
 663                         opts.logfile = opts.tmpdir + '/log'
 664         if opts.logfile is not None:
 665                 trace_stream = open(opts.logfile, 'w', 0)
 666         if opts.summary is not None:
 667                 summary_stream = open(opts.summary, 'w', 0)
 668
 669         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 670
 671 def finalise_options():
 672         global opts, tb, build_essential
 673
 674         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 675                 opts.user = ''
 676
 677         if opts.user is None:
 678                 su = 'suggested-normal-user='
 679                 ul = [
 680                         e[len(su):]
 681                         for e in testbed.caps
 682                         if e.startswith(su)
 683                         ]
 684                 if ul:
 685                         opts.user = ul[0]
 686                 else:
 687                         opts.user = ''
 688
 689         if opts.user:
 690                 if 'root-on-testbed' not in testbed.caps:
 691                         pstderr("warning: virtualisation"
 692                                 " system does not offer root on testbed,"
 693                                 " but --user option specified: failure likely")
 694                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 695         else:
 696                 opts.user_wrap = lambda x: x
 697
 698         if opts.gainroot is None:
 699                 opts.gainroot = ''
 700                 if (opts.user or
 701                     'root-on-testbed' not in testbed.caps):
 702                         opts.gainroot = 'fakeroot'
 703                         build_essential += ['fakeroot']
 704
 705         if opts.gnupghome.startswith('~/'):
 706                 try: home = os.environ['HOME']
 707                 except KeyError:
 708                         parser.error("HOME environment variable"
 709                                 " not set, needed for --gnupghome=`%s"
 710                                 % opts.gnupghome)
 711                 opts.gnupghome = home + opts.gnupghome[1:]
 712         elif opts.gnupghome == 'fresh':
 713                 opts.gnupghome = None
 714
 715 #---------- testbed management - the Testbed class
 716
 717 class Testbed:
 718  def __init__(tb):
 719         tb.sp = None
 720         tb.lastsend = None
 721         tb.scratch = None
 722         tb.modified = False
 723         tb.blamed = []
 724         tb._ephemeral = []
 725         tb._debug('init')
 726         tb._need_reset_apt = False
 727  def _debug(tb, m, minlevel=0):
 728         debug('** '+m, minlevel)
 729  def start(tb):
 730         tb._debug('start')
 731         p = subprocess.PIPE
 732         debug_subprocess('vserver', opts.vserver)
 733         tb._errplumb = Errplumb(True)
 734         tb.sp = subprocess.Popen(opts.vserver,
 735                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 736         tb.expect('ok')
 737         tb.caps = tb.commandr('capabilities')
 738         if not 'print-execute-command' in tb.caps:
 739                 tb.bomb('testbed does not support print-execute-command')
 740  def stop(tb):
 741         tb._debug('stop')
 742         tb.close()
 743         if tb.sp is None: return
 744         ec = tb.sp.returncode
 745         if ec is None:
 746                 tb.sp.stdout.close()
 747                 tb.send('quit')
 748                 tb.sp.stdin.close()
 749                 ec = tb.sp.wait()
 750         if ec:
 751                 tb.bomb('testbed gave exit status %d after quit' % ec)
 752         tb._errplumb.wait()
 753  def open(tb):
 754         tb._debug('open, scratch=%s' % tb.scratch)
 755         if tb.scratch is not None: return
 756         pl = tb.commandr('open')
 757         tb.scratch = InputDir('tb-scratch', pl[0], True)
 758         tb.deps_processed = []
 759         tb._auxverbscript_make()
 760
 761  def _auxverbscript_make(tb):
 762         pec = tb.commandr('print-execute-command')
 763         if len(pec) < 2: tb.bomb('too few results from print-execute-command')
 764         tb.ec_cmdl = map(urllib.unquote, pec[0].split(','))
 765         tb.ec_mode = urllib.unquote(pec[1])
 766         tb.ec_infos = map(urllib.unquote,pec[2:])
 767
 768         shellquote_re = regexp.compile(r'([\\"$`])')
 769         def shellquote_arg(s): return '"' + shellquote_re.sub(r'\\\1', s) + '"'
 770         def shellquote_cmdl(l): return ' '.join(map(shellquote_arg,l))
 771
 772         tb._debug('tb.ec_cmdl = %s' % (`tb.ec_cmdl`))
 773
 774         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 775         print >>open(tb.ec_auxverbscript.write(),'w'), (
 776 '''#!/bin/sh
 777 set -ex
 778 echo >&2 ": $*"
 779 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 780         set -- dpkg --print-architecture
 781 fi
 782 if [ "x$1" = xsh ] && [ "x$2" = x-c ]; then
 783         shift; shift
 784         # what a horrible hack!
 785 fi
 786 exec '''+shellquote_cmdl(tb.ec_cmdl)+' "$*"'+"\n"
 787                 )
 788         os.chmod(tb.ec_auxverbscript.write(), 0755)
 789
 790  def mungeing_apt(tb):
 791         if not 'revert' in tb.caps:
 792                 tb._need_reset_apt = True
 793  def reset_apt(tb):
 794         if not tb._need_reset_apt: return
 795         what = 'aptget-update-reset'
 796         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 797         rc = tb.execute(what, cmdl, kind='install')
 798         if rc:
 799                 pstderr("\n" "warning: failed to restore"
 800                         " testbed apt cache, exit code %d" % rc)
 801         tb._need_reset_apt = False
 802  def close(tb):
 803         tb._debug('close, scratch=%s' % tb.scratch)
 804         if tb.scratch is None: return
 805         tb.scratch = None
 806         if tb.sp is None: return
 807         tb.command('close')
 808  def prepare1(tb, deps_new):
 809         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 810                 (tb.modified, tb.deps_processed, deps_new), 1)
 811         if 'revert' in tb.caps and (tb.modified or
 812             [d for d in tb.deps_processed if d not in deps_new]):
 813                 for af in tb._ephemeral: af.read(False)
 814                 tb._debug('reset **')
 815                 tb.command('revert')
 816                 tb.blamed = []
 817                 for af in tb._ephemeral: af.invalidate(True)
 818                 tb._auxverbscript_make()
 819         tb.modified = False
 820  def prepare2(tb, deps_new):
 821         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 822         binaries.publish()
 823         tb._install_deps(deps_new)
 824  def prepare(tb, deps_new):
 825         tb.prepare1(deps_new)
 826         tb.prepare2(deps_new)
 827  def register_ephemeral(tb, af):
 828         tb._ephemeral.append(af)
 829  def _install_deps(tb, deps_new):
 830         tb._debug(' installing dependencies '+`deps_new`, 1)
 831         tb.deps_processed = deps_new
 832         if not deps_new: return
 833         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 834  def needs_reset(tb):
 835         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 836         tb.modified = True
 837  def blame(tb, m):
 838         tb._debug('blame += %s' % m, 1)
 839         tb.blamed.append(m)
 840  def bomb(tb, m):
 841         tb._debug('bomb %s' % m)
 842         if tb.sp is not None:
 843                 tb.sp.stdout.close()
 844                 tb.sp.stdin.close()
 845                 ec = tb.sp.wait()
 846                 if ec: pstderr('adt-run: testbed failing,'
 847                                 ' exit status %d' % ec)
 848         tb.sp = None
 849         raise Quit(16, 'testbed failed: %s' % m)
 850  def send(tb, string):
 851         tb.sp.stdin
 852         try:
 853                 debug('>> '+string, 2)
 854                 print >>tb.sp.stdin, string
 855                 tb.sp.stdin.flush()
 856                 tb.lastsend = string
 857         except:
 858                 (type, value, dummy) = sys.exc_info()
 859                 tb.bomb('cannot send to testbed: %s' % traceback.
 860                         format_exception_only(type, value))
 861  def expect(tb, keyword, nresults=None):
 862         l = tb.sp.stdout.readline()
 863         if not l: tb.bomb('unexpected eof from the testbed')
 864         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 865         l = l.rstrip('\n')
 866         debug('<< '+l, 2)
 867         ll = l.split()
 868         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 869         if ll[0] != keyword:
 870                 if tb.lastsend is None:
 871                         tb.bomb("got banner `%s', expected `%s...'" %
 872                                 (l, keyword))
 873                 else:
 874                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 875                                 (tb.lastsend, l, keyword))
 876         ll = ll[1:]
 877         if nresults is not None and len(ll) != nresults:
 878                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 879                         " expected %d result parameters" %
 880                         (string, l, len(ll), nresults))
 881         return ll
 882  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 883         # pass args=[None,...] or =(None,...) to avoid more url quoting
 884         if type(cmd) is str: cmd = [cmd]
 885         if len(args) and args[0] is None: args = args[1:]
 886         else: args = map(urllib.quote, args)
 887         al = cmd + args
 888         tb.send(string.join(al))
 889         ll = tb.expect('ok', nresults)
 890         if unquote: ll = map(urllib.unquote, ll)
 891         return ll
 892  def command(tb, cmd, args=()):
 893         tb.commandr(cmd, args, 0)
 894  def commandr1(tb, cmd, args=()):
 895         rl = tb.commandr(cmd, args, 1)
 896         return rl[0]
 897  def execute(tb, what, cmdl,
 898                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 899                 script=False, tmpdir=None, kind='short'):
 900         # Options for script:
 901         #   False - do not call debug_subprocess, no synch. reporting required
 902         #   None or string - call debug_subprocess with that value,
 903         #                       plumb stderr through synchronously if possible
 904         # Options for se:
 905         #   None - usual Errplumb (output is of kind 2c)
 906         #   string - path on testbed (output is of kind 2d)
 907
 908         timeout = timeouts[kind]
 909
 910         if script is not False: debug_subprocess(what, cmdl, script=script)
 911         if cwd is None: cwd = tb.scratch.write(True)
 912
 913         xdump = None
 914         if se is None:
 915                 ep = Errplumb()
 916                 se_catch = TemporaryFile(what+'-xerr')
 917                 se_use = se_catch.write(True)
 918                 if not opts.quiet: xdump = 'debug=2-2'
 919                 elif trace_stream is not None:
 920                         xdump = 'debug=2-%d' % trace_stream.fileno()
 921         else:
 922                 ep = None
 923                 se_catch = None
 924                 se_use = se
 925
 926         cmdl = [None,
 927                 ','.join(map(urllib.quote, cmdl)),
 928                 si, so, se_use, cwd]
 929
 930         if timeout is not None and timeout > 0:
 931                 cmdl.append('timeout=%d' % timeout)
 932
 933         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 934         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 935         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 936         if opts.set_lang is not False:
 937                 cmdl.append('env=LANG=%s' % opts.set_lang)
 938
 939         rc = tb.commandr1('execute', cmdl)
 940         try: rc = int(rc)
 941         except ValueError: bomb("execute for %s gave invalid response `%s'"
 942                                         % (what,rc))
 943
 944         if se_catch is not None:
 945                 debug_file(se_catch.read())
 946         if ep is not None:
 947                 ep.wait()
 948
 949         return rc
 950
 951  def satisfy_dependencies_string(tb, deps, what):
 952         # Must have called Binaries.configure_apt
 953         debug('dependencies: %s: satisfying %s' % (what,deps))
 954         dsc = TemporaryFile('deps.dsc')
 955         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 956         # pbuilder-satisfydepends has a bug where it ignores the
 957         #  Build-Depends if it's the last line in the dsc
 958         tb.satisfy_dependencies_dsc(dsc, what)
 959
 960  def satisfy_dependencies_dsc(tb, dsc, what):
 961         # Must have called Binaries.configure_apt
 962         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 963                 '--binary-all', # --check-key
 964                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 965                 '-c',dsc.read()
 966         ]
 967         debug('dependencies: %s: running %s' % (what,`cmdl`))
 968         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 969         if rc: badpkg('dependency install failed, exit code %d' % rc)
 970
 971 #---------- representation of test control files: Field*, Test, etc.
 972
 973 class FieldBase:
 974  def __init__(f, fname, stz, base, tnames, vl):
 975         assert(vl)
 976         f.stz = stz
 977         f.base = base
 978         f.tnames = tnames
 979         f.vl = vl
 980  def words(f):
 981         def distribute(vle):
 982                 (lno, v) = vle
 983                 r = v.split()
 984                 r = map((lambda w: (lno, w)), r)
 985                 return r
 986         return flatten(map(distribute, f.vl))
 987  def atmostone(f):
 988         if len(vl) == 1:
 989                 (f.lno, f.v) = vl[0]
 990         else:
 991                 raise Unsupported(f.vl[1][0],
 992                         'only one %s field allowed' % fn)
 993         return f.v
 994
 995 class FieldIgnore(FieldBase):
 996  def parse(f): pass
 997
 998 class Restriction:
 999  def __init__(r,rname,base): pass
1000
1001 class Restriction_rw_build_tree(Restriction): pass
1002 class Restriction_breaks_testbed(Restriction):
1003  def __init__(r, rname, base):
1004         if 'revert' not in testbed.caps:
1005                 raise Unsupported(f.lno,
1006                         'Test breaks testbed but testbed cannot revert')
1007 class Restriction_needs_root(Restriction):
1008  def __init__(r, rname, base):
1009         if 'root-on-testbed' not in testbed.caps:
1010                 raise Unsupported(f.lno,
1011                         'Test needs root on testbed which is not available')
1012
1013 class Field_Restrictions(FieldBase):
1014  def parse(f):
1015         for wle in f.words():
1016                 (lno, rname) = wle
1017                 nrname = rname.replace('-','_')
1018                 try: rclass = globals()['Restriction_'+nrname]
1019                 except KeyError: raise Unsupported(lno,
1020                         'unknown restriction %s' % rname)
1021                 r = rclass(nrname, f.base)
1022                 f.base['restriction_names'].append(rname)
1023                 f.base['restrictions'].append(r)
1024
1025 class Field_Features(FieldIgnore):
1026  def parse(f):
1027         for wle in f.words():
1028                 (lno, fname) = wle
1029                 f.base['feature_names'].append(fname)
1030                 nfname = fname.replace('-','_')
1031                 try: fclass = globals()['Feature_'+nfname]
1032                 except KeyError: continue
1033                 ft = fclass(nfname, f.base)
1034                 f.base['features'].append(ft)
1035
1036 class Field_Tests(FieldIgnore): pass
1037
1038 class Field_Depends(FieldBase):
1039  def parse(f):
1040         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
1041         dl = map(lambda x: x.strip(),
1042                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1043         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1044         for d in dl:
1045                 if re.search(d):
1046                         badpkg("Test Depends field contains dependency"
1047                                " `%s' with invalid characters" % d)
1048         f.base['depends'] = dl
1049
1050 class Field_Tests_directory(FieldBase):
1051  def parse(f):
1052         td = atmostone(f)
1053         if td.startswith('/'): raise Unspported(f.lno,
1054                 'Tests-Directory may not be absolute')
1055         f.base['testsdir'] = td
1056
1057 def run_tests(stanzas, tree):
1058         global errorcode, testbed
1059         if stanzas == ():
1060                 report('*', 'SKIP no tests in this package')
1061                 errorcode |= 8
1062         for stanza in stanzas:
1063                 tests = stanza[' tests']
1064                 if not tests:
1065                         report('*', 'SKIP package has metadata but no tests')
1066                         errorcode |= 8
1067                 for t in tests:
1068                         t.prepare()
1069                         t.run(tree)
1070                         if 'breaks-testbed' in t.restriction_names:
1071                                 testbed.needs_reset()
1072                 testbed.needs_reset()
1073
1074 class Test:
1075  def __init__(t, tname, base, act):
1076         if '/' in tname: raise Unsupported(base[' lno'],
1077                 'test name may not contain / character')
1078         for k in base: setattr(t,k,base[k])
1079         t.tname = tname
1080         t.act = act
1081         t.what = act.what+'t-'+tname
1082         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1083         else: t.path = tname
1084         t._debug('constructed; path=%s' % t.path)
1085         t._debug(' .depends=%s' % t.depends)
1086  def _debug(t, m):
1087         debug('& %s: %s' % (t.what, m))
1088  def report(t, m):
1089         report(t.what, m)
1090  def reportfail(t, m):
1091         global errorcode
1092         errorcode |= 4
1093         report(t.what, 'FAIL ' + m)
1094  def prepare(t):
1095         t._debug('preparing')
1096         dn = []
1097         for d in t.depends:
1098                 t._debug(' processing dependency '+d)
1099                 if not '@' in d:
1100                         t._debug('  literal dependency '+d)
1101                         dn.append(d)
1102                 else:
1103                         for (pkg,bin) in t.act.binaries:
1104                                 d = d.replace('@',pkg)
1105                                 t._debug('  synthesised dependency '+d)
1106                                 dn.append(d)
1107         testbed.prepare(dn)
1108  def run(t, tree):
1109         t._debug('[----------------------------------------')
1110         def stdouterr(oe):
1111                 idstr = t.what + '-' + oe
1112                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1113                         use_dir = opts.output_dir
1114                 else:
1115                         use_dir = testbed.scratch
1116                 return RelativeOutputFile(idstr, use_dir, idstr)
1117
1118         if hasattr(t.act,'work'): t.act.work.write(True)
1119         tree.read(True)
1120
1121         af = RelativeInputFile(t.what, tree, t.path)
1122         so = stdouterr('stdout')
1123         se = stdouterr('stderr')
1124
1125         tf = af.read(True)
1126         tmpdir = None
1127
1128         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1129         if rc: bomb('failed to chmod +x %s' % tf)
1130
1131         if 'needs-root' not in t.restriction_names and opts.user is not None:
1132                 tfl = ['su',opts.user,'-c',tf]
1133                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1134                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1135                 if opts.user:
1136                         script += '; chown %s "$1"' % opts.user
1137                         if 'rw-build-tree' in t.restriction_names:
1138                                 script += '; chown -R %s "$2"' % opts.user
1139                 rc = testbed.execute('mktmpdir-'+t.what,
1140                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1141                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1142                                 % (tmpdir, rc))
1143         else:
1144                 tfl = [tf]
1145
1146         rc = testbed.execute('test-'+t.what, tfl,
1147                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1148                 tmpdir=tmpdir, kind='test')
1149
1150         so_read = so.read()
1151         se_read = se.read()
1152
1153         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1154         stab = os.stat(se_read)
1155         if stab.st_size != 0:
1156                 l = open(se_read).readline()
1157                 l = l.rstrip('\n \t\r')
1158                 if len(l) > 35: l = l[:35] + '...'
1159                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1160                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1161                 debug_file(se_read)
1162         elif rc != 0:
1163                 t.reportfail('non-zero exit status %d' % rc)
1164         else:
1165                 t.report('PASS')
1166
1167         stab = os.stat(so_read)
1168         if stab.st_size != 0:
1169                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1170                 debug_file(so_read)
1171
1172         t._debug('----------------------------------------]')
1173
1174 def read_control(act, tree, control_override):
1175         stanzas = [ ]
1176
1177         if control_override is not None:
1178                 control_af = control_override
1179                 testbed.blame('arg:'+control_override.spec)
1180         else:
1181                 if act.missing_tests_control:
1182                         return ()
1183                 control_af = RelativeInputFile(act.what+'-testcontrol',
1184                         tree, 'debian/tests/control')
1185         try:
1186                 control = open(control_af.read(), 'r')
1187         except (IOError,OSError), oe:
1188                 if oe[0] != errno.ENOENT: raise
1189                 return []
1190
1191         lno = 0
1192         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1193         stz = None      # stz[field_name][index] = (lno, value)
1194                         # special field names:
1195                         # stz[' lno'] = number
1196                         # stz[' tests'] = list of Test objects
1197         def end_stanza(stz):
1198                 if stz is None: return
1199                 stz[' errs'] = 0
1200                 stanzas.append(stz)
1201                 stz = None
1202                 hcurrent = None
1203
1204         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1205         while 1:
1206                 l = control.readline()
1207                 if not l: break
1208                 lno += 1
1209                 if not l.endswith('\n'): badctrl('unterminated line')
1210                 if regexp.compile('\s*\#').match(l): continue
1211                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1212                 initmat = initre.match(l)
1213                 if initmat:
1214                         (fname, l) = initmat.groups()
1215                         fname = string.capwords(fname)
1216                         if stz is None:
1217                                 stz = { ' lno': lno, ' tests': [] }
1218                         if not stz.has_key(fname): stz[fname] = [ ]
1219                         hcurrent = stz[fname]
1220                 elif regexp.compile('\s').match(l):
1221                         if not hcurrent: badctrl('unexpected continuation')
1222                 else:
1223                         badctrl('syntax error')
1224                 hcurrent.append((lno, l))
1225         end_stanza(stz)
1226
1227         def testbadctrl(stz, lno, m):
1228                 report_badctrl(lno, m)
1229                 stz[' errs'] += 1
1230
1231         for stz in stanzas:
1232                 try:
1233                         try: tnames = stz['Tests']
1234                         except KeyError:
1235                                 tnames = ['*']
1236                                 raise Unsupported(stz[' lno'],
1237                                         'no Tests field')
1238                         tnames = map((lambda lt: lt[1]), tnames)
1239                         tnames = string.join(tnames).split()
1240                         base = {
1241                                 'restriction_names': [],
1242                                 'restrictions': [],
1243                                 'feature_names': [],
1244                                 'features': [],
1245                                 'testsdir': 'debian/tests',
1246                                 'depends' : '@'
1247                         }
1248                         for fname in stz.keys():
1249                                 if fname.startswith(' '): continue
1250                                 vl = stz[fname]
1251                                 try: fclass = globals()['Field_'+
1252                                         fname.replace('-','_')]
1253                                 except KeyError: raise Unsupported(vl[0][0],
1254                                         'unknown metadata field %s' % fname)
1255                                 f = fclass(stz, fname, base, tnames, vl)
1256                                 f.parse()
1257                         for tname in tnames:
1258                                 t = Test(tname, base, act)
1259                                 stz[' tests'].append(t)
1260                 except Unsupported, u:
1261                         for tname in tnames: u.report(tname)
1262                         continue
1263
1264         return stanzas
1265
1266 def print_exception(ei, msgprefix=''):
1267         if msgprefix: pstderr(msgprefix)
1268         (et, q, tb) = ei
1269         if et is Quit:
1270                 pstderr('adt-run: ' + q.m)
1271                 psummary('quitting: '+q.m)
1272                 return q.ec
1273         else:
1274                 pstderr("adt-run: unexpected, exceptional, error:")
1275                 psummary('quitting: unexpected error, consult transcript')
1276                 traceback.print_exc(None, sys.stderr)
1277                 if trace_stream is not None:
1278                         traceback.print_exc(None, trace_stream)
1279                 return 20
1280
1281 def cleanup():
1282         global trace_stream
1283         try:
1284                 if testbed is not None:
1285                         testbed.reset_apt()
1286                         testbed.stop()
1287                 if opts.tmpdir is None and tmpdir is not None:
1288                         rmtree('tmpdir', tmpdir)
1289                 if trace_stream is not None:
1290                         trace_stream.close()
1291                         trace_stream = None
1292         except:
1293                 print_exception(sys.exc_info(),
1294                         '\nadt-run: error cleaning up:\n')
1295                 os._exit(20)
1296
1297 #---------- registration, installation etc. of .deb's: Binaries
1298
1299 def determine_package(act):
1300         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1301         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1302         if rc: badpkg('failed to parse binary package, code %d' % rc)
1303         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1304         act.pkg = None
1305         for l in output.split('\n'):
1306                 m = re.match(l)
1307                 if not m: continue
1308                 if act.pkg: badpkg('two Package: lines in control file')
1309                 act.pkg = m.groups()[0]
1310         if not act.pkg: badpkg('no good Package: line in control file')
1311
1312 class Binaries:
1313  def __init__(b):
1314         b.dir = TemporaryDir('binaries')
1315         b.dir.write()
1316         ok = False
1317
1318         if opts.gnupghome is None:
1319                 opts.gnupghome = tmpdir+'/gnupg'
1320
1321         b._debug('initialising')
1322         try:
1323                 for x in ['pubring','secring']:
1324                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1325                 ok = True
1326         except (IOError,OSError), oe:
1327                 if oe.errno != errno.ENOENT: raise
1328
1329         if ok: b._debug('no key generation needed')
1330         else: b.genkey()
1331
1332  def _debug(b, s):
1333         debug('* '+s)
1334
1335  def genkey(b):
1336         b._debug('preparing for key generation')
1337
1338         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1339         mkdir_okexist(opts.gnupghome, 0700)
1340
1341         script = '''
1342   exec >&2
1343   cd "$1"
1344   cat <<"END" >key-gen-params
1345 Key-Type: DSA
1346 Key-Length: 1024
1347 Key-Usage: sign
1348 Name-Real: autopkgtest per-run key
1349 Name-Comment: do not trust this key
1350 Name-Email: autopkgtest@example.com
1351 END
1352   set -x
1353   gpg --homedir="$1" --batch --gen-key key-gen-params
1354                 '''
1355         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1356         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1357         if rc: bomb('key generation failed, code %d' % rc)
1358
1359  def apt_configs(b):
1360         return {
1361                 "Debug::pkgProblemResolver": "true",
1362                 "APT::Get::force-yes" : "true",
1363                 "APT::Get::Assume-Yes" : "true",
1364                 "quiet" : "true",
1365         }
1366
1367  def _configure_apt(b, tb):
1368         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1369                         True)
1370         f = open(config.write(),'w')
1371         for (k,v) in b.apt_configs().iteritems():
1372                 print >>f, '%s { "%s"; };' % (k, v)
1373         f.close()
1374         config.read(True)
1375
1376         prefs = OutputFile('apt-prefs','/etc/apt/preferences.d/90autopkgtest',
1377                         True)
1378         print >>open(prefs.write(),'w'), '''
1379 Package: *
1380 Pin: origin ""
1381 Pin-Priority: 1002
1382 '''
1383         prefs.read(True)
1384
1385  def _apt_get(b):
1386         ag = ['apt-get','-q']
1387         for kv in b.apt_configs().iteritems():
1388                 ag += ['-o', '%s=%s' % kv]
1389         return ' '.join(ag)
1390
1391  def reset(b):
1392         b._debug('reset')
1393         rmtree('binaries', b.dir.read())
1394         b.dir.invalidate()
1395         b.dir.write()
1396         b.install = []
1397         b.blamed = []
1398         b.registered = set()
1399
1400  def register(b, act, pkg, af, forwhat, blamed):
1401         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1402                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1403
1404         if act.ah['deb_'+forwhat] == 'ignore': return
1405
1406         b.blamed += testbed.blamed
1407
1408         leafname = pkg+'.deb'
1409         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1410
1411         try: os.remove(dest.write())
1412         except (IOError,OSError), oe:
1413                 if oe.errno != errno.ENOENT: raise e
1414
1415         try: os.link(af.read(), dest.write())
1416         except (IOError,OSError), oe:
1417                 if oe.errno != errno.EXDEV: raise e
1418                 shutil.copy(af.read(), dest)
1419
1420         if act.ah['deb_'+forwhat] == 'install':
1421                 b.install.append(pkg)
1422
1423         b.registered.add(pkg)
1424
1425  def publish(b):
1426         b._debug('publish')
1427
1428         b._configure_apt(testbed)
1429
1430         script = '''
1431   exec >&2
1432   cd "$1"
1433   apt-ftparchive packages . >Packages
1434   gzip <Packages >Packages.gz
1435   apt-ftparchive release . >Release
1436   rm -f Release.gpg
1437   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1438   gpg --homedir="$2" --batch --export >archive-key.pgp
1439                 '''
1440         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1441         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1442         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1443
1444         b.dir.invalidate(True)
1445         apt_source = b.dir.read(True)
1446
1447         so = TemporaryFile('vlds')
1448         script = '''
1449   exec 3>&1 >&2
1450   apt-key add archive-key.pgp
1451   echo "deb file://'''+apt_source+''' /" >/etc/apt/sources.list.d/autopkgtest.list
1452   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1453     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1454   fi
1455   '''+ b._apt_get() +''' update >&2
1456   cat /var/lib/dpkg/status >&3
1457                 '''
1458         testbed.mungeing_apt()
1459         rc = testbed.execute('apt-key', ['sh','-ec',script],
1460                                 so=so.write(True), cwd=b.dir.write(True),
1461                                 script=script, kind='install')
1462         if rc: bomb('apt setup failed with exit code %d' % rc)
1463
1464         testbed.blamed += b.blamed
1465
1466         b._debug('publish reinstall checking...')
1467         pkgs_reinstall = set()
1468         pkg = None
1469         for l in open(so.read()):
1470                 if l.startswith('Package: '):
1471                         pkg = l[9:].rstrip()
1472                 elif l.startswith('Status: install '):
1473                         if pkg in b.registered:
1474                                 pkgs_reinstall.add(pkg)
1475                                 b._debug(' publish reinstall needs '+pkg)
1476
1477         if pkgs_reinstall:
1478                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1479                 what = 'apt-get-reinstall'
1480                 cmdl = (b._apt_get() + ' --reinstall install '+
1481                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1482                 cmdl = ['sh','-c',cmdl]
1483                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1484                 if rc: badpkg("installation of basic binarries failed,"
1485                                 " exit code %d" % rc)
1486
1487         b._debug('publish install...')
1488         for pkg in b.install:
1489                 what = 'apt-get-install-%s' % pkg
1490                 testbed.blame(pkg)
1491                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1492                 cmdl = ['sh','-c',cmdl]
1493                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1494                 if rc: badpkg("installation of %s failed, exit code %d"
1495                                 % (pkg, rc))
1496
1497         b._debug('publish done')
1498
1499 #---------- processing of sources (building)
1500
1501 def source_rules_command(act,script,what,which,work,cwd,
1502                                 results_lines=0,xargs=[]):
1503         script = [      "exec 3>&1 >&2",
1504                         "set -x"        ] + script
1505         script = '\n'.join(script)
1506         so = TemporaryFile('%s-%s-results' % (what,which))
1507         rc = testbed.execute('%s-%s' % (what,which),
1508                         ['sh','-ec',script]+xargs, script=script,
1509                         so=so.write(True), cwd=cwd, kind='build')
1510         results = open(so.read()).read().rstrip('\n')
1511         if len(results): results = results.split("\n")
1512         else: results = []
1513         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1514         if results_lines is not None and len(results) != results_lines:
1515                 badpkg("got %d lines of results from %s where %d expected"
1516                         % (len(results), which, results_lines))
1517         if results_lines==1: return results[0]
1518         return results
1519
1520 def build_source(act, control_override):
1521         act.blame = 'arg:'+act.af.spec
1522         testbed.blame(act.blame)
1523         testbed.prepare1([])
1524         testbed.needs_reset()
1525
1526         what = act.what
1527         basename = act.af.spec
1528         debiancontrol = None
1529         act.binaries = []
1530
1531         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1532
1533         if act.kind == 'dsc':
1534                 dsc = act.af
1535                 dsc_file = open(dsc.read())
1536                 in_files = False
1537                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1538                 for l in dsc_file:
1539                         l = l.rstrip('\n')
1540                         if l.startswith('Files:'): in_files = True; continue
1541                         elif l.startswith('#'): pass
1542                         elif not l.startswith(' '):
1543                                 in_files = False
1544                                 if l.startswith('Source:'):
1545                                         act.blame = 'dsc:'+l[7:].strip()
1546                                         testbed.blame(act.blame)
1547                         if not in_files: continue
1548
1549                         m = fre.match(l)
1550                         if not m: badpkg(".dsc contains unparseable line"
1551                                         " in Files: `%s'" % l)
1552                         leaf = m.groups(0)[0]
1553                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1554                                         sibling=True)
1555                         subfile.read(True)
1556                 dsc.read(True)
1557
1558         if act.kind == 'ubtree':
1559                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1560                         act.af, 'debian/control')
1561                 dsc = TemporaryFile(what+'-fakedsc')
1562                 dsc_w = open(dsc.write(), 'w')
1563                 for l in open(debiancontrol.read()):
1564                         l = l.rstrip('\n')
1565                         if not len(l): break
1566                         print >>dsc_w, l
1567                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1568                 dsc_w.close()
1569
1570         if act.kind == 'dsc':
1571                 testbed.prepare2([])
1572                 testbed.satisfy_dependencies_string('dpkg-dev',
1573                                                 'install dpkg-dev')
1574
1575         work = TemporaryDir(what+'-build')
1576         act.work = work
1577
1578         tmpdir = work.write(True)+'/tmpdir'
1579         tmpdir_script = [
1580                         'TMPDIR="$1"',
1581                         'rm -rf -- "$TMPDIR"',
1582                         'export TMPDIR',
1583                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1584                 ]
1585
1586         if act.kind == 'ubtree':
1587                 spec = '%s/real-tree' % work.write(True)
1588                 create_command = '''
1589                         rm -rf "$spec"
1590                         mkdir "$spec"
1591                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1592                         '''
1593                 initcwd = act.af.read(True)
1594
1595         if act.kind == 'dsc':
1596                 spec = dsc.read(True)
1597                 create_command = '''
1598                         dpkg-source -x $spec
1599                         '''
1600                 initcwd = work.write(True)
1601
1602         script = [
1603                 'spec="$2"',
1604                 'origpwd=`pwd`',
1605                 'cd '+work.write(True)
1606         ]
1607
1608         if opts.user:
1609                 script += ([ 'chown '+opts.user+' .' ] +
1610                         tmpdir_script +
1611                         [ 'spec="$spec" origpwd="$origpwd" '
1612                                 +opts.user_wrap(create_command) ])
1613         else:
1614                 script += (tmpdir_script +
1615                         [ create_command ])
1616
1617         script += [
1618                         'cd [a-z0-9]*-*/.',
1619                         'pwd >&3',
1620                         'set +e; test -f debian/tests/control; echo $? >&3'
1621                 ]
1622         (result_pwd, control_test_rc) = source_rules_command(
1623                         act,script,what,'extract',work,
1624                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1625
1626         filter = act.ah['dsc_filter']
1627
1628         if control_test_rc == '1': act.missing_tests_control = True
1629
1630         # For optional builds:
1631         #
1632         # We might need to build the package because:
1633         #   - we want its binaries (filter isn't _ and at least one of the
1634         #       deb_... isn't ignore)
1635         #   - the test control file says so
1636         #       (assuming we have any tests)
1637
1638         class NeedBuildException: pass
1639         def build_needed(m):
1640                 debug_b('build needed for %s' % m)
1641                 raise NeedBuildException()
1642
1643         try:
1644                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1645                                       act.ah['deb_fortests'] != 'ignore'):
1646                         build_needed('binaries')
1647
1648                 result_pwd_af = InputDir(what+'-treeforcontrol',
1649                         result_pwd, True)
1650                 stanzas = read_control(act, result_pwd_af, control_override)
1651                 for stanza in stanzas:
1652                         for t in stanza[' tests']:
1653                                 if 'no-build-needed' not in t.feature_names:
1654                                         build_needed('test %s' % t.tname)
1655                                 for d in t.depends:
1656                                         if '@' in d:
1657                                                 build_needed('test %s '
1658                                                  'dependency %s' % (t.tname,d))
1659
1660                 debug_b('build not needed')
1661                 built = False
1662
1663         except NeedBuildException:
1664
1665                 if act.kind != 'dsc':
1666                         testbed.prepare2([])
1667
1668                 testbed.satisfy_dependencies_string('build-essential',
1669                                 'install build-essential')
1670                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1671
1672                 script = tmpdir_script + [
1673                         'cd "$2"',
1674                         'dpkg-checkbuilddeps',
1675                         opts.user_wrap('debian/rules build'),
1676                         ]
1677                 source_rules_command(act,script,what,'build',work,
1678                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1679
1680                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1681                         badpkg("results dir `%s' is not in expected parent"
1682                                " dir `%s'" % (result_pwd, work.read(True)))
1683
1684                 built = True
1685
1686         act.tests_tree = InputDir(what+'-tests-tree',
1687                                 work.read(True)+os.path.basename(result_pwd),
1688                                 True)
1689         if act.ah['dsc_tests']:
1690                 testbed.register_ephemeral(act.work)
1691                 testbed.register_ephemeral(act.tests_tree)
1692
1693         if not built:
1694                 act.blamed = []
1695                 return
1696
1697         act.blamed = copy.copy(testbed.blamed)
1698
1699         debug_b('filter=%s' % filter)
1700         if filter != '_':
1701                 script = tmpdir_script + [
1702                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1703                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1704                         'cd ..',
1705                         'echo *.deb >&3',
1706                         ]
1707                 result_debs = source_rules_command(act,script,what,
1708                                 'binary',work,work.write(True),
1709                                 results_lines=1, xargs=['x',tmpdir])
1710                 if result_debs == '*.deb': debs = []
1711                 else: debs = result_debs.split(' ')
1712                 debug_b('debs='+`debs`)
1713                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1714                 for deb in debs:
1715                         m = re.match(deb)
1716                         if not m: badpkg("badly-named binary `%s'" % deb)
1717                         pkg = m.groups()[0]
1718                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1719                         for pat in filter.split(','):
1720                                 debug_b('  pat=%s' % pat)
1721                                 if not fnmatch.fnmatchcase(pkg,pat):
1722                                         debug_b('   no match')
1723                                         continue
1724                                 deb_what = pkg+'_'+what+'.deb'
1725                                 bin = RelativeInputFile(deb_what,work,deb,True)
1726                                 debug_b('  deb_what=%s, bin=%s' %
1727                                         (deb_what, str(bin)))
1728                                 binaries.register(act,pkg,bin,
1729                                         'forbuilds',testbed.blamed)
1730                                 act.binaries.append((pkg,bin))
1731                                 break
1732                 debug_b('all done.')
1733
1734 #---------- main processing loop and main program
1735
1736 def process_actions():
1737         global binaries
1738
1739         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1740         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1741         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1742
1743         debug_a1('starting')
1744         testbed.open()
1745         binaries = Binaries()
1746
1747         for act in opts.actions:
1748                 if act.af is not None and not act.af.spec_tbp:
1749                         testbed.register_ephemeral(act.af)
1750
1751         binaries.reset()
1752         control_override = None
1753
1754         debug_a1('builds ...')
1755         for act in opts.actions:
1756                 debug_a2('%s %s' %
1757                         (act.kind, act.what))
1758
1759                 if act.kind == 'control':
1760                         control_override = act.af
1761                 if act.kind == 'deb':
1762                         testbed.blame('arg:'+act.af.spec)
1763                         determine_package(act)
1764                         testbed.blame('deb:'+act.pkg)
1765                         binaries.register(act,act.pkg,act.af,
1766                                 'forbuilds',testbed.blamed)
1767                 if act.kind == 'dsc' or act.kind == 'ubtree':
1768                         build_source(act, control_override)
1769                 if act.kind == 'tree':
1770                         act.binaries = []
1771                 if act.kind.endswith('tree') or act.kind == 'dsc':
1772                         control_override = None
1773                 if act.kind == 'instantiate':
1774                         pass
1775
1776         debug_a1('builds done.')
1777
1778         binaries.reset()
1779         control_override = None
1780
1781         debug_a1('tests ...')
1782         for act in opts.actions:
1783                 debug_a2('test %s %s' % (act.kind, act.what))
1784
1785                 testbed.needs_reset()
1786                 if act.kind == 'control':
1787                         control_override = act.af
1788                 if act.kind == 'deb':
1789                         binaries.register(act,act.pkg,act.af,'fortests',
1790                                 ['deb:'+act.pkg])
1791                 if act.kind == 'dsc' or act.kind == 'ubtree':
1792                         for (pkg,bin) in act.binaries:
1793                                 binaries.register(act,pkg,bin,'fortests',
1794                                         act.blamed)
1795                 if act.kind == 'dsc':
1796                         if act.ah['dsc_tests']:
1797                                 debug_a3('read control ...')
1798                                 stanzas = read_control(act, act.tests_tree,
1799                                                 control_override)
1800                                 testbed.blamed += act.blamed
1801                                 debug_a3('run_tests ...')
1802                                 run_tests(stanzas, act.tests_tree)
1803                         control_override = None
1804                 if act.kind == 'tree' or act.kind == 'ubtree':
1805                         testbed.blame('arg:'+act.af.spec)
1806                         stanzas = read_control(act, act.af, control_override)
1807                         debug_a3('run_tests ...')
1808                         run_tests(stanzas, act.af)
1809                         control_override = None
1810                 if act.kind == 'instantiate':
1811                         testbed.prepare([])
1812         debug_a1('tests done.')
1813
1814 def main():
1815         global testbed
1816         global tmpdir
1817         try:
1818                 parse_args()
1819         except SystemExit, se:
1820                 os._exit(20)
1821         try:
1822                 setup_trace()
1823                 testbed = Testbed()
1824                 testbed.start()
1825                 finalise_options()
1826                 process_actions()
1827         except:
1828                 ec = print_exception(sys.exc_info(), '')
1829                 cleanup()
1830                 os._exit(ec)
1831         cleanup()
1832         os._exit(errorcode)
1833
1834 main()