runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         try: shutil.rmtree(pathname)
 192         except (IOError,OSError), oe:
 193                 if oe.errno != errno.EEXIST: raise
 194
 195 def debug_subprocess(what, cmdl=None, script=None):
 196         o = '$ '+what+':'
 197         if cmdl is not None:
 198                 ol = []
 199                 for x in cmdl:
 200                         if x is script: x = '<SCRIPT>'
 201                         ol.append(x.    replace('\\','\\\\').
 202                                         replace(' ','\\ ')      )
 203                 o += ' '+ ' '.join(ol)
 204         debug(o)
 205         if script is not None and opts.debuglevel >= 1:
 206                 o = ''
 207                 for l in script.rstrip('\n').split('\n'):
 208                         o += '$     '+l+'\n'
 209                 debug(o,1)
 210
 211 def flatten(l):
 212         return reduce((lambda a,b: a + b), l, [])
 213
 214 #---------- fancy automatic file-copying class
 215
 216 class AutoFile:
 217         # p.what
 218         # p.path[tb]    None or path    not None => path known
 219         # p.file[tb]    None or path    not None => file exists
 220         # p.spec        string or None
 221         # p.spec_tbp    True or False, or not set if spec is None
 222         # p.dir         '' or '/'
 223
 224  def __init__(p, what):
 225         p.what = what
 226         p.path = [None,None]
 227         p.file = [None,None]
 228         p.spec = None
 229         p.dir = ''
 230
 231  def __repr__(p):
 232         return "<AF@%s>" % p.__str__()
 233  def __str__(p):
 234         def ptbp(tbp):
 235                 if p.path[tbp] is None: return '-'+p.dir
 236                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 237                 else: return p.path[tbp]+p.dir+'!'
 238         out = p.what
 239         if p.spec is not None:
 240                 if not hasattr(p,'spec_tb'): out += '~'
 241                 elif p.spec_tbp: out += '#'
 242                 else: out += '='
 243                 out += p.spec
 244         out += ':'
 245         out += ptbp(False)
 246         out += '|'
 247         out += ptbp(True)
 248         return out
 249
 250  def _wrong(p, how):
 251         xtra = ''
 252         if p.spec is not None:
 253                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 254         raise ("internal error: %s (%s)" % (how, str(p)))
 255
 256  def _ensure_path(p, tbp):
 257         if p.path[tbp] is None:
 258                 if '/' in p.what:
 259                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 260                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 261                 if not tbp:
 262                         p.path[tbp] = tmpdir+'/'+p.what
 263                 else:
 264                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 265
 266  def write(p, tbp=False):
 267         p._debug('write %s...' % 'HT'[tbp])
 268         p._ensure_path(tbp)
 269
 270         if p.dir and not p.file[tbp]:
 271                 if not tbp:
 272                         p._debug('mkdir H')
 273                         mkdir_okexist(p.path[tbp])
 274                 else:
 275                         cmdl = ['sh','-ec',
 276                                 'test -d "$1" || mkdir -p "$1"',
 277                                 'x', p.path[tbp]]
 278                         tf_what = urllib.quote(p.what).replace('/','%2F')
 279                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 280                         if rc: bomb('failed to create directory %s' %
 281                                 p.path[tbp])
 282
 283         p.file[tbp] = p.path[tbp]
 284         return p.path[tbp]
 285
 286  def read(p, tbp=False):
 287         p._debug('read %s...' % 'HT'[tbp])
 288         p._ensure_path(tbp)
 289
 290         if p.file[tbp] is None:
 291                 if p.file[not tbp] is None:
 292                         p._wrong("requesting read but nonexistent")
 293                 cud = ['copyup','copydown'][tbp]
 294                 src = p.file[not tbp] + p.dir
 295                 dst = p.path[tbp] + p.dir
 296                 testbed.command(cud, (src, dst))
 297                 p.file[tbp] = p.path[tbp]
 298
 299         return p.file[tbp] + p.dir
 300
 301  def invalidate(p, tbp=False):
 302         p.file[tbp] = None
 303         p._debug('invalidated %s' % 'HT'[tbp])
 304
 305  def _debug(p, m):
 306         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 307
 308  def _constructed(p):
 309         p._debug('constructed: '+str(p))
 310         p._check()
 311
 312  def _check(p):
 313         for tbp in [False,True]:
 314          for pf in [p.path, p.file]:
 315                 if pf[tbp] is None: continue
 316                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 317                 if p.dir and pf[tbp].endswith('/'):
 318                         pf[tbp] = pf[tbp].rstrip('/')
 319                         if not pf[tbp]: pf[tbp] = '/'
 320                 if not p.dir and pf[tbp].endswith('/'):
 321                         bomb("directory `%s' specified for "
 322                              "non-directory %s" % (pf[tbp], p.what))
 323
 324  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 325         AutoFile.__init__(p,what)
 326         sh_on = ''; sh_sibl = ''
 327         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 328         if sibling: sh_sibl=' (sibling)'
 329         parent._debug('using as base: %s: %s%s%s' %
 330                         (str(parent), leaf, sh_on, sh_sibl))
 331         if not sibling and not parent.dir:
 332                 parent._wrong('asked for non-sibling relative path of non-dir')
 333         if sibling: trim = os.path.dirname
 334         else: trim = lambda x: x
 335         for tbp in [False,True]:
 336                 if parent.path[tbp] is None: continue
 337                 trimmed = trim(parent.path[tbp])
 338                 if trimmed: trimmed += '/'
 339                 p.path[tbp] = trimmed + leaf
 340                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 341                         p.file[tbp] = p.path[tbp]
 342
 343 class InputFile(AutoFile):
 344  def _init(p, what, spec, spec_tbp=False):
 345         AutoFile.__init__(p, what)
 346         p.spec = spec
 347         p.spec_tbp = spec_tbp
 348         p.path[spec_tbp] = spec
 349         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 350  def __init__(p, what, spec, spec_tbp=False):
 351         p._init(what,spec,spec_tbp)
 352         p._constructed()
 353
 354 class InputDir(InputFile):
 355  def __init__(p, what, spec, spec_tbp=False):
 356         InputFile._init(p,what,spec,spec_tbp)
 357         p.dir = '/'
 358         p._constructed()
 359
 360 class OutputFile(AutoFile):
 361  def _init(p, what, spec, spec_tbp=False):
 362         AutoFile.__init__(p, what)
 363         p.spec = spec
 364         p.spec_tbp = spec_tbp
 365         p.path[spec_tbp] = spec
 366  def __init__(p, what, spec, spec_tbp=False):
 367         p._init(what,spec,spec_tbp)
 368         p._constructed()
 369
 370 class OutputDir(OutputFile):
 371  def __init__(p, what, spec, spec_tbp=False):
 372         OutputFile._init(p,what,spec,spec_tbp)
 373         p.dir = '/'
 374         p._constructed()
 375
 376 class RelativeInputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 378         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 379         p._constructed()
 380
 381 class RelativeOutputFile(AutoFile):
 382  def __init__(p, what, parent, leaf, sibling=False):
 383         p._relative_init(what, parent, leaf, None, False, sibling)
 384         p._constructed()
 385
 386 class TemporaryFile(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p._constructed()
 390
 391 class TemporaryDir(AutoFile):
 392  def __init__(p, what):
 393         AutoFile.__init__(p, what)
 394         p.dir = '/'
 395         p._constructed()
 396
 397 #---------- parsing and representation of the arguments
 398
 399 class Action:
 400  def __init__(a, kind, af, arghandling, what):
 401         # extra attributes get added during processing
 402         a.kind = kind
 403         a.af = af
 404         a.ah = arghandling
 405         a.what = what
 406         a.missing_tests_control = False
 407  def __repr__(a):
 408         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 409
 410 def parse_args():
 411         global opts, timeouts
 412         global n_non_actions # argh, stupid python scoping rules
 413         usage = "%prog <options> --- <virt-server>..."
 414         parser = OptionParser(usage=usage)
 415         pa = parser.add_option
 416         pe = parser.add_option
 417
 418         arghandling = {
 419                 'dsc_tests': True,
 420                 'dsc_filter': '*',
 421                 'deb_forbuilds': 'auto',
 422                 'deb_fortests': 'auto',
 423                 'tb': False,
 424                 'override_control': None,
 425                 'set_lang': 'C'
 426         }
 427         initial_arghandling = arghandling.copy()
 428         n_non_actions = 0
 429
 430         #----------
 431         # actions (ie, test sets to run, sources to build, binaries to use):
 432
 433         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 434                 global n_non_actions
 435                 parser.largs.append((value,kindpath))
 436                 n_non_actions += not(is_act)
 437
 438         def pa_action(long, metavar, kindpath, help, is_act=True):
 439                 pa('','--'+long, action='callback', callback=cb_action,
 440                         nargs=1, type='string',
 441                         callback_args=(long,kindpath,is_act), help=help)
 442
 443         pa_action('built-tree',         'TREE', '@/',
 444                 help='run tests from build tree TREE')
 445
 446         pa_action('unbuilt-tree',       'TREE', '@//',
 447                 help='run tests from build tree TREE')
 448
 449         pa_action('source',             'DSC', '@.dsc',
 450                 help='build DSC and use its tests and/or'
 451                     ' generated binary packages')
 452
 453         pa_action('binary',             'DEB', '@.deb',
 454                help='use binary package DEB according'
 455                     ' to most recent --binaries-* settings')
 456
 457         def cb_actnoarg(op,optstr,value,parser, largsentry):
 458                 parser.largs.append(largsentry)
 459         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 460                 callback_args=((None, ('instantiate',)),),
 461                 help='instantiate testbed now (during testing phase)'
 462                      ' and install packages'
 463                      ' selected for automatic installation, even'
 464                      ' if this might apparently not be required otherwise')
 465
 466         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 467                help='run tests from control file CONTROL instead,'
 468                     ' (applies to next test suite only)')
 469
 470         #----------
 471         # argument handling settings (what ways to use action
 472         #  arguments, and pathname processing):
 473
 474         def cb_setah(option, opt_str, value, parser, toset,setval):
 475                 if type(setval) == list:
 476                         if not value in setval:
 477                                 parser.error('value for %s option (%s) is not '
 478                                  'one of the permitted values (%s)' %
 479                                  (value, opt_str, setval.join(' ')))
 480                 elif setval is not None:
 481                         value = setval
 482                 for v in toset:
 483                         arghandling[v] = value
 484                 parser.largs.append(arghandling.copy())
 485
 486         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 487                 type = metavar
 488                 if type is not None: type = 'string'
 489                 pa('',long, action='callback', callback=cb_setah,
 490                    callback_args=(affected,effect), **kwargs)
 491
 492         #---- paths: host or testbed:
 493         #
 494         pa_setah('--paths-testbed', ['tb'],True,
 495                 help='subsequent path specifications refer to the testbed')
 496         pa_setah('--paths-host', ['tb'],False,
 497                 help='subsequent path specifications refer to the host')
 498
 499         #---- source processing settings:
 500
 501         pa_setah('--sources-tests', ['dsc_tests'],True,
 502                 help='run tests from builds of subsequent sources')
 503         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 504                 help='do not run tests from builds of subsequent sources')
 505
 506         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 507                 type='string', metavar='PATTERN-LIST',
 508                 help='from subsequent sources, use binaries matching'
 509                      ' PATTERN-LIST (comma-separated glob patterns)'
 510                      ' according to most recent --binaries-* settings')
 511         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 512                 help='from subsequent sources, do not use any binaries')
 513         #---- binary package processing settings:
 514
 515         def pa_setahbins(long,toset,how):
 516          pa_setah(long, toset,['ignore','auto','install'],
 517                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 518                 help=how+' ignore binaries, install them as needed'
 519                         ' for dependencies, or unconditionally install'
 520                         ' them, respectively')
 521         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 522         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 523         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 524
 525         #----------
 526         # general options:
 527
 528         def cb_vserv(op,optstr,value,parser):
 529                 parser.values.vserver = list(parser.rargs)
 530                 del parser.rargs[:]
 531
 532         def cb_path(op,optstr,value,parser, constructor,long,dir):
 533                 name = long.replace('-','_')
 534                 af = constructor(long, value,arghandling['tb'])
 535                 setattr(parser.values, name, af)
 536
 537         def pa_path(long, constructor, help, dir=False):
 538                 pa('','--'+long, action='callback', callback=cb_path,
 539                         callback_args=(constructor,long,dir),
 540                         nargs=1, type='string',
 541                         help=help, metavar='PATH')
 542
 543         pa_path('output-dir', OutputDir, dir=True,
 544                 help='write stderr/out files in PATH')
 545
 546         pa('--leave-lang', dest='set_lang', action='store_false',
 547                 help="leave LANG on testbed set to testbed's default")
 548         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 549                 help='set LANG on testbed to LANGVAL', default='C')
 550
 551         pa('','--tmp-dir',              type='string', dest='tmpdir',
 552                 help='write temporary files to TMPDIR, emptying it'
 553                      ' beforehand and leaving it behind at the end')
 554         pa('','--log-file',             type='string', dest='logfile',
 555                 help='write the log LOGFILE, emptying it beforehand,'
 556                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 557         pa('','--summary-file',         type='string', dest='summary',
 558                 help='write a summary report to SUMMARY,'
 559                      ' emptying it beforehand')
 560
 561         for k in timeouts.keys():
 562                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 563                         metavar='T', help='set %s timeout to T')
 564         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 565                         metavar='FACTOR', default=1.0,
 566                         help='multiply all default timeouts by FACTOR')
 567
 568         pa('','--user',                 type='string', dest='user',
 569                 help='run tests as USER (needs root on testbed)')
 570         pa('','--gain-root',            type='string', dest='gainroot',
 571                 help='prefix debian/rules binary with GAINROOT')
 572         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 573         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 574         pa('','--gnupg-home',           type='string', dest='gnupghome',
 575                 default='~/.autopkgtest/gpg',
 576                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 577                         " signing private apt archive);"
 578                         " `fresh' means generate new key each time.")
 579
 580         #----------
 581         # actual meat:
 582
 583         class SpecialOption(optparse.Option): pass
 584         vs_op = SpecialOption('','--VSERVER-DUMMY')
 585         vs_op.action = 'callback'
 586         vs_op.type = None
 587         vs_op.default = None
 588         vs_op.nargs = 0
 589         vs_op.callback = cb_vserv
 590         vs_op.callback_args = ( )
 591         vs_op.callback_kwargs = { }
 592         vs_op.help = 'introduces virtualisation server and args'
 593         vs_op._short_opts = []
 594         vs_op._long_opts = ['---']
 595
 596         pa(vs_op)
 597
 598         (opts,args) = parser.parse_args()
 599         if not hasattr(opts,'vserver'):
 600                 parser.error('you must specifiy --- <virt-server>...')
 601         if n_non_actions >= len(parser.largs):
 602                 parser.error('nothing to do specified')
 603
 604         for k in timeouts.keys():
 605                 t = getattr(opts,'timeout_'+k)
 606                 if t is None: t = timeouts[k] * opts.timeout_factor
 607                 timeouts[k] = int(t)
 608
 609         arghandling = initial_arghandling
 610         opts.actions = []
 611         ix = 0
 612         for act in args:
 613                 if type(act) == dict:
 614                         arghandling = act
 615                         continue
 616                 elif type(act) == tuple:
 617                         pass
 618                 elif type(act) == str:
 619                         act = (act,act)
 620                 else:
 621                         raise ("unknown action in list `%s' having"
 622                               " type `%s'" % (act, type(act)))
 623                 (pathstr, kindpath) = act
 624
 625                 constructor = InputFile
 626                 if type(kindpath) is tuple:             kind = kindpath[0]
 627                 elif kindpath.endswith('.deb'):         kind = 'deb'
 628                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 629                 elif kindpath.endswith('//'):
 630                         kind = 'ubtree'
 631                         constructor = InputDir
 632                 elif kindpath.endswith('/'):
 633                         kind = 'tree'
 634                         constructor = InputDir
 635                 else: parser.error("do not know how to handle filename `%s';"
 636                         " specify --source --binary or --build-tree" %
 637                         kindpath)
 638
 639                 what = '%s%s' % (kind,ix); ix += 1
 640
 641                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 642                 else: fwhatx = '-'+kind
 643
 644                 if pathstr is None: af = None
 645                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 646                 opts.actions.append(Action(kind, af, arghandling, what))
 647
 648 def setup_trace():
 649         global trace_stream, tmpdir, summary_stream
 650
 651         if opts.tmpdir is not None:
 652                 rmtree('tmpdir(specified)',opts.tmpdir)
 653                 mkdir_okexist(opts.tmpdir, 0700)
 654                 tmpdir = opts.tmpdir
 655         else:
 656                 assert(tmpdir is None)
 657                 tmpdir = tempfile.mkdtemp()
 658
 659         if opts.logfile is None:
 660                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 661                         opts.logfile = opts.output_dir.spec + '/log'
 662                 elif opts.tmpdir is not None:
 663                         opts.logfile = opts.tmpdir + '/log'
 664         if opts.logfile is not None:
 665                 trace_stream = open(opts.logfile, 'w', 0)
 666         if opts.summary is not None:
 667                 summary_stream = open(opts.summary, 'w', 0)
 668
 669         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 670
 671 def finalise_options():
 672         global opts, tb, build_essential
 673
 674         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 675                 opts.user = ''
 676
 677         if opts.user is None:
 678                 su = 'suggested-normal-user='
 679                 ul = [
 680                         e[len(su):]
 681                         for e in testbed.caps
 682                         if e.startswith(su)
 683                         ]
 684                 if ul:
 685                         opts.user = ul[0]
 686                 else:
 687                         opts.user = ''
 688
 689         if opts.user:
 690                 if 'root-on-testbed' not in testbed.caps:
 691                         pstderr("warning: virtualisation"
 692                                 " system does not offer root on testbed,"
 693                                 " but --user option specified: failure likely")
 694                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 695         else:
 696                 opts.user_wrap = lambda x: x
 697
 698         if opts.gainroot is None:
 699                 opts.gainroot = ''
 700                 if (opts.user or
 701                     'root-on-testbed' not in testbed.caps):
 702                         opts.gainroot = 'fakeroot'
 703                         build_essential += ['fakeroot']
 704
 705         if opts.gnupghome.startswith('~/'):
 706                 try: home = os.environ['HOME']
 707                 except KeyError:
 708                         parser.error("HOME environment variable"
 709                                 " not set, needed for --gnupghome=`%s"
 710                                 % opts.gnupghome)
 711                 opts.gnupghome = home + opts.gnupghome[1:]
 712         elif opts.gnupghome == 'fresh':
 713                 opts.gnupghome = None
 714
 715 #---------- testbed management - the Testbed class
 716
 717 class Testbed:
 718  def __init__(tb):
 719         tb.sp = None
 720         tb.lastsend = None
 721         tb.scratch = None
 722         tb.modified = False
 723         tb.blamed = []
 724         tb._ephemeral = []
 725         tb._debug('init')
 726         tb._need_reset_apt = False
 727  def _debug(tb, m, minlevel=0):
 728         debug('** '+m, minlevel)
 729  def start(tb):
 730         tb._debug('start')
 731         p = subprocess.PIPE
 732         debug_subprocess('vserver', opts.vserver)
 733         tb._errplumb = Errplumb(True)
 734         tb.sp = subprocess.Popen(opts.vserver,
 735                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 736         tb.expect('ok')
 737         tb.caps = tb.commandr('capabilities')
 738         if not 'print-execute-command' in tb.caps:
 739                 tb.bomb('testbed does not support print-execute-command')
 740  def stop(tb):
 741         tb._debug('stop')
 742         tb.close()
 743         if tb.sp is None: return
 744         ec = tb.sp.returncode
 745         if ec is None:
 746                 tb.sp.stdout.close()
 747                 tb.send('quit')
 748                 tb.sp.stdin.close()
 749                 ec = tb.sp.wait()
 750         if ec:
 751                 tb.bomb('testbed gave exit status %d after quit' % ec)
 752         tb._errplumb.wait()
 753  def open(tb):
 754         tb._debug('open, scratch=%s' % tb.scratch)
 755         if tb.scratch is not None: return
 756         pl = tb.commandr('open')
 757         tb.scratch = InputDir('tb-scratch', pl[0], True)
 758         tb.deps_processed = []
 759         tb._auxverbscript_make()
 760
 761  def _auxverbscript_make(tb):
 762         pec = tb.commandr('print-execute-command')
 763         if len(pec) < 2: tb.bomb('too few results from print-execute-command')
 764         tb.ec_cmdl = map(urllib.unquote, pec[0].split(','))
 765         tb.ec_mode = urllib.unquote(pec[1])
 766         tb.ec_infos = map(urllib.unquote,pec[2:])
 767
 768         shellquote_re = regexp.compile(r'([\\"$`])')
 769         def shellquote_arg(s): return '"' + shellquote_re.sub(r'\\\1', s) + '"'
 770         def shellquote_cmdl(l): return ' '.join(map(shellquote_arg,l))
 771
 772         tb._debug('tb.ec_cmdl = %s' % (`tb.ec_cmdl`))
 773
 774         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 775         print >>open(tb.ec_auxverbscript.write(),'w'), (
 776 '''#!/bin/sh
 777 set -ex
 778 echo >&2 ": $*"
 779 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 780         set -- dpkg --print-architecture
 781 fi
 782 if [ "x$1" = xsh ] && [ "x$2" = x-c ]; then
 783         shift; shift
 784         # what a horrible hack!
 785 fi
 786 exec '''+shellquote_cmdl(tb.ec_cmdl)+' "$*"'+"\n"
 787                 )
 788         os.chmod(tb.ec_auxverbscript.write(), 0755)
 789
 790  def mungeing_apt(tb):
 791         if not 'revert' in tb.caps:
 792                 tb._need_reset_apt = True
 793  def reset_apt(tb):
 794         if not tb._need_reset_apt: return
 795         what = 'aptget-update-reset'
 796         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 797         rc = tb.execute(what, cmdl, kind='install')
 798         if rc:
 799                 pstderr("\n" "warning: failed to restore"
 800                         " testbed apt cache, exit code %d" % rc)
 801         tb._need_reset_apt = False
 802  def close(tb):
 803         tb._debug('close, scratch=%s' % tb.scratch)
 804         if tb.scratch is None: return
 805         tb.scratch = None
 806         if tb.sp is None: return
 807         tb.command('close')
 808  def prepare1(tb, deps_new):
 809         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 810                 (tb.modified, tb.deps_processed, deps_new), 1)
 811         if 'revert' in tb.caps and (tb.modified or
 812             [d for d in tb.deps_processed if d not in deps_new]):
 813                 for af in tb._ephemeral: af.read(False)
 814                 tb._debug('reset **')
 815                 tb.command('revert')
 816                 tb.blamed = []
 817                 for af in tb._ephemeral: af.invalidate(True)
 818                 tb._auxverbscript_make()
 819         tb.modified = False
 820  def prepare2(tb, deps_new):
 821         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 822         binaries.publish()
 823         tb._install_deps(deps_new)
 824  def prepare(tb, deps_new):
 825         tb.prepare1(deps_new)
 826         tb.prepare2(deps_new)
 827  def register_ephemeral(tb, af):
 828         tb._ephemeral.append(af)
 829  def _install_deps(tb, deps_new):
 830         tb._debug(' installing dependencies '+`deps_new`, 1)
 831         tb.deps_processed = deps_new
 832         if not deps_new: return
 833         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 834  def needs_reset(tb):
 835         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 836         tb.modified = True
 837  def blame(tb, m):
 838         tb._debug('blame += %s' % m, 1)
 839         tb.blamed.append(m)
 840  def bomb(tb, m):
 841         tb._debug('bomb %s' % m)
 842         if tb.sp is not None:
 843                 tb.sp.stdout.close()
 844                 tb.sp.stdin.close()
 845                 ec = tb.sp.wait()
 846                 if ec: pstderr('adt-run: testbed failing,'
 847                                 ' exit status %d' % ec)
 848         tb.sp = None
 849         raise Quit(16, 'testbed failed: %s' % m)
 850  def send(tb, string):
 851         tb.sp.stdin
 852         try:
 853                 debug('>> '+string, 2)
 854                 print >>tb.sp.stdin, string
 855                 tb.sp.stdin.flush()
 856                 tb.lastsend = string
 857         except:
 858                 (type, value, dummy) = sys.exc_info()
 859                 tb.bomb('cannot send to testbed: %s' % traceback.
 860                         format_exception_only(type, value))
 861  def expect(tb, keyword, nresults=None):
 862         l = tb.sp.stdout.readline()
 863         if not l: tb.bomb('unexpected eof from the testbed')
 864         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 865         l = l.rstrip('\n')
 866         debug('<< '+l, 2)
 867         ll = l.split()
 868         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 869         if ll[0] != keyword:
 870                 if tb.lastsend is None:
 871                         tb.bomb("got banner `%s', expected `%s...'" %
 872                                 (l, keyword))
 873                 else:
 874                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 875                                 (tb.lastsend, l, keyword))
 876         ll = ll[1:]
 877         if nresults is not None and len(ll) != nresults:
 878                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 879                         " expected %d result parameters" %
 880                         (string, l, len(ll), nresults))
 881         return ll
 882  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 883         # pass args=[None,...] or =(None,...) to avoid more url quoting
 884         if type(cmd) is str: cmd = [cmd]
 885         if len(args) and args[0] is None: args = args[1:]
 886         else: args = map(urllib.quote, args)
 887         al = cmd + args
 888         tb.send(string.join(al))
 889         ll = tb.expect('ok', nresults)
 890         if unquote: ll = map(urllib.unquote, ll)
 891         return ll
 892  def command(tb, cmd, args=()):
 893         tb.commandr(cmd, args, 0)
 894  def commandr1(tb, cmd, args=()):
 895         rl = tb.commandr(cmd, args, 1)
 896         return rl[0]
 897  def execute(tb, what, cmdl,
 898                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 899                 script=False, tmpdir=None, kind='short'):
 900         # Options for script:
 901         #   False - do not call debug_subprocess, no synch. reporting required
 902         #   None or string - call debug_subprocess with that value,
 903         #                       plumb stderr through synchronously if possible
 904         # Options for se:
 905         #   None - usual Errplumb (output is of kind 2c)
 906         #   string - path on testbed (output is of kind 2d)
 907
 908         timeout = timeouts[kind]
 909
 910         if script is not False: debug_subprocess(what, cmdl, script=script)
 911         if cwd is None: cwd = tb.scratch.write(True)
 912
 913         xdump = None
 914         if se is None:
 915                 ep = Errplumb()
 916                 se_catch = TemporaryFile(what+'-xerr')
 917                 se_use = se_catch.write(True)
 918                 if not opts.quiet: xdump = 'debug=2-2'
 919                 elif trace_stream is not None:
 920                         xdump = 'debug=2-%d' % trace_stream.fileno()
 921         else:
 922                 ep = None
 923                 se_catch = None
 924                 se_use = se
 925
 926         cmdl = [None,
 927                 ','.join(map(urllib.quote, cmdl)),
 928                 si, so, se_use, cwd]
 929
 930         if timeout is not None and timeout > 0:
 931                 cmdl.append('timeout=%d' % timeout)
 932
 933         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 934         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 935         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 936         if opts.set_lang is not False:
 937                 cmdl.append('env=LANG=%s' % opts.set_lang)
 938
 939         rc = tb.commandr1('execute', cmdl)
 940         try: rc = int(rc)
 941         except ValueError: bomb("execute for %s gave invalid response `%s'"
 942                                         % (what,rc))
 943
 944         if se_catch is not None:
 945                 debug_file(se_catch.read())
 946         if ep is not None:
 947                 ep.wait()
 948
 949         return rc
 950
 951  def satisfy_dependencies_string(tb, deps, what):
 952         # Must have called Binaries.configure_apt
 953         debug('dependencies: %s: satisfying %s' % (what,deps))
 954         dsc = TemporaryFile('deps.dsc')
 955         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 956         # pbuilder-satisfydepends has a bug where it ignores the
 957         #  Build-Depends if it's the last line in the dsc
 958         tb.satisfy_dependencies_dsc(dsc, what)
 959
 960  def satisfy_dependencies_dsc(tb, dsc, what):
 961         # Must have called Binaries.configure_apt
 962         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 963                 '--binary-all', # --check-key
 964                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 965                 '-c',dsc.read()
 966         ]
 967         debug('dependencies: %s: running %s' % (what,`cmdl`))
 968         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 969         if rc: badpkg('dependency install failed, exit code %d' % rc)
 970
 971 #---------- representation of test control files: Field*, Test, etc.
 972
 973 class FieldBase:
 974  def __init__(f, fname, stz, base, tnames, vl):
 975         assert(vl)
 976         f.stz = stz
 977         f.base = base
 978         f.tnames = tnames
 979         f.vl = vl
 980  def words(f):
 981         def distribute(vle):
 982                 (lno, v) = vle
 983                 r = v.split()
 984                 r = map((lambda w: (lno, w)), r)
 985                 return r
 986         return flatten(map(distribute, f.vl))
 987  def atmostone(f):
 988         if len(vl) == 1:
 989                 (f.lno, f.v) = vl[0]
 990         else:
 991                 raise Unsupported(f.vl[1][0],
 992                         'only one %s field allowed' % fn)
 993         return f.v
 994
 995 class FieldIgnore(FieldBase):
 996  def parse(f): pass
 997
 998 class Restriction:
 999  def __init__(r,rname,base): pass
1000
1001 class Restriction_rw_build_tree(Restriction): pass
1002 class Restriction_breaks_testbed(Restriction):
1003  def __init__(r, rname, base):
1004         if 'revert' not in testbed.caps:
1005                 raise Unsupported(f.lno,
1006                         'Test breaks testbed but testbed cannot revert')
1007 class Restriction_needs_root(Restriction):
1008  def __init__(r, rname, base):
1009         if 'root-on-testbed' not in testbed.caps:
1010                 raise Unsupported(f.lno,
1011                         'Test needs root on testbed which is not available')
1012
1013 class Field_Restrictions(FieldBase):
1014  def parse(f):
1015         for wle in f.words():
1016                 (lno, rname) = wle
1017                 nrname = rname.replace('-','_')
1018                 try: rclass = globals()['Restriction_'+nrname]
1019                 except KeyError: raise Unsupported(lno,
1020                         'unknown restriction %s' % rname)
1021                 r = rclass(nrname, f.base)
1022                 f.base['restriction_names'].append(rname)
1023                 f.base['restrictions'].append(r)
1024
1025 class Field_Features(FieldIgnore):
1026  def parse(f):
1027         for wle in f.words():
1028                 (lno, fname) = wle
1029                 f.base['feature_names'].append(fname)
1030                 nfname = fname.replace('-','_')
1031                 try: fclass = globals()['Feature_'+nfname]
1032                 except KeyError: continue
1033                 ft = fclass(nfname, f.base)
1034                 f.base['features'].append(ft)
1035
1036 class Field_Tests(FieldIgnore): pass
1037
1038 class Field_Depends(FieldBase):
1039  def parse(f):
1040         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
1041         dl = map(lambda x: x.strip(),
1042                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1043         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1044         for d in dl:
1045                 if re.search(d):
1046                         badpkg("Test Depends field contains dependency"
1047                                " `%s' with invalid characters" % d)
1048         f.base['depends'] = dl
1049
1050 class Field_Tests_directory(FieldBase):
1051  def parse(f):
1052         td = atmostone(f)
1053         if td.startswith('/'): raise Unspported(f.lno,
1054                 'Tests-Directory may not be absolute')
1055         f.base['testsdir'] = td
1056
1057 def run_tests(stanzas, tree):
1058         global errorcode, testbed
1059         if stanzas == ():
1060                 report('*', 'SKIP no tests in this package')
1061                 errorcode |= 8
1062         for stanza in stanzas:
1063                 tests = stanza[' tests']
1064                 if not tests:
1065                         report('*', 'SKIP package has metadata but no tests')
1066                         errorcode |= 8
1067                 for t in tests:
1068                         t.prepare()
1069                         t.run(tree)
1070                         if 'breaks-testbed' in t.restriction_names:
1071                                 testbed.needs_reset()
1072                 testbed.needs_reset()
1073
1074 class Test:
1075  def __init__(t, tname, base, act):
1076         if '/' in tname: raise Unsupported(base[' lno'],
1077                 'test name may not contain / character')
1078         for k in base: setattr(t,k,base[k])
1079         t.tname = tname
1080         t.act = act
1081         t.what = act.what+'t-'+tname
1082         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1083         else: t.path = tname
1084         t._debug('constructed; path=%s' % t.path)
1085         t._debug(' .depends=%s' % t.depends)
1086  def _debug(t, m):
1087         debug('& %s: %s' % (t.what, m))
1088  def report(t, m):
1089         report(t.what, m)
1090  def reportfail(t, m):
1091         global errorcode
1092         errorcode |= 4
1093         report(t.what, 'FAIL ' + m)
1094  def prepare(t):
1095         t._debug('preparing')
1096         dn = []
1097         for d in t.depends:
1098                 t._debug(' processing dependency '+d)
1099                 if not '@' in d:
1100                         t._debug('  literal dependency '+d)
1101                         dn.append(d)
1102                 else:
1103                         for (pkg,bin) in t.act.binaries:
1104                                 d = d.replace('@',pkg)
1105                                 t._debug('  synthesised dependency '+d)
1106                                 dn.append(d)
1107         testbed.prepare(dn)
1108  def run(t, tree):
1109         t._debug('[----------------------------------------')
1110         def stdouterr(oe):
1111                 idstr = t.what + '-' + oe
1112                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1113                         use_dir = opts.output_dir
1114                 else:
1115                         use_dir = testbed.scratch
1116                 return RelativeOutputFile(idstr, use_dir, idstr)
1117
1118         if hasattr(t.act,'work'): t.act.work.write(True)
1119         tree.read(True)
1120
1121         af = RelativeInputFile(t.what, tree, t.path)
1122         so = stdouterr('stdout')
1123         se = stdouterr('stderr')
1124
1125         tf = af.read(True)
1126         tmpdir = None
1127
1128         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1129         if rc: bomb('failed to chmod +x %s' % tf)
1130
1131         if 'needs-root' not in t.restriction_names and opts.user is not None:
1132                 tfl = ['su',opts.user,'-c',tf]
1133                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1134                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1135                 if opts.user: script += '; chown %s "$1"' % opts.user
1136                 if 'rw-build-tree' in t.restriction_names:
1137                         script += '; chown -R %s "$2"' % opts.user
1138                 rc = testbed.execute('mktmpdir-'+t.what,
1139                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1140                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1141                                 % (tmpdir, rc))
1142         else:
1143                 tfl = [tf]
1144
1145         rc = testbed.execute('test-'+t.what, tfl,
1146                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1147                 tmpdir=tmpdir, kind='test')
1148
1149         so_read = so.read()
1150         se_read = se.read()
1151
1152         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1153         stab = os.stat(se_read)
1154         if stab.st_size != 0:
1155                 l = open(se_read).readline()
1156                 l = l.rstrip('\n \t\r')
1157                 if len(l) > 35: l = l[:35] + '...'
1158                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1159                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1160                 debug_file(se_read)
1161         elif rc != 0:
1162                 t.reportfail('non-zero exit status %d' % rc)
1163         else:
1164                 t.report('PASS')
1165
1166         stab = os.stat(so_read)
1167         if stab.st_size != 0:
1168                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1169                 debug_file(so_read)
1170
1171         t._debug('----------------------------------------]')
1172
1173 def read_control(act, tree, control_override):
1174         stanzas = [ ]
1175
1176         if control_override is not None:
1177                 control_af = control_override
1178                 testbed.blame('arg:'+control_override.spec)
1179         else:
1180                 if act.missing_tests_control:
1181                         return ()
1182                 control_af = RelativeInputFile(act.what+'-testcontrol',
1183                         tree, 'debian/tests/control')
1184         try:
1185                 control = open(control_af.read(), 'r')
1186         except (IOError,OSError), oe:
1187                 if oe[0] != errno.ENOENT: raise
1188                 return []
1189
1190         lno = 0
1191         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1192         stz = None      # stz[field_name][index] = (lno, value)
1193                         # special field names:
1194                         # stz[' lno'] = number
1195                         # stz[' tests'] = list of Test objects
1196         def end_stanza(stz):
1197                 if stz is None: return
1198                 stz[' errs'] = 0
1199                 stanzas.append(stz)
1200                 stz = None
1201                 hcurrent = None
1202
1203         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1204         while 1:
1205                 l = control.readline()
1206                 if not l: break
1207                 lno += 1
1208                 if not l.endswith('\n'): badctrl('unterminated line')
1209                 if regexp.compile('\s*\#').match(l): continue
1210                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1211                 initmat = initre.match(l)
1212                 if initmat:
1213                         (fname, l) = initmat.groups()
1214                         fname = string.capwords(fname)
1215                         if stz is None:
1216                                 stz = { ' lno': lno, ' tests': [] }
1217                         if not stz.has_key(fname): stz[fname] = [ ]
1218                         hcurrent = stz[fname]
1219                 elif regexp.compile('\s').match(l):
1220                         if not hcurrent: badctrl('unexpected continuation')
1221                 else:
1222                         badctrl('syntax error')
1223                 hcurrent.append((lno, l))
1224         end_stanza(stz)
1225
1226         def testbadctrl(stz, lno, m):
1227                 report_badctrl(lno, m)
1228                 stz[' errs'] += 1
1229
1230         for stz in stanzas:
1231                 try:
1232                         try: tnames = stz['Tests']
1233                         except KeyError:
1234                                 tnames = ['*']
1235                                 raise Unsupported(stz[' lno'],
1236                                         'no Tests field')
1237                         tnames = map((lambda lt: lt[1]), tnames)
1238                         tnames = string.join(tnames).split()
1239                         base = {
1240                                 'restriction_names': [],
1241                                 'restrictions': [],
1242                                 'feature_names': [],
1243                                 'features': [],
1244                                 'testsdir': 'debian/tests',
1245                                 'depends' : '@'
1246                         }
1247                         for fname in stz.keys():
1248                                 if fname.startswith(' '): continue
1249                                 vl = stz[fname]
1250                                 try: fclass = globals()['Field_'+
1251                                         fname.replace('-','_')]
1252                                 except KeyError: raise Unsupported(vl[0][0],
1253                                         'unknown metadata field %s' % fname)
1254                                 f = fclass(stz, fname, base, tnames, vl)
1255                                 f.parse()
1256                         for tname in tnames:
1257                                 t = Test(tname, base, act)
1258                                 stz[' tests'].append(t)
1259                 except Unsupported, u:
1260                         for tname in tnames: u.report(tname)
1261                         continue
1262
1263         return stanzas
1264
1265 def print_exception(ei, msgprefix=''):
1266         if msgprefix: pstderr(msgprefix)
1267         (et, q, tb) = ei
1268         if et is Quit:
1269                 pstderr('adt-run: ' + q.m)
1270                 psummary('quitting: '+q.m)
1271                 return q.ec
1272         else:
1273                 pstderr("adt-run: unexpected, exceptional, error:")
1274                 psummary('quitting: unexpected error, consult transcript')
1275                 traceback.print_exc(None, sys.stderr)
1276                 if trace_stream is not None:
1277                         traceback.print_exc(None, trace_stream)
1278                 return 20
1279
1280 def cleanup():
1281         global trace_stream
1282         try:
1283                 if testbed is not None:
1284                         testbed.reset_apt()
1285                         testbed.stop()
1286                 if opts.tmpdir is None and tmpdir is not None:
1287                         rmtree('tmpdir', tmpdir)
1288                 if trace_stream is not None:
1289                         trace_stream.close()
1290                         trace_stream = None
1291         except:
1292                 print_exception(sys.exc_info(),
1293                         '\nadt-run: error cleaning up:\n')
1294                 os._exit(20)
1295
1296 #---------- registration, installation etc. of .deb's: Binaries
1297
1298 def determine_package(act):
1299         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1300         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1301         if rc: badpkg('failed to parse binary package, code %d' % rc)
1302         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1303         act.pkg = None
1304         for l in output.split('\n'):
1305                 m = re.match(l)
1306                 if not m: continue
1307                 if act.pkg: badpkg('two Package: lines in control file')
1308                 act.pkg = m.groups()[0]
1309         if not act.pkg: badpkg('no good Package: line in control file')
1310
1311 class Binaries:
1312  def __init__(b):
1313         b.dir = TemporaryDir('binaries')
1314         b.dir.write()
1315         ok = False
1316
1317         if opts.gnupghome is None:
1318                 opts.gnupghome = tmpdir+'/gnupg'
1319
1320         b._debug('initialising')
1321         try:
1322                 for x in ['pubring','secring']:
1323                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1324                 ok = True
1325         except (IOError,OSError), oe:
1326                 if oe.errno != errno.ENOENT: raise
1327
1328         if ok: b._debug('no key generation needed')
1329         else: b.genkey()
1330
1331  def _debug(b, s):
1332         debug('* '+s)
1333
1334  def genkey(b):
1335         b._debug('preparing for key generation')
1336
1337         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1338         mkdir_okexist(opts.gnupghome, 0700)
1339
1340         script = '''
1341   exec >&2
1342   cd "$1"
1343   cat <<"END" >key-gen-params
1344 Key-Type: DSA
1345 Key-Length: 1024
1346 Key-Usage: sign
1347 Name-Real: autopkgtest per-run key
1348 Name-Comment: do not trust this key
1349 Name-Email: autopkgtest@example.com
1350 END
1351   set -x
1352   gpg --homedir="$1" --batch --gen-key key-gen-params
1353                 '''
1354         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1355         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1356         if rc: bomb('key generation failed, code %d' % rc)
1357
1358  def apt_configs(b):
1359         return {
1360                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1361                 "Debug::pkgProblemResolver": "true",
1362         }
1363
1364  def _configure_apt(b, tb):
1365         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1366                         True)
1367         f = open(config.write(),'w')
1368         for (k,v) in b.apt_configs().iteritems():
1369                 print >>f, '%s { "%s"; };' % (k, v)
1370         f.close()
1371
1372  def _apt_get(b):
1373         ag = ['apt-get','-qy']
1374         for kv in b.apt_configs().iteritems():
1375                 ag += ['-o', '%s=%s' % kv]
1376         return ' '.join(ag)
1377
1378  def reset(b):
1379         b._debug('reset')
1380         rmtree('binaries', b.dir.read())
1381         b.dir.invalidate()
1382         b.dir.write()
1383         b.install = []
1384         b.blamed = []
1385         b.registered = set()
1386
1387  def register(b, act, pkg, af, forwhat, blamed):
1388         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1389                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1390
1391         if act.ah['deb_'+forwhat] == 'ignore': return
1392
1393         b.blamed += testbed.blamed
1394
1395         leafname = pkg+'.deb'
1396         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1397
1398         try: os.remove(dest.write())
1399         except (IOError,OSError), oe:
1400                 if oe.errno != errno.ENOENT: raise e
1401
1402         try: os.link(af.read(), dest.write())
1403         except (IOError,OSError), oe:
1404                 if oe.errno != errno.EXDEV: raise e
1405                 shutil.copy(af.read(), dest)
1406
1407         if act.ah['deb_'+forwhat] == 'install':
1408                 b.install.append(pkg)
1409
1410         b.registered.add(pkg)
1411
1412  def publish(b):
1413         b._debug('publish')
1414
1415         b._configure_apt(testbed)
1416
1417         script = '''
1418   exec >&2
1419   cd "$1"
1420   apt-ftparchive packages . >Packages
1421   gzip <Packages >Packages.gz
1422   apt-ftparchive release . >Release
1423   rm -f Release.gpg
1424   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1425   gpg --homedir="$2" --batch --export >archive-key.pgp
1426                 '''
1427         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1428         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1429         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1430
1431         b.dir.invalidate(True)
1432         apt_source = b.dir.read(True)
1433
1434         so = TemporaryFile('vlds')
1435         script = '''
1436   exec 3>&1 >&2
1437   apt-key add archive-key.pgp
1438   echo "deb file://'''+apt_source+''' /" >sources.list
1439   cat /etc/apt/sources.list >>sources.list
1440   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1441     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1442   fi
1443   '''+ b._apt_get() +''' update >&2
1444   cat /var/lib/dpkg/status >&3
1445                 '''
1446         testbed.mungeing_apt()
1447         rc = testbed.execute('apt-key', ['sh','-ec',script],
1448                                 so=so.write(True), cwd=b.dir.write(True),
1449                                 script=script, kind='install')
1450         if rc: bomb('apt setup failed with exit code %d' % rc)
1451
1452         testbed.blamed += b.blamed
1453
1454         b._debug('publish reinstall checking...')
1455         pkgs_reinstall = set()
1456         pkg = None
1457         for l in open(so.read()):
1458                 if l.startswith('Package: '):
1459                         pkg = l[9:].rstrip()
1460                 elif l.startswith('Status: install '):
1461                         if pkg in b.registered:
1462                                 pkgs_reinstall.add(pkg)
1463                                 b._debug(' publish reinstall needs '+pkg)
1464
1465         if pkgs_reinstall:
1466                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1467                 what = 'apt-get-reinstall'
1468                 cmdl = (b._apt_get() + ' --reinstall install '+
1469                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1470                 cmdl = ['sh','-c',cmdl]
1471                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1472                 if rc: badpkg("installation of basic binarries failed,"
1473                                 " exit code %d" % rc)
1474
1475         b._debug('publish install...')
1476         for pkg in b.install:
1477                 what = 'apt-get-install-%s' % pkg
1478                 testbed.blame(pkg)
1479                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1480                 cmdl = ['sh','-c',cmdl]
1481                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1482                 if rc: badpkg("installation of %s failed, exit code %d"
1483                                 % (pkg, rc))
1484
1485         b._debug('publish done')
1486
1487 #---------- processing of sources (building)
1488
1489 def source_rules_command(act,script,what,which,work,cwd,
1490                                 results_lines=0,xargs=[]):
1491         script = [      "exec 3>&1 >&2",
1492                         "set -x"        ] + script
1493         script = '\n'.join(script)
1494         so = TemporaryFile('%s-%s-results' % (what,which))
1495         rc = testbed.execute('%s-%s' % (what,which),
1496                         ['sh','-ec',script]+xargs, script=script,
1497                         so=so.write(True), cwd=cwd, kind='build')
1498         results = open(so.read()).read().rstrip('\n')
1499         if len(results): results = results.split("\n")
1500         else: results = []
1501         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1502         if results_lines is not None and len(results) != results_lines:
1503                 badpkg("got %d lines of results from %s where %d expected"
1504                         % (len(results), which, results_lines))
1505         if results_lines==1: return results[0]
1506         return results
1507
1508 def build_source(act, control_override):
1509         act.blame = 'arg:'+act.af.spec
1510         testbed.blame(act.blame)
1511         testbed.prepare1([])
1512         testbed.needs_reset()
1513
1514         what = act.what
1515         basename = act.af.spec
1516         debiancontrol = None
1517         act.binaries = []
1518
1519         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1520
1521         if act.kind == 'dsc':
1522                 dsc = act.af
1523                 dsc_file = open(dsc.read())
1524                 in_files = False
1525                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1526                 for l in dsc_file:
1527                         l = l.rstrip('\n')
1528                         if l.startswith('Files:'): in_files = True; continue
1529                         elif l.startswith('#'): pass
1530                         elif not l.startswith(' '):
1531                                 in_files = False
1532                                 if l.startswith('Source:'):
1533                                         act.blame = 'dsc:'+l[7:].strip()
1534                                         testbed.blame(act.blame)
1535                         if not in_files: continue
1536
1537                         m = fre.match(l)
1538                         if not m: badpkg(".dsc contains unparseable line"
1539                                         " in Files: `%s'" % l)
1540                         leaf = m.groups(0)[0]
1541                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1542                                         sibling=True)
1543                         subfile.read(True)
1544                 dsc.read(True)
1545
1546         if act.kind == 'ubtree':
1547                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1548                         act.af, 'debian/control')
1549                 dsc = TemporaryFile(what+'-fakedsc')
1550                 dsc_w = open(dsc.write(), 'w')
1551                 for l in open(debiancontrol.read()):
1552                         l = l.rstrip('\n')
1553                         if not len(l): break
1554                         print >>dsc_w, l
1555                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1556                 dsc_w.close()
1557
1558         if act.kind == 'dsc':
1559                 testbed.prepare2([])
1560                 testbed.satisfy_dependencies_string('dpkg-dev',
1561                                                 'install dpkg-dev')
1562
1563         work = TemporaryDir(what+'-build')
1564         act.work = work
1565
1566         tmpdir = work.write(True)+'/tmpdir'
1567         tmpdir_script = [
1568                         'TMPDIR="$1"',
1569                         'rm -rf -- "$TMPDIR"',
1570                         'export TMPDIR',
1571                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1572                 ]
1573
1574         if act.kind == 'ubtree':
1575                 spec = '%s/real-tree' % work.write(True)
1576                 create_command = '''
1577                         rm -rf "$spec"
1578                         mkdir "$spec"
1579                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1580                         '''
1581                 initcwd = act.af.read(True)
1582
1583         if act.kind == 'dsc':
1584                 spec = dsc.read(True)
1585                 create_command = '''
1586                         dpkg-source -x $spec
1587                         '''
1588                 initcwd = work.write(True)
1589
1590         script = [
1591                 'spec="$2"',
1592                 'origpwd=`pwd`',
1593                 'cd '+work.write(True)
1594         ]
1595
1596         if opts.user:
1597                 script += ([ 'chown '+opts.user+' .' ] +
1598                         tmpdir_script +
1599                         [ 'spec="$spec" origpwd="$origpwd" '
1600                                 +opts.user_wrap(create_command) ])
1601         else:
1602                 script += (tmpdir_script +
1603                         [ create_command ])
1604
1605         script += [
1606                         'cd [a-z0-9]*-*/.',
1607                         'pwd >&3',
1608                         'set +e; test -f debian/tests/control; echo $? >&3'
1609                 ]
1610         (result_pwd, control_test_rc) = source_rules_command(
1611                         act,script,what,'extract',work,
1612                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1613
1614         filter = act.ah['dsc_filter']
1615
1616         if control_test_rc == '1': act.missing_tests_control = True
1617
1618         # For optional builds:
1619         #
1620         # We might need to build the package because:
1621         #   - we want its binaries (filter isn't _ and at least one of the
1622         #       deb_... isn't ignore)
1623         #   - the test control file says so
1624         #       (assuming we have any tests)
1625
1626         class NeedBuildException: pass
1627         def build_needed(m):
1628                 debug_b('build needed for %s' % m)
1629                 raise NeedBuildException()
1630
1631         try:
1632                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1633                                       act.ah['deb_fortests'] != 'ignore'):
1634                         build_needed('binaries')
1635
1636                 result_pwd_af = InputDir(what+'-treeforcontrol',
1637                         result_pwd, True)
1638                 stanzas = read_control(act, result_pwd_af, control_override)
1639                 for stanza in stanzas:
1640                         for t in stanza[' tests']:
1641                                 if 'no-build-needed' not in t.feature_names:
1642                                         build_needed('test %s' % t.tname)
1643                                 for d in t.depends:
1644                                         if '@' in d:
1645                                                 build_needed('test %s '
1646                                                  'dependency %s' % (t.tname,d))
1647
1648                 debug_b('build not needed')
1649                 built = False
1650
1651         except NeedBuildException:
1652
1653                 if act.kind != 'dsc':
1654                         testbed.prepare2([])
1655
1656                 testbed.satisfy_dependencies_string('build-essential',
1657                                 'install build-essential')
1658                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1659
1660                 script = tmpdir_script + [
1661                         'cd "$2"',
1662                         'dpkg-checkbuilddeps',
1663                         opts.user_wrap('debian/rules build'),
1664                         ]
1665                 source_rules_command(act,script,what,'build',work,
1666                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1667
1668                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1669                         badpkg("results dir `%s' is not in expected parent"
1670                                " dir `%s'" % (result_pwd, work.read(True)))
1671
1672                 built = True
1673
1674         act.tests_tree = InputDir(what+'-tests-tree',
1675                                 work.read(True)+os.path.basename(result_pwd),
1676                                 True)
1677         if act.ah['dsc_tests']:
1678                 testbed.register_ephemeral(act.work)
1679                 testbed.register_ephemeral(act.tests_tree)
1680
1681         if not built:
1682                 act.blamed = []
1683                 return
1684
1685         act.blamed = copy.copy(testbed.blamed)
1686
1687         debug_b('filter=%s' % filter)
1688         if filter != '_':
1689                 script = tmpdir_script + [
1690                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1691                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1692                         'cd ..',
1693                         'echo *.deb >&3',
1694                         ]
1695                 result_debs = source_rules_command(act,script,what,
1696                                 'binary',work,work.write(True),
1697                                 results_lines=1, xargs=['x',tmpdir])
1698                 if result_debs == '*.deb': debs = []
1699                 else: debs = result_debs.split(' ')
1700                 debug_b('debs='+`debs`)
1701                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1702                 for deb in debs:
1703                         m = re.match(deb)
1704                         if not m: badpkg("badly-named binary `%s'" % deb)
1705                         pkg = m.groups()[0]
1706                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1707                         for pat in filter.split(','):
1708                                 debug_b('  pat=%s' % pat)
1709                                 if not fnmatch.fnmatchcase(pkg,pat):
1710                                         debug_b('   no match')
1711                                         continue
1712                                 deb_what = pkg+'_'+what+'.deb'
1713                                 bin = RelativeInputFile(deb_what,work,deb,True)
1714                                 debug_b('  deb_what=%s, bin=%s' %
1715                                         (deb_what, str(bin)))
1716                                 binaries.register(act,pkg,bin,
1717                                         'forbuilds',testbed.blamed)
1718                                 act.binaries.append((pkg,bin))
1719                                 break
1720                 debug_b('all done.')
1721
1722 #---------- main processing loop and main program
1723
1724 def process_actions():
1725         global binaries
1726
1727         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1728         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1729         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1730
1731         debug_a1('starting')
1732         testbed.open()
1733         binaries = Binaries()
1734
1735         for act in opts.actions:
1736                 if act.af is not None and not act.af.spec_tbp:
1737                         testbed.register_ephemeral(act.af)
1738
1739         binaries.reset()
1740         control_override = None
1741
1742         debug_a1('builds ...')
1743         for act in opts.actions:
1744                 debug_a2('%s %s' %
1745                         (act.kind, act.what))
1746
1747                 if act.kind == 'control':
1748                         control_override = act.af
1749                 if act.kind == 'deb':
1750                         testbed.blame('arg:'+act.af.spec)
1751                         determine_package(act)
1752                         testbed.blame('deb:'+act.pkg)
1753                         binaries.register(act,act.pkg,act.af,
1754                                 'forbuilds',testbed.blamed)
1755                 if act.kind == 'dsc' or act.kind == 'ubtree':
1756                         build_source(act, control_override)
1757                 if act.kind == 'tree':
1758                         act.binaries = []
1759                 if act.kind.endswith('tree') or act.kind == 'dsc':
1760                         control_override = None
1761                 if act.kind == 'instantiate':
1762                         pass
1763
1764         debug_a1('builds done.')
1765
1766         binaries.reset()
1767         control_override = None
1768
1769         debug_a1('tests ...')
1770         for act in opts.actions:
1771                 debug_a2('test %s %s' % (act.kind, act.what))
1772
1773                 testbed.needs_reset()
1774                 if act.kind == 'control':
1775                         control_override = act.af
1776                 if act.kind == 'deb':
1777                         binaries.register(act,act.pkg,act.af,'fortests',
1778                                 ['deb:'+act.pkg])
1779                 if act.kind == 'dsc' or act.kind == 'ubtree':
1780                         for (pkg,bin) in act.binaries:
1781                                 binaries.register(act,pkg,bin,'fortests',
1782                                         act.blamed)
1783                 if act.kind == 'dsc':
1784                         if act.ah['dsc_tests']:
1785                                 debug_a3('read control ...')
1786                                 stanzas = read_control(act, act.tests_tree,
1787                                                 control_override)
1788                                 testbed.blamed += act.blamed
1789                                 debug_a3('run_tests ...')
1790                                 run_tests(stanzas, act.tests_tree)
1791                         control_override = None
1792                 if act.kind == 'tree' or act.kind == 'ubtree':
1793                         testbed.blame('arg:'+act.af.spec)
1794                         stanzas = read_control(act, act.af, control_override)
1795                         debug_a3('run_tests ...')
1796                         run_tests(stanzas, act.af)
1797                         control_override = None
1798                 if act.kind == 'instantiate':
1799                         testbed.prepare([])
1800         debug_a1('tests done.')
1801
1802 def main():
1803         global testbed
1804         global tmpdir
1805         try:
1806                 parse_args()
1807         except SystemExit, se:
1808                 os._exit(20)
1809         try:
1810                 setup_trace()
1811                 testbed = Testbed()
1812                 testbed.start()
1813                 finalise_options()
1814                 process_actions()
1815         except:
1816                 ec = print_exception(sys.exc_info(), '')
1817                 cleanup()
1818                 os._exit(ec)
1819         cleanup()
1820         os._exit(errorcode)
1821
1822 main()