runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         try: shutil.rmtree(pathname)
 192         except (IOError,OSError), oe:
 193                 if oe.errno != errno.EEXIST: raise
 194
 195 def debug_subprocess(what, cmdl=None, script=None):
 196         o = '$ '+what+':'
 197         if cmdl is not None:
 198                 ol = []
 199                 for x in cmdl:
 200                         if x is script: x = '<SCRIPT>'
 201                         ol.append(x.    replace('\\','\\\\').
 202                                         replace(' ','\\ ')      )
 203                 o += ' '+ ' '.join(ol)
 204         debug(o)
 205         if script is not None and opts.debuglevel >= 1:
 206                 o = ''
 207                 for l in script.rstrip('\n').split('\n'):
 208                         o += '$     '+l+'\n'
 209                 debug(o,1)
 210
 211 def flatten(l):
 212         return reduce((lambda a,b: a + b), l, [])
 213
 214 #---------- fancy automatic file-copying class
 215
 216 class AutoFile:
 217         # p.what
 218         # p.path[tb]    None or path    not None => path known
 219         # p.file[tb]    None or path    not None => file exists
 220         # p.spec        string or None
 221         # p.spec_tbp    True or False, or not set if spec is None
 222         # p.dir         '' or '/'
 223
 224  def __init__(p, what):
 225         p.what = what
 226         p.path = [None,None]
 227         p.file = [None,None]
 228         p.spec = None
 229         p.dir = ''
 230
 231  def __repr__(p):
 232         return "<AF@%s>" % p.__str__()
 233  def __str__(p):
 234         def ptbp(tbp):
 235                 if p.path[tbp] is None: return '-'+p.dir
 236                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 237                 else: return p.path[tbp]+p.dir+'!'
 238         out = p.what
 239         if p.spec is not None:
 240                 if not hasattr(p,'spec_tb'): out += '~'
 241                 elif p.spec_tbp: out += '#'
 242                 else: out += '='
 243                 out += p.spec
 244         out += ':'
 245         out += ptbp(False)
 246         out += '|'
 247         out += ptbp(True)
 248         return out
 249
 250  def _wrong(p, how):
 251         xtra = ''
 252         if p.spec is not None:
 253                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 254         raise ("internal error: %s (%s)" % (how, str(p)))
 255
 256  def _ensure_path(p, tbp):
 257         if p.path[tbp] is None:
 258                 if '/' in p.what:
 259                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 260                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 261                 if not tbp:
 262                         p.path[tbp] = tmpdir+'/'+p.what
 263                 else:
 264                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 265
 266  def write(p, tbp=False):
 267         p._debug('write %s...' % 'HT'[tbp])
 268         p._ensure_path(tbp)
 269
 270         if p.dir and not p.file[tbp]:
 271                 if not tbp:
 272                         p._debug('mkdir H')
 273                         mkdir_okexist(p.path[tbp])
 274                 else:
 275                         cmdl = ['sh','-ec',
 276                                 'test -d "$1" || mkdir -p "$1"',
 277                                 'x', p.path[tbp]]
 278                         tf_what = urllib.quote(p.what).replace('/','%2F')
 279                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 280                         if rc: bomb('failed to create directory %s' %
 281                                 p.path[tbp])
 282
 283         p.file[tbp] = p.path[tbp]
 284         return p.path[tbp]
 285
 286  def read(p, tbp=False):
 287         p._debug('read %s...' % 'HT'[tbp])
 288         p._ensure_path(tbp)
 289
 290         if p.file[tbp] is None:
 291                 if p.file[not tbp] is None:
 292                         p._wrong("requesting read but nonexistent")
 293                 cud = ['copyup','copydown'][tbp]
 294                 src = p.file[not tbp] + p.dir
 295                 dst = p.path[tbp] + p.dir
 296                 testbed.command(cud, (src, dst))
 297                 p.file[tbp] = p.path[tbp]
 298
 299         return p.file[tbp] + p.dir
 300
 301  def invalidate(p, tbp=False):
 302         p.file[tbp] = None
 303         p._debug('invalidated %s' % 'HT'[tbp])
 304
 305  def _debug(p, m):
 306         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 307
 308  def _constructed(p):
 309         p._debug('constructed: '+str(p))
 310         p._check()
 311
 312  def _check(p):
 313         for tbp in [False,True]:
 314          for pf in [p.path, p.file]:
 315                 if pf[tbp] is None: continue
 316                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 317                 if p.dir and pf[tbp].endswith('/'):
 318                         pf[tbp] = pf[tbp].rstrip('/')
 319                         if not pf[tbp]: pf[tbp] = '/'
 320                 if not p.dir and pf[tbp].endswith('/'):
 321                         bomb("directory `%s' specified for "
 322                              "non-directory %s" % (pf[tbp], p.what))
 323
 324  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 325         AutoFile.__init__(p,what)
 326         sh_on = ''; sh_sibl = ''
 327         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 328         if sibling: sh_sibl=' (sibling)'
 329         parent._debug('using as base: %s: %s%s%s' %
 330                         (str(parent), leaf, sh_on, sh_sibl))
 331         if not sibling and not parent.dir:
 332                 parent._wrong('asked for non-sibling relative path of non-dir')
 333         if sibling: trim = os.path.dirname
 334         else: trim = lambda x: x
 335         for tbp in [False,True]:
 336                 if parent.path[tbp] is None: continue
 337                 trimmed = trim(parent.path[tbp])
 338                 if trimmed: trimmed += '/'
 339                 p.path[tbp] = trimmed + leaf
 340                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 341                         p.file[tbp] = p.path[tbp]
 342
 343 class InputFile(AutoFile):
 344  def _init(p, what, spec, spec_tbp=False):
 345         AutoFile.__init__(p, what)
 346         p.spec = spec
 347         p.spec_tbp = spec_tbp
 348         p.path[spec_tbp] = spec
 349         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 350  def __init__(p, what, spec, spec_tbp=False):
 351         p._init(what,spec,spec_tbp)
 352         p._constructed()
 353
 354 class InputDir(InputFile):
 355  def __init__(p, what, spec, spec_tbp=False):
 356         InputFile._init(p,what,spec,spec_tbp)
 357         p.dir = '/'
 358         p._constructed()
 359
 360 class OutputFile(AutoFile):
 361  def _init(p, what, spec, spec_tbp=False):
 362         AutoFile.__init__(p, what)
 363         p.spec = spec
 364         p.spec_tbp = spec_tbp
 365         p.path[spec_tbp] = spec
 366  def __init__(p, what, spec, spec_tbp=False):
 367         p._init(what,spec,spec_tbp)
 368         p._constructed()
 369
 370 class OutputDir(OutputFile):
 371  def __init__(p, what, spec, spec_tbp=False):
 372         OutputFile._init(p,what,spec,spec_tbp)
 373         p.dir = '/'
 374         p._constructed()
 375
 376 class RelativeInputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 378         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 379         p._constructed()
 380
 381 class RelativeOutputFile(AutoFile):
 382  def __init__(p, what, parent, leaf, sibling=False):
 383         p._relative_init(what, parent, leaf, None, False, sibling)
 384         p._constructed()
 385
 386 class TemporaryFile(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p._constructed()
 390
 391 class TemporaryDir(AutoFile):
 392  def __init__(p, what):
 393         AutoFile.__init__(p, what)
 394         p.dir = '/'
 395         p._constructed()
 396
 397 #---------- parsing and representation of the arguments
 398
 399 class Action:
 400  def __init__(a, kind, af, arghandling, what):
 401         # extra attributes get added during processing
 402         a.kind = kind
 403         a.af = af
 404         a.ah = arghandling
 405         a.what = what
 406         a.missing_tests_control = False
 407  def __repr__(a):
 408         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 409
 410 def parse_args():
 411         global opts, timeouts
 412         global n_non_actions # argh, stupid python scoping rules
 413         usage = "%prog <options> --- <virt-server>..."
 414         parser = OptionParser(usage=usage)
 415         pa = parser.add_option
 416         pe = parser.add_option
 417
 418         arghandling = {
 419                 'dsc_tests': True,
 420                 'dsc_filter': '*',
 421                 'deb_forbuilds': 'auto',
 422                 'deb_fortests': 'auto',
 423                 'tb': False,
 424                 'override_control': None,
 425                 'set_lang': 'C'
 426         }
 427         initial_arghandling = arghandling.copy()
 428         n_non_actions = 0
 429
 430         #----------
 431         # actions (ie, test sets to run, sources to build, binaries to use):
 432
 433         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 434                 global n_non_actions
 435                 parser.largs.append((value,kindpath))
 436                 n_non_actions += not(is_act)
 437
 438         def pa_action(long, metavar, kindpath, help, is_act=True):
 439                 pa('','--'+long, action='callback', callback=cb_action,
 440                         nargs=1, type='string',
 441                         callback_args=(long,kindpath,is_act), help=help)
 442
 443         pa_action('built-tree',         'TREE', '@/',
 444                 help='run tests from build tree TREE')
 445
 446         pa_action('unbuilt-tree',       'TREE', '@//',
 447                 help='run tests from build tree TREE')
 448
 449         pa_action('source',             'DSC', '@.dsc',
 450                 help='build DSC and use its tests and/or'
 451                     ' generated binary packages')
 452
 453         pa_action('binary',             'DEB', '@.deb',
 454                help='use binary package DEB according'
 455                     ' to most recent --binaries-* settings')
 456
 457         def cb_actnoarg(op,optstr,value,parser, largsentry):
 458                 parser.largs.append(largsentry)
 459         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 460                 callback_args=((None, ('instantiate',)),),
 461                 help='instantiate testbed now (during testing phase)'
 462                      ' and install packages'
 463                      ' selected for automatic installation, even'
 464                      ' if this might apparently not be required otherwise')
 465
 466         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 467                help='run tests from control file CONTROL instead,'
 468                     ' (applies to next test suite only)')
 469
 470         #----------
 471         # argument handling settings (what ways to use action
 472         #  arguments, and pathname processing):
 473
 474         def cb_setah(option, opt_str, value, parser, toset,setval):
 475                 if type(setval) == list:
 476                         if not value in setval:
 477                                 parser.error('value for %s option (%s) is not '
 478                                  'one of the permitted values (%s)' %
 479                                  (value, opt_str, setval.join(' ')))
 480                 elif setval is not None:
 481                         value = setval
 482                 for v in toset:
 483                         arghandling[v] = value
 484                 parser.largs.append(arghandling.copy())
 485
 486         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 487                 type = metavar
 488                 if type is not None: type = 'string'
 489                 pa('',long, action='callback', callback=cb_setah,
 490                    callback_args=(affected,effect), **kwargs)
 491
 492         #---- paths: host or testbed:
 493         #
 494         pa_setah('--paths-testbed', ['tb'],True,
 495                 help='subsequent path specifications refer to the testbed')
 496         pa_setah('--paths-host', ['tb'],False,
 497                 help='subsequent path specifications refer to the host')
 498
 499         #---- source processing settings:
 500
 501         pa_setah('--sources-tests', ['dsc_tests'],True,
 502                 help='run tests from builds of subsequent sources')
 503         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 504                 help='do not run tests from builds of subsequent sources')
 505
 506         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 507                 type='string', metavar='PATTERN-LIST',
 508                 help='from subsequent sources, use binaries matching'
 509                      ' PATTERN-LIST (comma-separated glob patterns)'
 510                      ' according to most recent --binaries-* settings')
 511         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 512                 help='from subsequent sources, do not use any binaries')
 513         #---- binary package processing settings:
 514
 515         def pa_setahbins(long,toset,how):
 516          pa_setah(long, toset,['ignore','auto','install'],
 517                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 518                 help=how+' ignore binaries, install them as needed'
 519                         ' for dependencies, or unconditionally install'
 520                         ' them, respectively')
 521         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 522         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 523         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 524
 525         #----------
 526         # general options:
 527
 528         def cb_vserv(op,optstr,value,parser):
 529                 parser.values.vserver = list(parser.rargs)
 530                 del parser.rargs[:]
 531
 532         def cb_path(op,optstr,value,parser, constructor,long,dir):
 533                 name = long.replace('-','_')
 534                 af = constructor(long, value,arghandling['tb'])
 535                 setattr(parser.values, name, af)
 536
 537         def pa_path(long, constructor, help, dir=False):
 538                 pa('','--'+long, action='callback', callback=cb_path,
 539                         callback_args=(constructor,long,dir),
 540                         nargs=1, type='string',
 541                         help=help, metavar='PATH')
 542
 543         pa_path('output-dir', OutputDir, dir=True,
 544                 help='write stderr/out files in PATH')
 545
 546         pa('--leave-lang', dest='set_lang', action='store_false',
 547                 help="leave LANG on testbed set to testbed's default")
 548         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 549                 help='set LANG on testbed to LANGVAL', default='C')
 550
 551         pa('','--tmp-dir',              type='string', dest='tmpdir',
 552                 help='write temporary files to TMPDIR, emptying it'
 553                      ' beforehand and leaving it behind at the end')
 554         pa('','--log-file',             type='string', dest='logfile',
 555                 help='write the log LOGFILE, emptying it beforehand,'
 556                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 557         pa('','--summary-file',         type='string', dest='summary',
 558                 help='write a summary report to SUMMARY,'
 559                      ' emptying it beforehand')
 560
 561         for k in timeouts.keys():
 562                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 563                         metavar='T', help='set %s timeout to T')
 564         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 565                         metavar='FACTOR', default=1.0,
 566                         help='multiply all default timeouts by FACTOR')
 567
 568         pa('','--user',                 type='string', dest='user',
 569                 help='run tests as USER (needs root on testbed)')
 570         pa('','--gain-root',            type='string', dest='gainroot',
 571                 help='prefix debian/rules binary with GAINROOT')
 572         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 573         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 574         pa('','--gnupg-home',           type='string', dest='gnupghome',
 575                 default='~/.autopkgtest/gpg',
 576                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 577                         " signing private apt archive);"
 578                         " `fresh' means generate new key each time.")
 579
 580         #----------
 581         # actual meat:
 582
 583         class SpecialOption(optparse.Option): pass
 584         vs_op = SpecialOption('','--VSERVER-DUMMY')
 585         vs_op.action = 'callback'
 586         vs_op.type = None
 587         vs_op.default = None
 588         vs_op.nargs = 0
 589         vs_op.callback = cb_vserv
 590         vs_op.callback_args = ( )
 591         vs_op.callback_kwargs = { }
 592         vs_op.help = 'introduces virtualisation server and args'
 593         vs_op._short_opts = []
 594         vs_op._long_opts = ['---']
 595
 596         pa(vs_op)
 597
 598         (opts,args) = parser.parse_args()
 599         if not hasattr(opts,'vserver'):
 600                 parser.error('you must specifiy --- <virt-server>...')
 601         if n_non_actions >= len(parser.largs):
 602                 parser.error('nothing to do specified')
 603
 604         for k in timeouts.keys():
 605                 t = getattr(opts,'timeout_'+k)
 606                 if t is None: t = timeouts[k] * opts.timeout_factor
 607                 timeouts[k] = int(t)
 608
 609         arghandling = initial_arghandling
 610         opts.actions = []
 611         ix = 0
 612         for act in args:
 613                 if type(act) == dict:
 614                         arghandling = act
 615                         continue
 616                 elif type(act) == tuple:
 617                         pass
 618                 elif type(act) == str:
 619                         act = (act,act)
 620                 else:
 621                         raise ("unknown action in list `%s' having"
 622                               " type `%s'" % (act, type(act)))
 623                 (pathstr, kindpath) = act
 624
 625                 constructor = InputFile
 626                 if type(kindpath) is tuple:             kind = kindpath[0]
 627                 elif kindpath.endswith('.deb'):         kind = 'deb'
 628                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 629                 elif kindpath.endswith('//'):
 630                         kind = 'ubtree'
 631                         constructor = InputDir
 632                 elif kindpath.endswith('/'):
 633                         kind = 'tree'
 634                         constructor = InputDir
 635                 else: parser.error("do not know how to handle filename `%s';"
 636                         " specify --source --binary or --build-tree" %
 637                         kindpath)
 638
 639                 what = '%s%s' % (kind,ix); ix += 1
 640
 641                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 642                 else: fwhatx = '-'+kind
 643
 644                 if pathstr is None: af = None
 645                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 646                 opts.actions.append(Action(kind, af, arghandling, what))
 647
 648 def setup_trace():
 649         global trace_stream, tmpdir, summary_stream
 650
 651         if opts.tmpdir is not None:
 652                 rmtree('tmpdir(specified)',opts.tmpdir)
 653                 mkdir_okexist(opts.tmpdir, 0700)
 654                 tmpdir = opts.tmpdir
 655         else:
 656                 assert(tmpdir is None)
 657                 tmpdir = tempfile.mkdtemp()
 658
 659         if opts.logfile is None:
 660                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 661                         opts.logfile = opts.output_dir.spec + '/log'
 662                 elif opts.tmpdir is not None:
 663                         opts.logfile = opts.tmpdir + '/log'
 664         if opts.logfile is not None:
 665                 trace_stream = open(opts.logfile, 'w', 0)
 666         if opts.summary is not None:
 667                 summary_stream = open(opts.summary, 'w', 0)
 668
 669         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 670
 671 def finalise_options():
 672         global opts, tb, build_essential
 673
 674         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 675                 opts.user = ''
 676
 677         if opts.user is None:
 678                 su = 'suggested-normal-user='
 679                 ul = [
 680                         e[len(su):]
 681                         for e in testbed.caps
 682                         if e.startswith(su)
 683                         ]
 684                 if ul:
 685                         opts.user = ul[0]
 686                 else:
 687                         opts.user = ''
 688
 689         if opts.user:
 690                 if 'root-on-testbed' not in testbed.caps:
 691                         pstderr("warning: virtualisation"
 692                                 " system does not offer root on testbed,"
 693                                 " but --user option specified: failure likely")
 694                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 695         else:
 696                 opts.user_wrap = lambda x: x
 697
 698         if opts.gainroot is None:
 699                 opts.gainroot = ''
 700                 if (opts.user or
 701                     'root-on-testbed' not in testbed.caps):
 702                         opts.gainroot = 'fakeroot'
 703                         build_essential += ['fakeroot']
 704
 705         if opts.gnupghome.startswith('~/'):
 706                 try: home = os.environ['HOME']
 707                 except KeyError:
 708                         parser.error("HOME environment variable"
 709                                 " not set, needed for --gnupghome=`%s"
 710                                 % opts.gnupghome)
 711                 opts.gnupghome = home + opts.gnupghome[1:]
 712         elif opts.gnupghome == 'fresh':
 713                 opts.gnupghome = None
 714
 715 #---------- testbed management - the Testbed class
 716
 717 class Testbed:
 718  def __init__(tb):
 719         tb.sp = None
 720         tb.lastsend = None
 721         tb.scratch = None
 722         tb.modified = False
 723         tb.blamed = []
 724         tb._ephemeral = []
 725         tb._debug('init')
 726         tb._need_reset_apt = False
 727  def _debug(tb, m, minlevel=0):
 728         debug('** '+m, minlevel)
 729  def start(tb):
 730         tb._debug('start')
 731         p = subprocess.PIPE
 732         debug_subprocess('vserver', opts.vserver)
 733         tb._errplumb = Errplumb(True)
 734         tb.sp = subprocess.Popen(opts.vserver,
 735                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 736         tb.expect('ok')
 737         tb.caps = tb.commandr('capabilities')
 738         if not 'print-execute-command' in tb.caps:
 739                 tb.bomb('testbed does not support print-execute-command')
 740  def stop(tb):
 741         tb._debug('stop')
 742         tb.close()
 743         if tb.sp is None: return
 744         ec = tb.sp.returncode
 745         if ec is None:
 746                 tb.sp.stdout.close()
 747                 tb.send('quit')
 748                 tb.sp.stdin.close()
 749                 ec = tb.sp.wait()
 750         if ec:
 751                 tb.bomb('testbed gave exit status %d after quit' % ec)
 752         tb._errplumb.wait()
 753  def open(tb):
 754         tb._debug('open, scratch=%s' % tb.scratch)
 755         if tb.scratch is not None: return
 756         pl = tb.commandr('open')
 757         tb.scratch = InputDir('tb-scratch', pl[0], True)
 758         tb.deps_processed = []
 759
 760         pec = tb.commandr('print-execute-command')
 761         if len(pec) < 2: tb.bomb('too few results from print-execute-command')
 762         tb.ec_cmdl = map(urllib.unquote, pec[0].split(','))
 763         tb.ec_mode = urllib.unquote(pec[1])
 764         tb.ec_infos = map(urllib.unquote(pec[2:]))
 765
 766         shellquote_re = regexp.compile(r'([\\"$`])')
 767         def shellquote_arg(s): '"' + shellquote_re.sub(r'\\\1', s) + '"
 768         def shellquote_cmdl(l): ' '.join(map(shellquote,l))
 769
 770         if tb.ec_mode eq 'auxverb':
 771                 tec_cmdl = tb.ec_cmdl
 772         elif tb.ec_mode eq 'shstring':
 773                 tec_cmdl = shellquote_cmdl(tb.ec_cmdl)
 774         else:
 775                 tb.bomb('print-execute-command unsupported mode %s'
 776                                  % tb.ec_mode)
 777
 778         tb.ec_auxverbscript = TemporaryFile('auxverb')
 779         print >>open(tb.ec_auxverbscript.write(),'w'), '\n'.join([
 780                 '#!/bin/sh',
 781                 'exec '.shellquote_cmdl(tec_cmdl).' "$@"'
 782                 ]
 783
 784  def mungeing_apt(tb):
 785         if not 'revert' in tb.caps:
 786                 tb._need_reset_apt = True
 787  def reset_apt(tb):
 788         if not tb._need_reset_apt: return
 789         what = 'aptget-update-reset'
 790         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 791         rc = tb.execute(what, cmdl, kind='install')
 792         if rc:
 793                 pstderr("\n" "warning: failed to restore"
 794                         " testbed apt cache, exit code %d" % rc)
 795         tb._need_reset_apt = False
 796  def close(tb):
 797         tb._debug('close, scratch=%s' % tb.scratch)
 798         if tb.scratch is None: return
 799         tb.scratch = None
 800         if tb.sp is None: return
 801         tb.command('close')
 802  def prepare1(tb, deps_new):
 803         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 804                 (tb.modified, tb.deps_processed, deps_new), 1)
 805         if 'revert' in tb.caps and (tb.modified or
 806             [d for d in tb.deps_processed if d not in deps_new]):
 807                 for af in tb._ephemeral: af.read(False)
 808                 tb._debug('reset **')
 809                 tb.command('revert')
 810                 tb.blamed = []
 811                 for af in tb._ephemeral: af.invalidate(True)
 812         tb.modified = False
 813  def prepare2(tb, deps_new):
 814         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 815         binaries.publish()
 816         tb._install_deps(deps_new)
 817  def prepare(tb, deps_new):
 818         tb.prepare1(deps_new)
 819         tb.prepare2(deps_new)
 820  def register_ephemeral(tb, af):
 821         tb._ephemeral.append(af)
 822  def _install_deps(tb, deps_new):
 823         tb._debug(' installing dependencies '+`deps_new`, 1)
 824         tb.deps_processed = deps_new
 825         if not deps_new: return
 826         binaries.satisfy_dependencies_string(tb, ', '.join(deps_new)
 827                                         'install-deps')
 828  def needs_reset(tb):
 829         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 830         tb.modified = True
 831  def blame(tb, m):
 832         tb._debug('blame += %s' % m, 1)
 833         tb.blamed.append(m)
 834  def bomb(tb, m):
 835         tb._debug('bomb %s' % m)
 836         if tb.sp is not None:
 837                 tb.sp.stdout.close()
 838                 tb.sp.stdin.close()
 839                 ec = tb.sp.wait()
 840                 if ec: pstderr('adt-run: testbed failing,'
 841                                 ' exit status %d' % ec)
 842         tb.sp = None
 843         raise Quit(16, 'testbed failed: %s' % m)
 844  def send(tb, string):
 845         tb.sp.stdin
 846         try:
 847                 debug('>> '+string, 2)
 848                 print >>tb.sp.stdin, string
 849                 tb.sp.stdin.flush()
 850                 tb.lastsend = string
 851         except:
 852                 (type, value, dummy) = sys.exc_info()
 853                 tb.bomb('cannot send to testbed: %s' % traceback.
 854                         format_exception_only(type, value))
 855  def expect(tb, keyword, nresults=None):
 856         l = tb.sp.stdout.readline()
 857         if not l: tb.bomb('unexpected eof from the testbed')
 858         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 859         l = l.rstrip('\n')
 860         debug('<< '+l, 2)
 861         ll = l.split()
 862         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 863         if ll[0] != keyword:
 864                 if tb.lastsend is None:
 865                         tb.bomb("got banner `%s', expected `%s...'" %
 866                                 (l, keyword))
 867                 else:
 868                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 869                                 (tb.lastsend, l, keyword))
 870         ll = ll[1:]
 871         if nresults is not None and len(ll) != nresults:
 872                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 873                         " expected %d result parameters" %
 874                         (string, l, len(ll), nresults))
 875         return ll
 876  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 877         # pass args=[None,...] or =(None,...) to avoid more url quoting
 878         if type(cmd) is str: cmd = [cmd]
 879         if len(args) and args[0] is None: args = args[1:]
 880         else: args = map(urllib.quote, args)
 881         al = cmd + args
 882         tb.send(string.join(al))
 883         ll = tb.expect('ok', nresults)
 884         if unquote: ll = map(urllib.unquote, ll)
 885         return ll
 886  def command(tb, cmd, args=()):
 887         tb.commandr(cmd, args, 0)
 888  def commandr1(tb, cmd, args=()):
 889         rl = tb.commandr(cmd, args, 1)
 890         return rl[0]
 891  def execute(tb, what, cmdl,
 892                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 893                 script=False, tmpdir=None, kind='short'):
 894         # Options for script:
 895         #   False - do not call debug_subprocess, no synch. reporting required
 896         #   None or string - call debug_subprocess with that value,
 897         #                       plumb stderr through synchronously if possible
 898         # Options for se:
 899         #   None - usual Errplumb (output is of kind 2c)
 900         #   string - path on testbed (output is of kind 2d)
 901
 902         timeout = timeouts[kind]
 903
 904         if script is not False: debug_subprocess(what, cmdl, script=script)
 905         if cwd is None: cwd = tb.scratch.write(True)
 906
 907         xdump = None
 908         if se is None:
 909                 ep = Errplumb()
 910                 se_catch = TemporaryFile(what+'-xerr')
 911                 se_use = se_catch.write(True)
 912                 if not opts.quiet: xdump = 'debug=2-2'
 913                 elif trace_stream is not None:
 914                         xdump = 'debug=2-%d' % trace_stream.fileno()
 915         else:
 916                 ep = None
 917                 se_catch = None
 918                 se_use = se
 919
 920         cmdl = [None,
 921                 ','.join(map(urllib.quote, cmdl)),
 922                 si, so, se_use, cwd]
 923
 924         if timeout is not None and timeout > 0:
 925                 cmdl.append('timeout=%d' % timeout)
 926
 927         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 928         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 929         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 930         if opts.set_lang is not False:
 931                 cmdl.append('env=LANG=%s' % opts.set_lang)
 932
 933         rc = tb.commandr1('execute', cmdl)
 934         try: rc = int(rc)
 935         except ValueError: bomb("execute for %s gave invalid response `%s'"
 936                                         % (what,rc))
 937
 938         if se_catch is not None:
 939                 debug_file(se_catch.read())
 940         if ep is not None:
 941                 ep.wait()
 942
 943         return rc
 944
 945  def satisfy_dependencies_string(tb, deps, what):
 946         # Must have called Binaries.configure_apt
 947         dsc = TemporaryFile('deps.dsc')
 948         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps
 949         tb.satisfy_dependencies_dsc(tb, dsc)
 950
 951  def satisfy_dependencies_dsc(tb, dsc, what):
 952         # Must have called Binaries.configure_apt
 953         rc = tb.execute(what,
 954                 'pbuilder-satisfydepends --check-key --binary-all',
 955                 ['--internal-chrootexec',tb.ec_auxverbscript.read(),
 956                  '-c',dsc.read()]
 957         if rc: badpkg('dependency install failed, exit code %d' % rc)
 958
 959 #---------- representation of test control files: Field*, Test, etc.
 960
 961 class FieldBase:
 962  def __init__(f, fname, stz, base, tnames, vl):
 963         assert(vl)
 964         f.stz = stz
 965         f.base = base
 966         f.tnames = tnames
 967         f.vl = vl
 968  def words(f):
 969         def distribute(vle):
 970                 (lno, v) = vle
 971                 r = v.split()
 972                 r = map((lambda w: (lno, w)), r)
 973                 return r
 974         return flatten(map(distribute, f.vl))
 975  def atmostone(f):
 976         if len(vl) == 1:
 977                 (f.lno, f.v) = vl[0]
 978         else:
 979                 raise Unsupported(f.vl[1][0],
 980                         'only one %s field allowed' % fn)
 981         return f.v
 982
 983 class FieldIgnore(FieldBase):
 984  def parse(f): pass
 985
 986 class Restriction:
 987  def __init__(r,rname,base): pass
 988
 989 class Restriction_rw_build_tree(Restriction): pass
 990 class Restriction_breaks_testbed(Restriction):
 991  def __init__(r, rname, base):
 992         if 'revert' not in testbed.caps:
 993                 raise Unsupported(f.lno,
 994                         'Test breaks testbed but testbed cannot revert')
 995 class Restriction_needs_root(Restriction):
 996  def __init__(r, rname, base):
 997         if 'root-on-testbed' not in testbed.caps:
 998                 raise Unsupported(f.lno,
 999                         'Test needs root on testbed which is not available')
1000
1001 class Field_Restrictions(FieldBase):
1002  def parse(f):
1003         for wle in f.words():
1004                 (lno, rname) = wle
1005                 nrname = rname.replace('-','_')
1006                 try: rclass = globals()['Restriction_'+nrname]
1007                 except KeyError: raise Unsupported(lno,
1008                         'unknown restriction %s' % rname)
1009                 r = rclass(nrname, f.base)
1010                 f.base['restriction_names'].append(rname)
1011                 f.base['restrictions'].append(r)
1012
1013 class Field_Features(FieldIgnore):
1014  def parse(f):
1015         for wle in f.words():
1016                 (lno, fname) = wle
1017                 f.base['feature_names'].append(fname)
1018                 nfname = fname.replace('-','_')
1019                 try: fclass = globals()['Feature_'+nfname]
1020                 except KeyError: continue
1021                 ft = fclass(nfname, f.base)
1022                 f.base['features'].append(ft)
1023
1024 class Field_Tests(FieldIgnore): pass
1025
1026 class Field_Depends(FieldBase):
1027  def parse(f):
1028         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
1029         dl = map(lambda x: x.strip(),
1030                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1031         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1032         for d in dl:
1033                 if re.search(d):
1034                         badpkg("Test Depends field contains dependency"
1035                                " `%s' with invalid characters" % d)
1036         f.base['depends'] = dl
1037
1038 class Field_Tests_directory(FieldBase):
1039  def parse(f):
1040         td = atmostone(f)
1041         if td.startswith('/'): raise Unspported(f.lno,
1042                 'Tests-Directory may not be absolute')
1043         f.base['testsdir'] = td
1044
1045 def run_tests(stanzas, tree):
1046         global errorcode, testbed
1047         if stanzas == ():
1048                 report('*', 'SKIP no tests in this package')
1049                 errorcode |= 8
1050         for stanza in stanzas:
1051                 tests = stanza[' tests']
1052                 if not tests:
1053                         report('*', 'SKIP package has metadata but no tests')
1054                         errorcode |= 8
1055                 for t in tests:
1056                         t.prepare()
1057                         t.run(tree)
1058                         if 'breaks-testbed' in t.restriction_names:
1059                                 testbed.needs_reset()
1060                 testbed.needs_reset()
1061
1062 class Test:
1063  def __init__(t, tname, base, act):
1064         if '/' in tname: raise Unsupported(base[' lno'],
1065                 'test name may not contain / character')
1066         for k in base: setattr(t,k,base[k])
1067         t.tname = tname
1068         t.act = act
1069         t.what = act.what+'t-'+tname
1070         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1071         else: t.path = tname
1072         t._debug('constructed; path=%s' % t.path)
1073         t._debug(' .depends=%s' % t.depends)
1074  def _debug(t, m):
1075         debug('& %s: %s' % (t.what, m))
1076  def report(t, m):
1077         report(t.what, m)
1078  def reportfail(t, m):
1079         global errorcode
1080         errorcode |= 4
1081         report(t.what, 'FAIL ' + m)
1082  def prepare(t):
1083         t._debug('preparing')
1084         dn = []
1085         for d in t.depends:
1086                 t._debug(' processing dependency '+d)
1087                 if not '@' in d:
1088                         t._debug('  literal dependency '+d)
1089                         dn.append(d)
1090                 else:
1091                         for (pkg,bin) in t.act.binaries:
1092                                 d = d.replace('@',pkg)
1093                                 t._debug('  synthesised dependency '+d)
1094                                 dn.append(d)
1095         testbed.prepare(dn)
1096  def run(t, tree):
1097         t._debug('[----------------------------------------')
1098         def stdouterr(oe):
1099                 idstr = t.what + '-' + oe
1100                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1101                         use_dir = opts.output_dir
1102                 else:
1103                         use_dir = testbed.scratch
1104                 return RelativeOutputFile(idstr, use_dir, idstr)
1105
1106         if hasattr(t.act,'work'): t.act.work.write(True)
1107         tree.read(True)
1108
1109         af = RelativeInputFile(t.what, tree, t.path)
1110         so = stdouterr('stdout')
1111         se = stdouterr('stderr')
1112
1113         tf = af.read(True)
1114         tmpdir = None
1115
1116         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1117         if rc: bomb('failed to chmod +x %s' % tf)
1118
1119         if 'needs-root' not in t.restriction_names and opts.user is not None:
1120                 tfl = ['su',opts.user,'-c',tf]
1121                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1122                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1123                 if opts.user: script += '; chown %s "$1"' % opts.user
1124                 if 'rw-build-tree' in t.restriction_names:
1125                         script += '; chown -R %s "$2"' % opts.user
1126                 rc = testbed.execute('mktmpdir-'+t.what,
1127                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1128                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1129                                 % (tmpdir, rc))
1130         else:
1131                 tfl = [tf]
1132
1133         rc = testbed.execute('test-'+t.what, tfl,
1134                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1135                 tmpdir=tmpdir, kind='test')
1136
1137         so_read = so.read()
1138         se_read = se.read()
1139
1140         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1141         stab = os.stat(se_read)
1142         if stab.st_size != 0:
1143                 l = open(se_read).readline()
1144                 l = l.rstrip('\n \t\r')
1145                 if len(l) > 35: l = l[:35] + '...'
1146                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1147                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1148                 debug_file(se_read)
1149         elif rc != 0:
1150                 t.reportfail('non-zero exit status %d' % rc)
1151         else:
1152                 t.report('PASS')
1153
1154         stab = os.stat(so_read)
1155         if stab.st_size != 0:
1156                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1157                 debug_file(so_read)
1158
1159         t._debug('----------------------------------------]')
1160
1161 def read_control(act, tree, control_override):
1162         stanzas = [ ]
1163
1164         if control_override is not None:
1165                 control_af = control_override
1166                 testbed.blame('arg:'+control_override.spec)
1167         else:
1168                 if act.missing_tests_control:
1169                         return ()
1170                 control_af = RelativeInputFile(act.what+'-testcontrol',
1171                         tree, 'debian/tests/control')
1172         try:
1173                 control = open(control_af.read(), 'r')
1174         except (IOError,OSError), oe:
1175                 if oe[0] != errno.ENOENT: raise
1176                 return []
1177
1178         lno = 0
1179         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1180         stz = None      # stz[field_name][index] = (lno, value)
1181                         # special field names:
1182                         # stz[' lno'] = number
1183                         # stz[' tests'] = list of Test objects
1184         def end_stanza(stz):
1185                 if stz is None: return
1186                 stz[' errs'] = 0
1187                 stanzas.append(stz)
1188                 stz = None
1189                 hcurrent = None
1190
1191         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1192         while 1:
1193                 l = control.readline()
1194                 if not l: break
1195                 lno += 1
1196                 if not l.endswith('\n'): badctrl('unterminated line')
1197                 if regexp.compile('\s*\#').match(l): continue
1198                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1199                 initmat = initre.match(l)
1200                 if initmat:
1201                         (fname, l) = initmat.groups()
1202                         fname = string.capwords(fname)
1203                         if stz is None:
1204                                 stz = { ' lno': lno, ' tests': [] }
1205                         if not stz.has_key(fname): stz[fname] = [ ]
1206                         hcurrent = stz[fname]
1207                 elif regexp.compile('\s').match(l):
1208                         if not hcurrent: badctrl('unexpected continuation')
1209                 else:
1210                         badctrl('syntax error')
1211                 hcurrent.append((lno, l))
1212         end_stanza(stz)
1213
1214         def testbadctrl(stz, lno, m):
1215                 report_badctrl(lno, m)
1216                 stz[' errs'] += 1
1217
1218         for stz in stanzas:
1219                 try:
1220                         try: tnames = stz['Tests']
1221                         except KeyError:
1222                                 tnames = ['*']
1223                                 raise Unsupported(stz[' lno'],
1224                                         'no Tests field')
1225                         tnames = map((lambda lt: lt[1]), tnames)
1226                         tnames = string.join(tnames).split()
1227                         base = {
1228                                 'restriction_names': [],
1229                                 'restrictions': [],
1230                                 'feature_names': [],
1231                                 'features': [],
1232                                 'testsdir': 'debian/tests',
1233                                 'depends' : '@'
1234                         }
1235                         for fname in stz.keys():
1236                                 if fname.startswith(' '): continue
1237                                 vl = stz[fname]
1238                                 try: fclass = globals()['Field_'+
1239                                         fname.replace('-','_')]
1240                                 except KeyError: raise Unsupported(vl[0][0],
1241                                         'unknown metadata field %s' % fname)
1242                                 f = fclass(stz, fname, base, tnames, vl)
1243                                 f.parse()
1244                         for tname in tnames:
1245                                 t = Test(tname, base, act)
1246                                 stz[' tests'].append(t)
1247                 except Unsupported, u:
1248                         for tname in tnames: u.report(tname)
1249                         continue
1250
1251         return stanzas
1252
1253 def print_exception(ei, msgprefix=''):
1254         if msgprefix: pstderr(msgprefix)
1255         (et, q, tb) = ei
1256         if et is Quit:
1257                 pstderr('adt-run: ' + q.m)
1258                 psummary('quitting: '+q.m)
1259                 return q.ec
1260         else:
1261                 pstderr("adt-run: unexpected, exceptional, error:")
1262                 psummary('quitting: unexpected error, consult transcript')
1263                 traceback.print_exc(None, sys.stderr)
1264                 if trace_stream is not None:
1265                         traceback.print_exc(None, trace_stream)
1266                 return 20
1267
1268 def cleanup():
1269         global trace_stream
1270         try:
1271                 if testbed is not None:
1272                         testbed.reset_apt()
1273                         testbed.stop()
1274                 if opts.tmpdir is None and tmpdir is not None:
1275                         rmtree('tmpdir', tmpdir)
1276                 if trace_stream is not None:
1277                         trace_stream.close()
1278                         trace_stream = None
1279         except:
1280                 print_exception(sys.exc_info(),
1281                         '\nadt-run: error cleaning up:\n')
1282                 os._exit(20)
1283
1284 #---------- registration, installation etc. of .deb's: Binaries
1285
1286 def determine_package(act):
1287         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1288         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1289         if rc: badpkg('failed to parse binary package, code %d' % rc)
1290         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1291         act.pkg = None
1292         for l in output.split('\n'):
1293                 m = re.match(l)
1294                 if not m: continue
1295                 if act.pkg: badpkg('two Package: lines in control file')
1296                 act.pkg = m.groups()[0]
1297         if not act.pkg: badpkg('no good Package: line in control file')
1298
1299 class Binaries:
1300  def __init__(b):
1301         b.dir = TemporaryDir('binaries')
1302         b.dir.write()
1303         ok = False
1304
1305         if opts.gnupghome is None:
1306                 opts.gnupghome = tmpdir+'/gnupg'
1307
1308         b._debug('initialising')
1309         try:
1310                 for x in ['pubring','secring']:
1311                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1312                 ok = True
1313         except (IOError,OSError), oe:
1314                 if oe.errno != errno.ENOENT: raise
1315
1316         if ok: b._debug('no key generation needed')
1317         else: b.genkey()
1318
1319  def _debug(b, s):
1320         debug('* '+s)
1321
1322  def genkey(b):
1323         b._debug('preparing for key generation')
1324
1325         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1326         mkdir_okexist(opts.gnupghome, 0700)
1327
1328         script = '''
1329   exec >&2
1330   cd "$1"
1331   cat <<"END" >key-gen-params
1332 Key-Type: DSA
1333 Key-Length: 1024
1334 Key-Usage: sign
1335 Name-Real: autopkgtest per-run key
1336 Name-Comment: do not trust this key
1337 Name-Email: autopkgtest@example.com
1338 END
1339   set -x
1340   gpg --homedir="$1" --batch --gen-key key-gen-params
1341                 '''
1342         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1343         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1344         if rc: bomb('key generation failed, code %d' % rc)
1345
1346  def apt_configs(b):
1347         return {
1348                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1349                 "Debug::pkgProblemResolver": "true",
1350         }
1351
1352  def _configure_apt(b, tb):
1353         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1354                         True)
1355         f = open(config.write(),'w')
1356         for (k,v) in b.apt_configs().iteritems():
1357                 print >>f, '%s { "%s"; };' % (k, v)
1358         f.close()
1359
1360  def _apt_get(b):
1361         ag = ['apt-get','-qy']
1362         for kv in b.apt_configs().iteritems():
1363                 ag += ['-o', '%s=%s' % kv]
1364         return ' '.join(ag)
1365
1366  def reset(b):
1367         b._debug('reset')
1368         rmtree('binaries', b.dir.read())
1369         b.dir.invalidate()
1370         b.dir.write()
1371         b.install = []
1372         b.blamed = []
1373         b.registered = set()
1374
1375  def register(b, act, pkg, af, forwhat, blamed):
1376         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1377                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1378
1379         if act.ah['deb_'+forwhat] == 'ignore': return
1380
1381         b.blamed += testbed.blamed
1382
1383         leafname = pkg+'.deb'
1384         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1385
1386         try: os.remove(dest.write())
1387         except (IOError,OSError), oe:
1388                 if oe.errno != errno.ENOENT: raise e
1389
1390         try: os.link(af.read(), dest.write())
1391         except (IOError,OSError), oe:
1392                 if oe.errno != errno.EXDEV: raise e
1393                 shutil.copy(af.read(), dest)
1394
1395         if act.ah['deb_'+forwhat] == 'install':
1396                 b.install.append(pkg)
1397
1398         b.registered.add(pkg)
1399
1400  def publish(b):
1401         b._debug('publish')
1402
1403         b._configure_apt(testbed)
1404
1405         script = '''
1406   exec >&2
1407   cd "$1"
1408   apt-ftparchive packages . >Packages
1409   gzip <Packages >Packages.gz
1410   apt-ftparchive release . >Release
1411   rm -f Release.gpg
1412   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1413   gpg --homedir="$2" --batch --export >archive-key.pgp
1414                 '''
1415         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1416         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1417         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1418
1419         b.dir.invalidate(True)
1420         apt_source = b.dir.read(True)
1421
1422         so = TemporaryFile('vlds')
1423         script = '''
1424   exec 3>&1 >&2
1425   apt-key add archive-key.pgp
1426   echo "deb file://'''+apt_source+''' /" >sources.list
1427   cat /etc/apt/sources.list >>sources.list
1428   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1429     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1430   fi
1431   '''+ b._apt_get() +''' update >&2
1432   cat /var/lib/dpkg/status >&3
1433                 '''
1434         testbed.mungeing_apt()
1435         rc = testbed.execute('apt-key', ['sh','-ec',script],
1436                                 so=so.write(True), cwd=b.dir.write(True),
1437                                 script=script, kind='install')
1438         if rc: bomb('apt setup failed with exit code %d' % rc)
1439
1440         testbed.blamed += b.blamed
1441
1442         b._debug('publish reinstall checking...')
1443         pkgs_reinstall = set()
1444         pkg = None
1445         for l in open(so.read()):
1446                 if l.startswith('Package: '):
1447                         pkg = l[9:].rstrip()
1448                 elif l.startswith('Status: install '):
1449                         if pkg in b.registered:
1450                                 pkgs_reinstall.add(pkg)
1451                                 b._debug(' publish reinstall needs '+pkg)
1452
1453         if pkgs_reinstall:
1454                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1455                 what = 'apt-get-reinstall'
1456                 cmdl = (b._apt_get() + ' --reinstall install '+
1457                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1458                 cmdl = ['sh','-c',cmdl]
1459                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1460                 if rc: badpkg("installation of basic binarries failed,"
1461                                 " exit code %d" % rc)
1462
1463         b._debug('publish install...')
1464         for pkg in b.install:
1465                 what = 'apt-get-install-%s' % pkg
1466                 testbed.blame(pkg)
1467                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1468                 cmdl = ['sh','-c',cmdl]
1469                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1470                 if rc: badpkg("installation of %s failed, exit code %d"
1471                                 % (pkg, rc))
1472
1473         b._debug('publish done')
1474
1475 #---------- processing of sources (building)
1476
1477 def source_rules_command(act,script,what,which,work,cwd,
1478                                 results_lines=0,xargs=[]):
1479         script = [      "exec 3>&1 >&2",
1480                         "set -x"        ] + script
1481         script = '\n'.join(script)
1482         so = TemporaryFile('%s-%s-results' % (what,which))
1483         rc = testbed.execute('%s-%s' % (what,which),
1484                         ['sh','-ec',script]+xargs, script=script,
1485                         so=so.write(True), cwd=cwd, kind='build')
1486         results = open(so.read()).read().rstrip('\n')
1487         if len(results): results = results.split("\n")
1488         else: results = []
1489         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1490         if results_lines is not None and len(results) != results_lines:
1491                 badpkg("got %d lines of results from %s where %d expected"
1492                         % (len(results), which, results_lines))
1493         if results_lines==1: return results[0]
1494         return results
1495
1496 def build_source(act, control_override):
1497         act.blame = 'arg:'+act.af.spec
1498         testbed.blame(act.blame)
1499         testbed.prepare1([])
1500         testbed.needs_reset()
1501
1502         what = act.what
1503         basename = act.af.spec
1504         debiancontrol = None
1505         act.binaries = []
1506
1507         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1508
1509         if act.kind == 'dsc':
1510                 dsc = act.af
1511                 dsc_file = open(dsc.read())
1512                 in_files = False
1513                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1514                 for l in dsc_file:
1515                         l = l.rstrip('\n')
1516                         if l.startswith('Files:'): in_files = True; continue
1517                         elif l.startswith('#'): pass
1518                         elif not l.startswith(' '):
1519                                 in_files = False
1520                                 if l.startswith('Source:'):
1521                                         act.blame = 'dsc:'+l[7:].strip()
1522                                         testbed.blame(act.blame)
1523                         if not in_files: continue
1524
1525                         m = fre.match(l)
1526                         if not m: badpkg(".dsc contains unparseable line"
1527                                         " in Files: `%s'" % l)
1528                         leaf = m.groups(0)[0]
1529                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1530                                         sibling=True)
1531                         subfile.read(True)
1532                 dsc.read(True)
1533
1534         if act.kind == 'ubtree':
1535                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1536                         act.af, 'debian/control')
1537                 dsc = TemporaryFile(what+'-fakedsc')
1538                 dsc_w = open(dsc.write(), 'w')
1539                 for l in open(debiancontrol.read()):
1540                         l = l.rstrip('\n')
1541                         if not len(l): break
1542                         print >>dsc_w, l
1543                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1544                 dsc_w.close()
1545
1546         if act.kind == 'dsc':
1547                 testbed.prepare2([])
1548                 tb.satisfy_dependencies_string('dpkg-dev', 'install dpkg-dev')
1549
1550         work = TemporaryDir(what+'-build')
1551         act.work = work
1552
1553         tmpdir = work.write(True)+'/tmpdir'
1554         tmpdir_script = [
1555                         'TMPDIR="$1"',
1556                         'rm -rf -- "$TMPDIR"',
1557                         'export TMPDIR',
1558                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1559                 ]
1560
1561         if act.kind == 'ubtree':
1562                 spec = '%s/real-tree' % work.write(True)
1563                 create_command = '''
1564                         rm -rf "$spec"
1565                         mkdir "$spec"
1566                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1567                         '''
1568                 initcwd = act.af.read(True)
1569
1570         if act.kind == 'dsc':
1571                 spec = dsc.read(True)
1572                 create_command = '''
1573                         dpkg-source -x $spec
1574                         '''
1575                 initcwd = work.write(True)
1576
1577         script = [
1578                 'spec="$2"',
1579                 'origpwd=`pwd`',
1580                 'cd '+work.write(True)
1581         ]
1582
1583         if opts.user:
1584                 script += ([ 'chown '+opts.user+' .' ] +
1585                         tmpdir_script +
1586                         [ 'spec="$spec" origpwd="$origpwd" '
1587                                 +opts.user_wrap(create_command) ])
1588         else:
1589                 script += (tmpdir_script +
1590                         [ create_command ])
1591
1592         script += [
1593                         'cd [a-z0-9]*-*/.',
1594                         'pwd >&3',
1595                         'set +e; test -f debian/tests/control; echo $? >&3'
1596                 ]
1597         (result_pwd, control_test_rc) = source_rules_command(
1598                         act,script,what,'extract',work,
1599                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1600
1601         filter = act.ah['dsc_filter']
1602
1603         if control_test_rc == '1': act.missing_tests_control = True
1604
1605         # For optional builds:
1606         #
1607         # We might need to build the package because:
1608         #   - we want its binaries (filter isn't _ and at least one of the
1609         #       deb_... isn't ignore)
1610         #   - the test control file says so
1611         #       (assuming we have any tests)
1612
1613         class NeedBuildException: pass
1614         def build_needed(m):
1615                 debug_b('build needed for %s' % m)
1616                 raise NeedBuildException()
1617
1618         try:
1619                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1620                                       act.ah['deb_fortests'] != 'ignore'):
1621                         build_needed('binaries')
1622
1623                 result_pwd_af = InputDir(what+'-treeforcontrol',
1624                         result_pwd, True)
1625                 stanzas = read_control(act, result_pwd_af, control_override)
1626                 for stanza in stanzas:
1627                         for t in stanza[' tests']:
1628                                 if 'no-build-needed' not in t.feature_names:
1629                                         build_needed('test %s' % t.tname)
1630                                 for d in t.depends:
1631                                         if '@' in d:
1632                                                 build_needed('test %s '
1633                                                  'dependency %s' % (t.tname,d))
1634
1635                 debug_b('build not needed')
1636                 built = False
1637
1638         except NeedBuildException:
1639
1640                 if act.kind != 'dsc':
1641                         testbed.prepare2([])
1642
1643                 tb.satisfy_dependencies_string('build-essential',
1644                                 'install build-essential')
1645                 tb.satisfy_dependencies_dsc(dsc, 'build dependencies')
1646
1647                 script = tmpdir_script + [
1648                         'cd "$2"',
1649                         'dpkg-checkbuilddeps',
1650                         opts.user_wrap('debian/rules build'),
1651                         ]
1652                 source_rules_command(act,script,what,'build',work,
1653                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1654
1655                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1656                         badpkg("results dir `%s' is not in expected parent"
1657                                " dir `%s'" % (result_pwd, work.read(True)))
1658
1659                 built = True
1660
1661         act.tests_tree = InputDir(what+'-tests-tree',
1662                                 work.read(True)+os.path.basename(result_pwd),
1663                                 True)
1664         if act.ah['dsc_tests']:
1665                 testbed.register_ephemeral(act.work)
1666                 testbed.register_ephemeral(act.tests_tree)
1667
1668         if not built:
1669                 act.blamed = []
1670                 return
1671
1672         act.blamed = copy.copy(testbed.blamed)
1673
1674         debug_b('filter=%s' % filter)
1675         if filter != '_':
1676                 script = tmpdir_script + [
1677                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1678                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1679                         'cd ..',
1680                         'echo *.deb >&3',
1681                         ]
1682                 result_debs = source_rules_command(act,script,what,
1683                                 'binary',work,work.write(True),
1684                                 results_lines=1, xargs=['x',tmpdir])
1685                 if result_debs == '*.deb': debs = []
1686                 else: debs = result_debs.split(' ')
1687                 debug_b('debs='+`debs`)
1688                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1689                 for deb in debs:
1690                         m = re.match(deb)
1691                         if not m: badpkg("badly-named binary `%s'" % deb)
1692                         pkg = m.groups()[0]
1693                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1694                         for pat in filter.split(','):
1695                                 debug_b('  pat=%s' % pat)
1696                                 if not fnmatch.fnmatchcase(pkg,pat):
1697                                         debug_b('   no match')
1698                                         continue
1699                                 deb_what = pkg+'_'+what+'.deb'
1700                                 bin = RelativeInputFile(deb_what,work,deb,True)
1701                                 debug_b('  deb_what=%s, bin=%s' %
1702                                         (deb_what, str(bin)))
1703                                 binaries.register(act,pkg,bin,
1704                                         'forbuilds',testbed.blamed)
1705                                 act.binaries.append((pkg,bin))
1706                                 break
1707                 debug_b('all done.')
1708
1709 #---------- main processing loop and main program
1710
1711 def process_actions():
1712         global binaries
1713
1714         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1715         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1716         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1717
1718         debug_a1('starting')
1719         testbed.open()
1720         binaries = Binaries()
1721
1722         for act in opts.actions:
1723                 if act.af is not None and not act.af.spec_tbp:
1724                         testbed.register_ephemeral(act.af)
1725
1726         binaries.reset()
1727         control_override = None
1728
1729         debug_a1('builds ...')
1730         for act in opts.actions:
1731                 debug_a2('%s %s' %
1732                         (act.kind, act.what))
1733
1734                 if act.kind == 'control':
1735                         control_override = act.af
1736                 if act.kind == 'deb':
1737                         testbed.blame('arg:'+act.af.spec)
1738                         determine_package(act)
1739                         testbed.blame('deb:'+act.pkg)
1740                         binaries.register(act,act.pkg,act.af,
1741                                 'forbuilds',testbed.blamed)
1742                 if act.kind == 'dsc' or act.kind == 'ubtree':
1743                         build_source(act, control_override)
1744                 if act.kind == 'tree':
1745                         act.binaries = []
1746                 if act.kind.endswith('tree') or act.kind == 'dsc':
1747                         control_override = None
1748                 if act.kind == 'instantiate':
1749                         pass
1750
1751         debug_a1('builds done.')
1752
1753         binaries.reset()
1754         control_override = None
1755
1756         debug_a1('tests ...')
1757         for act in opts.actions:
1758                 debug_a2('test %s %s' % (act.kind, act.what))
1759
1760                 testbed.needs_reset()
1761                 if act.kind == 'control':
1762                         control_override = act.af
1763                 if act.kind == 'deb':
1764                         binaries.register(act,act.pkg,act.af,'fortests',
1765                                 ['deb:'+act.pkg])
1766                 if act.kind == 'dsc' or act.kind == 'ubtree':
1767                         for (pkg,bin) in act.binaries:
1768                                 binaries.register(act,pkg,bin,'fortests',
1769                                         act.blamed)
1770                 if act.kind == 'dsc':
1771                         if act.ah['dsc_tests']:
1772                                 debug_a3('read control ...')
1773                                 stanzas = read_control(act, act.tests_tree,
1774                                                 control_override)
1775                                 testbed.blamed += act.blamed
1776                                 debug_a3('run_tests ...')
1777                                 run_tests(stanzas, act.tests_tree)
1778                         control_override = None
1779                 if act.kind == 'tree' or act.kind == 'ubtree':
1780                         testbed.blame('arg:'+act.af.spec)
1781                         stanzas = read_control(act, act.af, control_override)
1782                         debug_a3('run_tests ...')
1783                         run_tests(stanzas, act.af)
1784                         control_override = None
1785                 if act.kind == 'instantiate':
1786                         testbed.prepare([])
1787         debug_a1('tests done.')
1788
1789 def main():
1790         global testbed
1791         global tmpdir
1792         try:
1793                 parse_args()
1794         except SystemExit, se:
1795                 os._exit(20)
1796         try:
1797                 setup_trace()
1798                 testbed = Testbed()
1799                 testbed.start()
1800                 finalise_options()
1801                 process_actions()
1802         except:
1803                 ec = print_exception(sys.exc_info(), '')
1804                 cleanup()
1805                 os._exit(ec)
1806         cleanup()
1807         os._exit(errorcode)
1808
1809 main()