runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         try: shutil.rmtree(pathname)
 192         except (IOError,OSError), oe:
 193                 if oe.errno != errno.EEXIST: raise
 194
 195 def debug_subprocess(what, cmdl=None, script=None):
 196         o = '$ '+what+':'
 197         if cmdl is not None:
 198                 ol = []
 199                 for x in cmdl:
 200                         if x is script: x = '<SCRIPT>'
 201                         ol.append(x.    replace('\\','\\\\').
 202                                         replace(' ','\\ ')      )
 203                 o += ' '+ ' '.join(ol)
 204         debug(o)
 205         if script is not None and opts.debuglevel >= 1:
 206                 o = ''
 207                 for l in script.rstrip('\n').split('\n'):
 208                         o += '$     '+l+'\n'
 209                 debug(o,1)
 210
 211 def flatten(l):
 212         return reduce((lambda a,b: a + b), l, [])
 213
 214 #---------- fancy automatic file-copying class
 215
 216 class AutoFile:
 217         # p.what
 218         # p.path[tb]    None or path    not None => path known
 219         # p.file[tb]    None or path    not None => file exists
 220         # p.spec        string or None
 221         # p.spec_tbp    True or False, or not set if spec is None
 222         # p.dir         '' or '/'
 223
 224  def __init__(p, what):
 225         p.what = what
 226         p.path = [None,None]
 227         p.file = [None,None]
 228         p.spec = None
 229         p.dir = ''
 230
 231  def __repr__(p):
 232         return "<AF@%s>" % p.__str__()
 233  def __str__(p):
 234         def ptbp(tbp):
 235                 if p.path[tbp] is None: return '-'+p.dir
 236                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 237                 else: return p.path[tbp]+p.dir+'!'
 238         out = p.what
 239         if p.spec is not None:
 240                 if not hasattr(p,'spec_tb'): out += '~'
 241                 elif p.spec_tbp: out += '#'
 242                 else: out += '='
 243                 out += p.spec
 244         out += ':'
 245         out += ptbp(False)
 246         out += '|'
 247         out += ptbp(True)
 248         return out
 249
 250  def _wrong(p, how):
 251         xtra = ''
 252         if p.spec is not None:
 253                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 254         raise ("internal error: %s (%s)" % (how, str(p)))
 255
 256  def _ensure_path(p, tbp):
 257         if p.path[tbp] is None:
 258                 if '/' in p.what:
 259                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 260                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 261                 if not tbp:
 262                         p.path[tbp] = tmpdir+'/'+p.what
 263                 else:
 264                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 265
 266  def write(p, tbp=False):
 267         p._debug('write %s...' % 'HT'[tbp])
 268         p._ensure_path(tbp)
 269
 270         if p.dir and not p.file[tbp]:
 271                 if not tbp:
 272                         p._debug('mkdir H')
 273                         mkdir_okexist(p.path[tbp])
 274                 else:
 275                         cmdl = ['sh','-ec',
 276                                 'test -d "$1" || mkdir -p "$1"',
 277                                 'x', p.path[tbp]]
 278                         tf_what = urllib.quote(p.what).replace('/','%2F')
 279                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 280                         if rc: bomb('failed to create directory %s' %
 281                                 p.path[tbp])
 282
 283         p.file[tbp] = p.path[tbp]
 284         return p.path[tbp]
 285
 286  def read(p, tbp=False):
 287         p._debug('read %s...' % 'HT'[tbp])
 288         p._ensure_path(tbp)
 289
 290         if p.file[tbp] is None:
 291                 if p.file[not tbp] is None:
 292                         p._wrong("requesting read but nonexistent")
 293                 cud = ['copyup','copydown'][tbp]
 294                 src = p.file[not tbp] + p.dir
 295                 dst = p.path[tbp] + p.dir
 296                 testbed.command(cud, (src, dst))
 297                 p.file[tbp] = p.path[tbp]
 298
 299         return p.file[tbp] + p.dir
 300
 301  def invalidate(p, tbp=False):
 302         p.file[tbp] = None
 303         p._debug('invalidated %s' % 'HT'[tbp])
 304
 305  def _debug(p, m):
 306         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 307
 308  def _constructed(p):
 309         p._debug('constructed: '+str(p))
 310         p._check()
 311
 312  def _check(p):
 313         for tbp in [False,True]:
 314          for pf in [p.path, p.file]:
 315                 if pf[tbp] is None: continue
 316                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 317                 if p.dir and pf[tbp].endswith('/'):
 318                         pf[tbp] = pf[tbp].rstrip('/')
 319                         if not pf[tbp]: pf[tbp] = '/'
 320                 if not p.dir and pf[tbp].endswith('/'):
 321                         bomb("directory `%s' specified for "
 322                              "non-directory %s" % (pf[tbp], p.what))
 323
 324  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 325         AutoFile.__init__(p,what)
 326         sh_on = ''; sh_sibl = ''
 327         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 328         if sibling: sh_sibl=' (sibling)'
 329         parent._debug('using as base: %s: %s%s%s' %
 330                         (str(parent), leaf, sh_on, sh_sibl))
 331         if not sibling and not parent.dir:
 332                 parent._wrong('asked for non-sibling relative path of non-dir')
 333         if sibling: trim = os.path.dirname
 334         else: trim = lambda x: x
 335         for tbp in [False,True]:
 336                 if parent.path[tbp] is None: continue
 337                 trimmed = trim(parent.path[tbp])
 338                 if trimmed: trimmed += '/'
 339                 p.path[tbp] = trimmed + leaf
 340                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 341                         p.file[tbp] = p.path[tbp]
 342
 343 class InputFile(AutoFile):
 344  def _init(p, what, spec, spec_tbp=False):
 345         AutoFile.__init__(p, what)
 346         p.spec = spec
 347         p.spec_tbp = spec_tbp
 348         p.path[spec_tbp] = spec
 349         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 350  def __init__(p, what, spec, spec_tbp=False):
 351         p._init(what,spec,spec_tbp)
 352         p._constructed()
 353
 354 class InputDir(InputFile):
 355  def __init__(p, what, spec, spec_tbp=False):
 356         InputFile._init(p,what,spec,spec_tbp)
 357         p.dir = '/'
 358         p._constructed()
 359
 360 class OutputFile(AutoFile):
 361  def _init(p, what, spec, spec_tbp=False):
 362         AutoFile.__init__(p, what)
 363         p.spec = spec
 364         p.spec_tbp = spec_tbp
 365         p.path[spec_tbp] = spec
 366  def __init__(p, what, spec, spec_tbp=False):
 367         p._init(what,spec,spec_tbp)
 368         p._constructed()
 369
 370 class OutputDir(OutputFile):
 371  def __init__(p, what, spec, spec_tbp=False):
 372         OutputFile._init(p,what,spec,spec_tbp)
 373         p.dir = '/'
 374         p._constructed()
 375
 376 class RelativeInputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 378         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 379         p._constructed()
 380
 381 class RelativeOutputFile(AutoFile):
 382  def __init__(p, what, parent, leaf, sibling=False):
 383         p._relative_init(what, parent, leaf, None, False, sibling)
 384         p._constructed()
 385
 386 class TemporaryFile(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p._constructed()
 390
 391 class TemporaryDir(AutoFile):
 392  def __init__(p, what):
 393         AutoFile.__init__(p, what)
 394         p.dir = '/'
 395         p._constructed()
 396
 397 #---------- parsing and representation of the arguments
 398
 399 class Action:
 400  def __init__(a, kind, af, arghandling, what):
 401         # extra attributes get added during processing
 402         a.kind = kind
 403         a.af = af
 404         a.ah = arghandling
 405         a.what = what
 406         a.missing_tests_control = False
 407  def __repr__(a):
 408         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 409
 410 def parse_args():
 411         global opts, timeouts
 412         global n_non_actions # argh, stupid python scoping rules
 413         usage = "%prog <options> --- <virt-server>..."
 414         parser = OptionParser(usage=usage)
 415         pa = parser.add_option
 416         pe = parser.add_option
 417
 418         arghandling = {
 419                 'dsc_tests': True,
 420                 'dsc_filter': '*',
 421                 'deb_forbuilds': 'auto',
 422                 'deb_fortests': 'auto',
 423                 'tb': False,
 424                 'override_control': None,
 425                 'set_lang': 'C'
 426         }
 427         initial_arghandling = arghandling.copy()
 428         n_non_actions = 0
 429
 430         #----------
 431         # actions (ie, test sets to run, sources to build, binaries to use):
 432
 433         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 434                 global n_non_actions
 435                 parser.largs.append((value,kindpath))
 436                 n_non_actions += not(is_act)
 437
 438         def pa_action(long, metavar, kindpath, help, is_act=True):
 439                 pa('','--'+long, action='callback', callback=cb_action,
 440                         nargs=1, type='string',
 441                         callback_args=(long,kindpath,is_act), help=help)
 442
 443         pa_action('built-tree',         'TREE', '@/',
 444                 help='run tests from build tree TREE')
 445
 446         pa_action('unbuilt-tree',       'TREE', '@//',
 447                 help='run tests from build tree TREE')
 448
 449         pa_action('source',             'DSC', '@.dsc',
 450                 help='build DSC and use its tests and/or'
 451                     ' generated binary packages')
 452
 453         pa_action('binary',             'DEB', '@.deb',
 454                help='use binary package DEB according'
 455                     ' to most recent --binaries-* settings')
 456
 457         def cb_actnoarg(op,optstr,value,parser, largsentry):
 458                 parser.largs.append(largsentry)
 459         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 460                 callback_args=((None, ('instantiate',)),),
 461                 help='instantiate testbed now (during testing phase)'
 462                      ' and install packages'
 463                      ' selected for automatic installation, even'
 464                      ' if this might apparently not be required otherwise')
 465
 466         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 467                help='run tests from control file CONTROL instead,'
 468                     ' (applies to next test suite only)')
 469
 470         #----------
 471         # argument handling settings (what ways to use action
 472         #  arguments, and pathname processing):
 473
 474         def cb_setah(option, opt_str, value, parser, toset,setval):
 475                 if type(setval) == list:
 476                         if not value in setval:
 477                                 parser.error('value for %s option (%s) is not '
 478                                  'one of the permitted values (%s)' %
 479                                  (value, opt_str, setval.join(' ')))
 480                 elif setval is not None:
 481                         value = setval
 482                 for v in toset:
 483                         arghandling[v] = value
 484                 parser.largs.append(arghandling.copy())
 485
 486         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 487                 type = metavar
 488                 if type is not None: type = 'string'
 489                 pa('',long, action='callback', callback=cb_setah,
 490                    callback_args=(affected,effect), **kwargs)
 491
 492         #---- paths: host or testbed:
 493         #
 494         pa_setah('--paths-testbed', ['tb'],True,
 495                 help='subsequent path specifications refer to the testbed')
 496         pa_setah('--paths-host', ['tb'],False,
 497                 help='subsequent path specifications refer to the host')
 498
 499         #---- source processing settings:
 500
 501         pa_setah('--sources-tests', ['dsc_tests'],True,
 502                 help='run tests from builds of subsequent sources')
 503         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 504                 help='do not run tests from builds of subsequent sources')
 505
 506         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 507                 type='string', metavar='PATTERN-LIST',
 508                 help='from subsequent sources, use binaries matching'
 509                      ' PATTERN-LIST (comma-separated glob patterns)'
 510                      ' according to most recent --binaries-* settings')
 511         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 512                 help='from subsequent sources, do not use any binaries')
 513         #---- binary package processing settings:
 514
 515         def pa_setahbins(long,toset,how):
 516          pa_setah(long, toset,['ignore','auto','install'],
 517                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 518                 help=how+' ignore binaries, install them as needed'
 519                         ' for dependencies, or unconditionally install'
 520                         ' them, respectively')
 521         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 522         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 523         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 524
 525         #----------
 526         # general options:
 527
 528         def cb_vserv(op,optstr,value,parser):
 529                 parser.values.vserver = list(parser.rargs)
 530                 del parser.rargs[:]
 531
 532         def cb_path(op,optstr,value,parser, constructor,long,dir):
 533                 name = long.replace('-','_')
 534                 af = constructor(long, value,arghandling['tb'])
 535                 setattr(parser.values, name, af)
 536
 537         def pa_path(long, constructor, help, dir=False):
 538                 pa('','--'+long, action='callback', callback=cb_path,
 539                         callback_args=(constructor,long,dir),
 540                         nargs=1, type='string',
 541                         help=help, metavar='PATH')
 542
 543         pa_path('output-dir', OutputDir, dir=True,
 544                 help='write stderr/out files in PATH')
 545
 546         pa('--leave-lang', dest='set_lang', action='store_false',
 547                 help="leave LANG on testbed set to testbed's default")
 548         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 549                 help='set LANG on testbed to LANGVAL', default='C')
 550
 551         pa('','--tmp-dir',              type='string', dest='tmpdir',
 552                 help='write temporary files to TMPDIR, emptying it'
 553                      ' beforehand and leaving it behind at the end')
 554         pa('','--log-file',             type='string', dest='logfile',
 555                 help='write the log LOGFILE, emptying it beforehand,'
 556                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 557         pa('','--summary-file',         type='string', dest='summary',
 558                 help='write a summary report to SUMMARY,'
 559                      ' emptying it beforehand')
 560
 561         for k in timeouts.keys():
 562                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 563                         metavar='T', help='set %s timeout to T')
 564         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 565                         metavar='FACTOR', default=1.0,
 566                         help='multiply all default timeouts by FACTOR')
 567
 568         pa('','--user',                 type='string', dest='user',
 569                 help='run tests as USER (needs root on testbed)')
 570         pa('','--gain-root',            type='string', dest='gainroot',
 571                 help='prefix debian/rules binary with GAINROOT')
 572         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 573         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 574         pa('','--gnupg-home',           type='string', dest='gnupghome',
 575                 default='~/.autopkgtest/gpg',
 576                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 577                         " signing private apt archive);"
 578                         " `fresh' means generate new key each time.")
 579
 580         #----------
 581         # actual meat:
 582
 583         class SpecialOption(optparse.Option): pass
 584         vs_op = SpecialOption('','--VSERVER-DUMMY')
 585         vs_op.action = 'callback'
 586         vs_op.type = None
 587         vs_op.default = None
 588         vs_op.nargs = 0
 589         vs_op.callback = cb_vserv
 590         vs_op.callback_args = ( )
 591         vs_op.callback_kwargs = { }
 592         vs_op.help = 'introduces virtualisation server and args'
 593         vs_op._short_opts = []
 594         vs_op._long_opts = ['---']
 595
 596         pa(vs_op)
 597
 598         (opts,args) = parser.parse_args()
 599         if not hasattr(opts,'vserver'):
 600                 parser.error('you must specifiy --- <virt-server>...')
 601         if n_non_actions >= len(parser.largs):
 602                 parser.error('nothing to do specified')
 603
 604         for k in timeouts.keys():
 605                 t = getattr(opts,'timeout_'+k)
 606                 if t is None: t = timeouts[k] * opts.timeout_factor
 607                 timeouts[k] = int(t)
 608
 609         arghandling = initial_arghandling
 610         opts.actions = []
 611         ix = 0
 612         for act in args:
 613                 if type(act) == dict:
 614                         arghandling = act
 615                         continue
 616                 elif type(act) == tuple:
 617                         pass
 618                 elif type(act) == str:
 619                         act = (act,act)
 620                 else:
 621                         raise ("unknown action in list `%s' having"
 622                               " type `%s'" % (act, type(act)))
 623                 (pathstr, kindpath) = act
 624
 625                 constructor = InputFile
 626                 if type(kindpath) is tuple:             kind = kindpath[0]
 627                 elif kindpath.endswith('.deb'):         kind = 'deb'
 628                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 629                 elif kindpath.endswith('//'):
 630                         kind = 'ubtree'
 631                         constructor = InputDir
 632                 elif kindpath.endswith('/'):
 633                         kind = 'tree'
 634                         constructor = InputDir
 635                 else: parser.error("do not know how to handle filename `%s';"
 636                         " specify --source --binary or --build-tree" %
 637                         kindpath)
 638
 639                 what = '%s%s' % (kind,ix); ix += 1
 640
 641                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 642                 else: fwhatx = '-'+kind
 643
 644                 if pathstr is None: af = None
 645                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 646                 opts.actions.append(Action(kind, af, arghandling, what))
 647
 648 def setup_trace():
 649         global trace_stream, tmpdir, summary_stream
 650
 651         if opts.tmpdir is not None:
 652                 rmtree('tmpdir(specified)',opts.tmpdir)
 653                 mkdir_okexist(opts.tmpdir, 0700)
 654                 tmpdir = opts.tmpdir
 655         else:
 656                 assert(tmpdir is None)
 657                 tmpdir = tempfile.mkdtemp()
 658
 659         if opts.logfile is None:
 660                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 661                         opts.logfile = opts.output_dir.spec + '/log'
 662                 elif opts.tmpdir is not None:
 663                         opts.logfile = opts.tmpdir + '/log'
 664         if opts.logfile is not None:
 665                 trace_stream = open(opts.logfile, 'w', 0)
 666         if opts.summary is not None:
 667                 summary_stream = open(opts.summary, 'w', 0)
 668
 669         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 670
 671 def finalise_options():
 672         global opts, tb, build_essential
 673
 674         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 675                 opts.user = ''
 676
 677         if opts.user is None:
 678                 su = 'suggested-normal-user='
 679                 ul = [
 680                         e[len(su):]
 681                         for e in testbed.caps
 682                         if e.startswith(su)
 683                         ]
 684                 if ul:
 685                         opts.user = ul[0]
 686                 else:
 687                         opts.user = ''
 688
 689         if opts.user:
 690                 if 'root-on-testbed' not in testbed.caps:
 691                         pstderr("warning: virtualisation"
 692                                 " system does not offer root on testbed,"
 693                                 " but --user option specified: failure likely")
 694                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 695         else:
 696                 opts.user_wrap = lambda x: x
 697
 698         if opts.gainroot is None:
 699                 opts.gainroot = ''
 700                 if (opts.user or
 701                     'root-on-testbed' not in testbed.caps):
 702                         opts.gainroot = 'fakeroot'
 703                         build_essential += ['fakeroot']
 704
 705         if opts.gnupghome.startswith('~/'):
 706                 try: home = os.environ['HOME']
 707                 except KeyError:
 708                         parser.error("HOME environment variable"
 709                                 " not set, needed for --gnupghome=`%s"
 710                                 % opts.gnupghome)
 711                 opts.gnupghome = home + opts.gnupghome[1:]
 712         elif opts.gnupghome == 'fresh':
 713                 opts.gnupghome = None
 714
 715 #---------- testbed management - the Testbed class
 716
 717 class Testbed:
 718  def __init__(tb):
 719         tb.sp = None
 720         tb.lastsend = None
 721         tb.scratch = None
 722         tb.modified = False
 723         tb.blamed = []
 724         tb._ephemeral = []
 725         tb._debug('init')
 726         tb._need_reset_apt = False
 727  def _debug(tb, m, minlevel=0):
 728         debug('** '+m, minlevel)
 729  def start(tb):
 730         tb._debug('start')
 731         p = subprocess.PIPE
 732         debug_subprocess('vserver', opts.vserver)
 733         tb._errplumb = Errplumb(True)
 734         tb.sp = subprocess.Popen(opts.vserver,
 735                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 736         tb.expect('ok')
 737         tb.caps = tb.commandr('capabilities')
 738         if not 'print-execute-command' in tb.caps:
 739                 tb.bomb('testbed does not support print-execute-command')
 740  def stop(tb):
 741         tb._debug('stop')
 742         tb.close()
 743         if tb.sp is None: return
 744         ec = tb.sp.returncode
 745         if ec is None:
 746                 tb.sp.stdout.close()
 747                 tb.send('quit')
 748                 tb.sp.stdin.close()
 749                 ec = tb.sp.wait()
 750         if ec:
 751                 tb.bomb('testbed gave exit status %d after quit' % ec)
 752         tb._errplumb.wait()
 753  def open(tb):
 754         tb._debug('open, scratch=%s' % tb.scratch)
 755         if tb.scratch is not None: return
 756         pl = tb.commandr('open')
 757         tb.scratch = InputDir('tb-scratch', pl[0], True)
 758         tb.deps_processed = []
 759         tb._auxverbscript_make()
 760
 761  def _auxverbscript_make(tb):
 762         pec = tb.commandr('print-execute-command')
 763         if len(pec) < 2: tb.bomb('too few results from print-execute-command')
 764         tb.ec_cmdl = map(urllib.unquote, pec[0].split(','))
 765         tb.ec_mode = urllib.unquote(pec[1])
 766         tb.ec_infos = map(urllib.unquote,pec[2:])
 767
 768         shellquote_re = regexp.compile(r'([\\"$`])')
 769         def shellquote_arg(s): return '"' + shellquote_re.sub(r'\\\1', s) + '"'
 770         def shellquote_cmdl(l): return ' '.join(map(shellquote_arg,l))
 771
 772         tb._debug('tb.ec_cmdl = %s' % (`tb.ec_cmdl`))
 773
 774         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 775         print >>open(tb.ec_auxverbscript.write(),'w'), (
 776 '''#!/bin/sh
 777 set -ex
 778 echo >&2 ": $*"
 779 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 780         set -- dpkg --print-architecture
 781 fi
 782 if [ "x$1" = xsh ] && [ "x$2" = x-c ]; then
 783         shift; shift
 784         # what a horrible hack!
 785 fi
 786 exec '''+shellquote_cmdl(tb.ec_cmdl)+' "$*"'+"\n"
 787                 )
 788         os.chmod(tb.ec_auxverbscript.write(), 0755)
 789
 790  def mungeing_apt(tb):
 791         if not 'revert' in tb.caps:
 792                 tb._need_reset_apt = True
 793  def reset_apt(tb):
 794         if not tb._need_reset_apt: return
 795         what = 'aptget-update-reset'
 796         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 797         rc = tb.execute(what, cmdl, kind='install')
 798         if rc:
 799                 pstderr("\n" "warning: failed to restore"
 800                         " testbed apt cache, exit code %d" % rc)
 801         tb._need_reset_apt = False
 802  def close(tb):
 803         tb._debug('close, scratch=%s' % tb.scratch)
 804         if tb.scratch is None: return
 805         tb.scratch = None
 806         if tb.sp is None: return
 807         tb.command('close')
 808  def prepare1(tb, deps_new):
 809         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 810                 (tb.modified, tb.deps_processed, deps_new), 1)
 811         if 'revert' in tb.caps and (tb.modified or
 812             [d for d in tb.deps_processed if d not in deps_new]):
 813                 for af in tb._ephemeral: af.read(False)
 814                 tb._debug('reset **')
 815                 tb.command('revert')
 816                 tb.blamed = []
 817                 for af in tb._ephemeral: af.invalidate(True)
 818                 tb._auxverbscript_make()
 819         tb.modified = False
 820  def prepare2(tb, deps_new):
 821         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 822         binaries.publish()
 823         tb._install_deps(deps_new)
 824  def prepare(tb, deps_new):
 825         tb.prepare1(deps_new)
 826         tb.prepare2(deps_new)
 827  def register_ephemeral(tb, af):
 828         tb._ephemeral.append(af)
 829  def _install_deps(tb, deps_new):
 830         tb._debug(' installing dependencies '+`deps_new`, 1)
 831         tb.deps_processed = deps_new
 832         if not deps_new: return
 833         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 834  def needs_reset(tb):
 835         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 836         tb.modified = True
 837  def blame(tb, m):
 838         tb._debug('blame += %s' % m, 1)
 839         tb.blamed.append(m)
 840  def bomb(tb, m):
 841         tb._debug('bomb %s' % m)
 842         if tb.sp is not None:
 843                 tb.sp.stdout.close()
 844                 tb.sp.stdin.close()
 845                 ec = tb.sp.wait()
 846                 if ec: pstderr('adt-run: testbed failing,'
 847                                 ' exit status %d' % ec)
 848         tb.sp = None
 849         raise Quit(16, 'testbed failed: %s' % m)
 850  def send(tb, string):
 851         tb.sp.stdin
 852         try:
 853                 debug('>> '+string, 2)
 854                 print >>tb.sp.stdin, string
 855                 tb.sp.stdin.flush()
 856                 tb.lastsend = string
 857         except:
 858                 (type, value, dummy) = sys.exc_info()
 859                 tb.bomb('cannot send to testbed: %s' % traceback.
 860                         format_exception_only(type, value))
 861  def expect(tb, keyword, nresults=None):
 862         l = tb.sp.stdout.readline()
 863         if not l: tb.bomb('unexpected eof from the testbed')
 864         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 865         l = l.rstrip('\n')
 866         debug('<< '+l, 2)
 867         ll = l.split()
 868         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 869         if ll[0] != keyword:
 870                 if tb.lastsend is None:
 871                         tb.bomb("got banner `%s', expected `%s...'" %
 872                                 (l, keyword))
 873                 else:
 874                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 875                                 (tb.lastsend, l, keyword))
 876         ll = ll[1:]
 877         if nresults is not None and len(ll) != nresults:
 878                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 879                         " expected %d result parameters" %
 880                         (string, l, len(ll), nresults))
 881         return ll
 882  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 883         # pass args=[None,...] or =(None,...) to avoid more url quoting
 884         if type(cmd) is str: cmd = [cmd]
 885         if len(args) and args[0] is None: args = args[1:]
 886         else: args = map(urllib.quote, args)
 887         al = cmd + args
 888         tb.send(string.join(al))
 889         ll = tb.expect('ok', nresults)
 890         if unquote: ll = map(urllib.unquote, ll)
 891         return ll
 892  def command(tb, cmd, args=()):
 893         tb.commandr(cmd, args, 0)
 894  def commandr1(tb, cmd, args=()):
 895         rl = tb.commandr(cmd, args, 1)
 896         return rl[0]
 897  def execute(tb, what, cmdl,
 898                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 899                 script=False, tmpdir=None, kind='short'):
 900         # Options for script:
 901         #   False - do not call debug_subprocess, no synch. reporting required
 902         #   None or string - call debug_subprocess with that value,
 903         #                       plumb stderr through synchronously if possible
 904         # Options for se:
 905         #   None - usual Errplumb (output is of kind 2c)
 906         #   string - path on testbed (output is of kind 2d)
 907
 908         timeout = timeouts[kind]
 909
 910         if script is not False: debug_subprocess(what, cmdl, script=script)
 911         if cwd is None: cwd = tb.scratch.write(True)
 912
 913         xdump = None
 914         if se is None:
 915                 ep = Errplumb()
 916                 se_catch = TemporaryFile(what+'-xerr')
 917                 se_use = se_catch.write(True)
 918                 if not opts.quiet: xdump = 'debug=2-2'
 919                 elif trace_stream is not None:
 920                         xdump = 'debug=2-%d' % trace_stream.fileno()
 921         else:
 922                 ep = None
 923                 se_catch = None
 924                 se_use = se
 925
 926         cmdl = [None,
 927                 ','.join(map(urllib.quote, cmdl)),
 928                 si, so, se_use, cwd]
 929
 930         if timeout is not None and timeout > 0:
 931                 cmdl.append('timeout=%d' % timeout)
 932
 933         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 934         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 935         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 936         if opts.set_lang is not False:
 937                 cmdl.append('env=LANG=%s' % opts.set_lang)
 938
 939         rc = tb.commandr1('execute', cmdl)
 940         try: rc = int(rc)
 941         except ValueError: bomb("execute for %s gave invalid response `%s'"
 942                                         % (what,rc))
 943
 944         if se_catch is not None:
 945                 debug_file(se_catch.read())
 946         if ep is not None:
 947                 ep.wait()
 948
 949         return rc
 950
 951  def satisfy_dependencies_string(tb, deps, what):
 952         # Must have called Binaries.configure_apt
 953         debug('dependencies: %s: satisfying %s' % (what,deps))
 954         dsc = TemporaryFile('deps.dsc')
 955         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 956         # pbuilder-satisfydepends has a bug where it ignores the
 957         #  Build-Depends if it's the last line in the dsc (#635696)
 958         tb.satisfy_dependencies_dsc(dsc, what)
 959
 960  def satisfy_dependencies_dsc(tb, dsc, what):
 961         # Must have called Binaries.configure_apt
 962         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 963                 '--binary-all', # --check-key
 964                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 965                 '-c',dsc.read()
 966         ]
 967         # The --internal-chrootexec option is really handy but
 968         #  perhaps we are not supposed to use it ?  See also #635697.
 969         debug('dependencies: %s: running %s' % (what,`cmdl`))
 970         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 971         if rc: badpkg('dependency install failed, exit code %d' % rc)
 972
 973 #---------- representation of test control files: Field*, Test, etc.
 974
 975 class FieldBase:
 976  def __init__(f, fname, stz, base, tnames, vl):
 977         assert(vl)
 978         f.stz = stz
 979         f.base = base
 980         f.tnames = tnames
 981         f.vl = vl
 982  def words(f):
 983         def distribute(vle):
 984                 (lno, v) = vle
 985                 r = v.split()
 986                 r = map((lambda w: (lno, w)), r)
 987                 return r
 988         return flatten(map(distribute, f.vl))
 989  def atmostone(f):
 990         if len(vl) == 1:
 991                 (f.lno, f.v) = vl[0]
 992         else:
 993                 raise Unsupported(f.vl[1][0],
 994                         'only one %s field allowed' % fn)
 995         return f.v
 996
 997 class FieldIgnore(FieldBase):
 998  def parse(f): pass
 999
1000 class Restriction:
1001  def __init__(r,rname,base): pass
1002
1003 class Restriction_rw_build_tree(Restriction): pass
1004 class Restriction_breaks_testbed(Restriction):
1005  def __init__(r, rname, base):
1006         if 'revert' not in testbed.caps:
1007                 raise Unsupported(f.lno,
1008                         'Test breaks testbed but testbed cannot revert')
1009 class Restriction_needs_root(Restriction):
1010  def __init__(r, rname, base):
1011         if 'root-on-testbed' not in testbed.caps:
1012                 raise Unsupported(f.lno,
1013                         'Test needs root on testbed which is not available')
1014
1015 class Field_Restrictions(FieldBase):
1016  def parse(f):
1017         for wle in f.words():
1018                 (lno, rname) = wle
1019                 nrname = rname.replace('-','_')
1020                 try: rclass = globals()['Restriction_'+nrname]
1021                 except KeyError: raise Unsupported(lno,
1022                         'unknown restriction %s' % rname)
1023                 r = rclass(nrname, f.base)
1024                 f.base['restriction_names'].append(rname)
1025                 f.base['restrictions'].append(r)
1026
1027 class Field_Features(FieldIgnore):
1028  def parse(f):
1029         for wle in f.words():
1030                 (lno, fname) = wle
1031                 f.base['feature_names'].append(fname)
1032                 nfname = fname.replace('-','_')
1033                 try: fclass = globals()['Feature_'+nfname]
1034                 except KeyError: continue
1035                 ft = fclass(nfname, f.base)
1036                 f.base['features'].append(ft)
1037
1038 class Field_Tests(FieldIgnore): pass
1039
1040 class Field_Depends(FieldBase):
1041  def parse(f):
1042         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
1043         dl = map(lambda x: x.strip(),
1044                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1045         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1046         for d in dl:
1047                 if re.search(d):
1048                         badpkg("Test Depends field contains dependency"
1049                                " `%s' with invalid characters" % d)
1050         f.base['depends'] = dl
1051
1052 class Field_Tests_directory(FieldBase):
1053  def parse(f):
1054         td = atmostone(f)
1055         if td.startswith('/'): raise Unspported(f.lno,
1056                 'Tests-Directory may not be absolute')
1057         f.base['testsdir'] = td
1058
1059 def run_tests(stanzas, tree):
1060         global errorcode, testbed
1061         if stanzas == ():
1062                 report('*', 'SKIP no tests in this package')
1063                 errorcode |= 8
1064         for stanza in stanzas:
1065                 tests = stanza[' tests']
1066                 if not tests:
1067                         report('*', 'SKIP package has metadata but no tests')
1068                         errorcode |= 8
1069                 for t in tests:
1070                         t.prepare()
1071                         t.run(tree)
1072                         if 'breaks-testbed' in t.restriction_names:
1073                                 testbed.needs_reset()
1074                 testbed.needs_reset()
1075
1076 class Test:
1077  def __init__(t, tname, base, act):
1078         if '/' in tname: raise Unsupported(base[' lno'],
1079                 'test name may not contain / character')
1080         for k in base: setattr(t,k,base[k])
1081         t.tname = tname
1082         t.act = act
1083         t.what = act.what+'t-'+tname
1084         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1085         else: t.path = tname
1086         t._debug('constructed; path=%s' % t.path)
1087         t._debug(' .depends=%s' % t.depends)
1088  def _debug(t, m):
1089         debug('& %s: %s' % (t.what, m))
1090  def report(t, m):
1091         report(t.what, m)
1092  def reportfail(t, m):
1093         global errorcode
1094         errorcode |= 4
1095         report(t.what, 'FAIL ' + m)
1096  def prepare(t):
1097         t._debug('preparing')
1098         dn = []
1099         for d in t.depends:
1100                 t._debug(' processing dependency '+d)
1101                 if not '@' in d:
1102                         t._debug('  literal dependency '+d)
1103                         dn.append(d)
1104                 else:
1105                         for (pkg,bin) in t.act.binaries:
1106                                 d = d.replace('@',pkg)
1107                                 t._debug('  synthesised dependency '+d)
1108                                 dn.append(d)
1109         testbed.prepare(dn)
1110  def run(t, tree):
1111         t._debug('[----------------------------------------')
1112         def stdouterr(oe):
1113                 idstr = t.what + '-' + oe
1114                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1115                         use_dir = opts.output_dir
1116                 else:
1117                         use_dir = testbed.scratch
1118                 return RelativeOutputFile(idstr, use_dir, idstr)
1119
1120         if hasattr(t.act,'work'): t.act.work.write(True)
1121         tree.read(True)
1122
1123         af = RelativeInputFile(t.what, tree, t.path)
1124         so = stdouterr('stdout')
1125         se = stdouterr('stderr')
1126
1127         tf = af.read(True)
1128         tmpdir = None
1129
1130         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1131         if rc: bomb('failed to chmod +x %s' % tf)
1132
1133         if 'needs-root' not in t.restriction_names and opts.user is not None:
1134                 tfl = ['su',opts.user,'-c',tf]
1135                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1136                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1137                 if opts.user:
1138                         script += '; chown %s "$1"' % opts.user
1139                         if 'rw-build-tree' in t.restriction_names:
1140                                 script += '; chown -R %s "$2"' % opts.user
1141                 rc = testbed.execute('mktmpdir-'+t.what,
1142                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1143                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1144                                 % (tmpdir, rc))
1145         else:
1146                 tfl = [tf]
1147
1148         rc = testbed.execute('test-'+t.what, tfl,
1149                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1150                 tmpdir=tmpdir, kind='test')
1151
1152         so_read = so.read()
1153         se_read = se.read()
1154
1155         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1156         stab = os.stat(se_read)
1157         if stab.st_size != 0:
1158                 l = open(se_read).readline()
1159                 l = l.rstrip('\n \t\r')
1160                 if len(l) > 35: l = l[:35] + '...'
1161                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1162                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1163                 debug_file(se_read)
1164         elif rc != 0:
1165                 t.reportfail('non-zero exit status %d' % rc)
1166         else:
1167                 t.report('PASS')
1168
1169         stab = os.stat(so_read)
1170         if stab.st_size != 0:
1171                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1172                 debug_file(so_read)
1173
1174         t._debug('----------------------------------------]')
1175
1176 def read_control(act, tree, control_override):
1177         stanzas = [ ]
1178
1179         if control_override is not None:
1180                 control_af = control_override
1181                 testbed.blame('arg:'+control_override.spec)
1182         else:
1183                 if act.missing_tests_control:
1184                         return ()
1185                 control_af = RelativeInputFile(act.what+'-testcontrol',
1186                         tree, 'debian/tests/control')
1187         try:
1188                 control = open(control_af.read(), 'r')
1189         except (IOError,OSError), oe:
1190                 if oe[0] != errno.ENOENT: raise
1191                 return []
1192
1193         lno = 0
1194         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1195         stz = None      # stz[field_name][index] = (lno, value)
1196                         # special field names:
1197                         # stz[' lno'] = number
1198                         # stz[' tests'] = list of Test objects
1199         def end_stanza(stz):
1200                 if stz is None: return
1201                 stz[' errs'] = 0
1202                 stanzas.append(stz)
1203                 stz = None
1204                 hcurrent = None
1205
1206         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1207         while 1:
1208                 l = control.readline()
1209                 if not l: break
1210                 lno += 1
1211                 if not l.endswith('\n'): badctrl('unterminated line')
1212                 if regexp.compile('\s*\#').match(l): continue
1213                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1214                 initmat = initre.match(l)
1215                 if initmat:
1216                         (fname, l) = initmat.groups()
1217                         fname = string.capwords(fname)
1218                         if stz is None:
1219                                 stz = { ' lno': lno, ' tests': [] }
1220                         if not stz.has_key(fname): stz[fname] = [ ]
1221                         hcurrent = stz[fname]
1222                 elif regexp.compile('\s').match(l):
1223                         if not hcurrent: badctrl('unexpected continuation')
1224                 else:
1225                         badctrl('syntax error')
1226                 hcurrent.append((lno, l))
1227         end_stanza(stz)
1228
1229         def testbadctrl(stz, lno, m):
1230                 report_badctrl(lno, m)
1231                 stz[' errs'] += 1
1232
1233         for stz in stanzas:
1234                 try:
1235                         try: tnames = stz['Tests']
1236                         except KeyError:
1237                                 tnames = ['*']
1238                                 raise Unsupported(stz[' lno'],
1239                                         'no Tests field')
1240                         tnames = map((lambda lt: lt[1]), tnames)
1241                         tnames = string.join(tnames).split()
1242                         base = {
1243                                 'restriction_names': [],
1244                                 'restrictions': [],
1245                                 'feature_names': [],
1246                                 'features': [],
1247                                 'testsdir': 'debian/tests',
1248                                 'depends' : '@'
1249                         }
1250                         for fname in stz.keys():
1251                                 if fname.startswith(' '): continue
1252                                 vl = stz[fname]
1253                                 try: fclass = globals()['Field_'+
1254                                         fname.replace('-','_')]
1255                                 except KeyError: raise Unsupported(vl[0][0],
1256                                         'unknown metadata field %s' % fname)
1257                                 f = fclass(stz, fname, base, tnames, vl)
1258                                 f.parse()
1259                         for tname in tnames:
1260                                 t = Test(tname, base, act)
1261                                 stz[' tests'].append(t)
1262                 except Unsupported, u:
1263                         for tname in tnames: u.report(tname)
1264                         continue
1265
1266         return stanzas
1267
1268 def print_exception(ei, msgprefix=''):
1269         if msgprefix: pstderr(msgprefix)
1270         (et, q, tb) = ei
1271         if et is Quit:
1272                 pstderr('adt-run: ' + q.m)
1273                 psummary('quitting: '+q.m)
1274                 return q.ec
1275         else:
1276                 pstderr("adt-run: unexpected, exceptional, error:")
1277                 psummary('quitting: unexpected error, consult transcript')
1278                 traceback.print_exc(None, sys.stderr)
1279                 if trace_stream is not None:
1280                         traceback.print_exc(None, trace_stream)
1281                 return 20
1282
1283 def cleanup():
1284         global trace_stream
1285         try:
1286                 if testbed is not None:
1287                         testbed.reset_apt()
1288                         testbed.stop()
1289                 if opts.tmpdir is None and tmpdir is not None:
1290                         rmtree('tmpdir', tmpdir)
1291                 if trace_stream is not None:
1292                         trace_stream.close()
1293                         trace_stream = None
1294         except:
1295                 print_exception(sys.exc_info(),
1296                         '\nadt-run: error cleaning up:\n')
1297                 os._exit(20)
1298
1299 #---------- registration, installation etc. of .deb's: Binaries
1300
1301 def determine_package(act):
1302         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1303         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1304         if rc: badpkg('failed to parse binary package, code %d' % rc)
1305         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1306         act.pkg = None
1307         for l in output.split('\n'):
1308                 m = re.match(l)
1309                 if not m: continue
1310                 if act.pkg: badpkg('two Package: lines in control file')
1311                 act.pkg = m.groups()[0]
1312         if not act.pkg: badpkg('no good Package: line in control file')
1313
1314 class Binaries:
1315  def __init__(b):
1316         b.dir = TemporaryDir('binaries')
1317         b.dir.write()
1318         ok = False
1319
1320         if opts.gnupghome is None:
1321                 opts.gnupghome = tmpdir+'/gnupg'
1322
1323         b._debug('initialising')
1324         try:
1325                 for x in ['pubring','secring']:
1326                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1327                 ok = True
1328         except (IOError,OSError), oe:
1329                 if oe.errno != errno.ENOENT: raise
1330
1331         if ok: b._debug('no key generation needed')
1332         else: b.genkey()
1333
1334  def _debug(b, s):
1335         debug('* '+s)
1336
1337  def genkey(b):
1338         b._debug('preparing for key generation')
1339
1340         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1341         mkdir_okexist(opts.gnupghome, 0700)
1342
1343         script = '''
1344   exec >&2
1345   cd "$1"
1346   cat <<"END" >key-gen-params
1347 Key-Type: DSA
1348 Key-Length: 1024
1349 Key-Usage: sign
1350 Name-Real: autopkgtest per-run key
1351 Name-Comment: do not trust this key
1352 Name-Email: autopkgtest@example.com
1353 END
1354   set -x
1355   gpg --homedir="$1" --batch --gen-key key-gen-params
1356                 '''
1357         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1358         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1359         if rc: bomb('key generation failed, code %d' % rc)
1360
1361  def apt_configs(b):
1362         return {
1363                 "Debug::pkgProblemResolver": "true",
1364                 "APT::Get::force-yes" : "true",
1365                 "APT::Get::Assume-Yes" : "true",
1366                 "quiet" : "true",
1367         }
1368
1369  def _configure_apt(b, tb):
1370         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1371                         True)
1372         f = open(config.write(),'w')
1373         for (k,v) in b.apt_configs().iteritems():
1374                 print >>f, '%s { "%s"; };' % (k, v)
1375         f.close()
1376         config.read(True)
1377
1378         prefs = OutputFile('apt-prefs','/etc/apt/preferences.d/90autopkgtest',
1379                         True)
1380         print >>open(prefs.write(),'w'), '''
1381 Package: *
1382 Pin: origin ""
1383 Pin-Priority: 1002
1384 '''
1385         prefs.read(True)
1386
1387  def _apt_get(b):
1388         ag = ['apt-get','-q']
1389         for kv in b.apt_configs().iteritems():
1390                 ag += ['-o', '%s=%s' % kv]
1391         return ' '.join(ag)
1392
1393  def reset(b):
1394         b._debug('reset')
1395         rmtree('binaries', b.dir.read())
1396         b.dir.invalidate()
1397         b.dir.write()
1398         b.install = []
1399         b.blamed = []
1400         b.registered = set()
1401
1402  def register(b, act, pkg, af, forwhat, blamed):
1403         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1404                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1405
1406         if act.ah['deb_'+forwhat] == 'ignore': return
1407
1408         b.blamed += testbed.blamed
1409
1410         leafname = pkg+'.deb'
1411         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1412
1413         try: os.remove(dest.write())
1414         except (IOError,OSError), oe:
1415                 if oe.errno != errno.ENOENT: raise e
1416
1417         try: os.link(af.read(), dest.write())
1418         except (IOError,OSError), oe:
1419                 if oe.errno != errno.EXDEV: raise e
1420                 shutil.copy(af.read(), dest)
1421
1422         if act.ah['deb_'+forwhat] == 'install':
1423                 b.install.append(pkg)
1424
1425         b.registered.add(pkg)
1426
1427  def publish(b):
1428         b._debug('publish')
1429
1430         b._configure_apt(testbed)
1431
1432         script = '''
1433   exec >&2
1434   cd "$1"
1435   apt-ftparchive packages . >Packages
1436   gzip <Packages >Packages.gz
1437   apt-ftparchive release . >Release
1438   rm -f Release.gpg
1439   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1440   gpg --homedir="$2" --batch --export >archive-key.pgp
1441                 '''
1442         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1443         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1444         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1445
1446         b.dir.invalidate(True)
1447         apt_source = b.dir.read(True)
1448
1449         so = TemporaryFile('vlds')
1450         script = '''
1451   exec 3>&1 >&2
1452   apt-key add archive-key.pgp
1453   echo "deb file://'''+apt_source+''' /" >/etc/apt/sources.list.d/autopkgtest.list
1454   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1455     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1456   fi
1457   '''+ b._apt_get() +''' update >&2
1458   cat /var/lib/dpkg/status >&3
1459                 '''
1460         testbed.mungeing_apt()
1461         rc = testbed.execute('apt-key', ['sh','-ec',script],
1462                                 so=so.write(True), cwd=b.dir.write(True),
1463                                 script=script, kind='install')
1464         if rc: bomb('apt setup failed with exit code %d' % rc)
1465
1466         testbed.blamed += b.blamed
1467
1468         b._debug('publish reinstall checking...')
1469         pkgs_reinstall = set()
1470         pkg = None
1471         for l in open(so.read()):
1472                 if l.startswith('Package: '):
1473                         pkg = l[9:].rstrip()
1474                 elif l.startswith('Status: install '):
1475                         if pkg in b.registered:
1476                                 pkgs_reinstall.add(pkg)
1477                                 b._debug(' publish reinstall needs '+pkg)
1478
1479         if pkgs_reinstall:
1480                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1481                 what = 'apt-get-reinstall'
1482                 cmdl = (b._apt_get() + ' --reinstall install '+
1483                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1484                 cmdl = ['sh','-c',cmdl]
1485                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1486                 if rc: badpkg("installation of basic binarries failed,"
1487                                 " exit code %d" % rc)
1488
1489         b._debug('publish install...')
1490         for pkg in b.install:
1491                 what = 'apt-get-install-%s' % pkg
1492                 testbed.blame(pkg)
1493                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1494                 cmdl = ['sh','-c',cmdl]
1495                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1496                 if rc: badpkg("installation of %s failed, exit code %d"
1497                                 % (pkg, rc))
1498
1499         b._debug('publish done')
1500
1501 #---------- processing of sources (building)
1502
1503 def source_rules_command(act,script,what,which,work,cwd,
1504                                 results_lines=0,xargs=[]):
1505         script = [      "exec 3>&1 >&2",
1506                         "set -x"        ] + script
1507         script = '\n'.join(script)
1508         so = TemporaryFile('%s-%s-results' % (what,which))
1509         rc = testbed.execute('%s-%s' % (what,which),
1510                         ['sh','-ec',script]+xargs, script=script,
1511                         so=so.write(True), cwd=cwd, kind='build')
1512         results = open(so.read()).read().rstrip('\n')
1513         if len(results): results = results.split("\n")
1514         else: results = []
1515         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1516         if results_lines is not None and len(results) != results_lines:
1517                 badpkg("got %d lines of results from %s where %d expected"
1518                         % (len(results), which, results_lines))
1519         if results_lines==1: return results[0]
1520         return results
1521
1522 def build_source(act, control_override):
1523         act.blame = 'arg:'+act.af.spec
1524         testbed.blame(act.blame)
1525         testbed.prepare1([])
1526         testbed.needs_reset()
1527
1528         what = act.what
1529         basename = act.af.spec
1530         debiancontrol = None
1531         act.binaries = []
1532
1533         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1534
1535         if act.kind == 'dsc':
1536                 dsc = act.af
1537                 dsc_file = open(dsc.read())
1538                 in_files = False
1539                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1540                 for l in dsc_file:
1541                         l = l.rstrip('\n')
1542                         if l.startswith('Files:'): in_files = True; continue
1543                         elif l.startswith('#'): pass
1544                         elif not l.startswith(' '):
1545                                 in_files = False
1546                                 if l.startswith('Source:'):
1547                                         act.blame = 'dsc:'+l[7:].strip()
1548                                         testbed.blame(act.blame)
1549                         if not in_files: continue
1550
1551                         m = fre.match(l)
1552                         if not m: badpkg(".dsc contains unparseable line"
1553                                         " in Files: `%s'" % l)
1554                         leaf = m.groups(0)[0]
1555                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1556                                         sibling=True)
1557                         subfile.read(True)
1558                 dsc.read(True)
1559
1560         if act.kind == 'ubtree':
1561                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1562                         act.af, 'debian/control')
1563                 dsc = TemporaryFile(what+'-fakedsc')
1564                 dsc_w = open(dsc.write(), 'w')
1565                 for l in open(debiancontrol.read()):
1566                         l = l.rstrip('\n')
1567                         if not len(l): break
1568                         print >>dsc_w, l
1569                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1570                 dsc_w.close()
1571
1572         if act.kind == 'dsc':
1573                 testbed.prepare2([])
1574                 testbed.satisfy_dependencies_string('dpkg-dev',
1575                                                 'install dpkg-dev')
1576
1577         work = TemporaryDir(what+'-build')
1578         act.work = work
1579
1580         tmpdir = work.write(True)+'/tmpdir'
1581         tmpdir_script = [
1582                         'TMPDIR="$1"',
1583                         'rm -rf -- "$TMPDIR"',
1584                         'export TMPDIR',
1585                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1586                 ]
1587
1588         if act.kind == 'ubtree':
1589                 spec = '%s/real-tree' % work.write(True)
1590                 create_command = '''
1591                         rm -rf "$spec"
1592                         mkdir "$spec"
1593                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1594                         '''
1595                 initcwd = act.af.read(True)
1596
1597         if act.kind == 'dsc':
1598                 spec = dsc.read(True)
1599                 create_command = '''
1600                         dpkg-source -x $spec
1601                         '''
1602                 initcwd = work.write(True)
1603
1604         script = [
1605                 'spec="$2"',
1606                 'origpwd=`pwd`',
1607                 'cd '+work.write(True)
1608         ]
1609
1610         if opts.user:
1611                 script += ([ 'chown '+opts.user+' .' ] +
1612                         tmpdir_script +
1613                         [ 'spec="$spec" origpwd="$origpwd" '
1614                                 +opts.user_wrap(create_command) ])
1615         else:
1616                 script += (tmpdir_script +
1617                         [ create_command ])
1618
1619         script += [
1620                         'cd [a-z0-9]*-*/.',
1621                         'pwd >&3',
1622                         'set +e; test -f debian/tests/control; echo $? >&3'
1623                 ]
1624         (result_pwd, control_test_rc) = source_rules_command(
1625                         act,script,what,'extract',work,
1626                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1627
1628         filter = act.ah['dsc_filter']
1629
1630         if control_test_rc == '1': act.missing_tests_control = True
1631
1632         # For optional builds:
1633         #
1634         # We might need to build the package because:
1635         #   - we want its binaries (filter isn't _ and at least one of the
1636         #       deb_... isn't ignore)
1637         #   - the test control file says so
1638         #       (assuming we have any tests)
1639
1640         class NeedBuildException: pass
1641         def build_needed(m):
1642                 debug_b('build needed for %s' % m)
1643                 raise NeedBuildException()
1644
1645         try:
1646                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1647                                       act.ah['deb_fortests'] != 'ignore'):
1648                         build_needed('binaries')
1649
1650                 result_pwd_af = InputDir(what+'-treeforcontrol',
1651                         result_pwd, True)
1652                 stanzas = read_control(act, result_pwd_af, control_override)
1653                 for stanza in stanzas:
1654                         for t in stanza[' tests']:
1655                                 if 'no-build-needed' not in t.feature_names:
1656                                         build_needed('test %s' % t.tname)
1657                                 for d in t.depends:
1658                                         if '@' in d:
1659                                                 build_needed('test %s '
1660                                                  'dependency %s' % (t.tname,d))
1661
1662                 debug_b('build not needed')
1663                 built = False
1664
1665         except NeedBuildException:
1666
1667                 if act.kind != 'dsc':
1668                         testbed.prepare2([])
1669
1670                 testbed.satisfy_dependencies_string('build-essential',
1671                                 'install build-essential')
1672                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1673
1674                 script = tmpdir_script + [
1675                         'cd "$2"',
1676                         'dpkg-checkbuilddeps',
1677                         opts.user_wrap('debian/rules build'),
1678                         ]
1679                 source_rules_command(act,script,what,'build',work,
1680                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1681
1682                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1683                         badpkg("results dir `%s' is not in expected parent"
1684                                " dir `%s'" % (result_pwd, work.read(True)))
1685
1686                 built = True
1687
1688         act.tests_tree = InputDir(what+'-tests-tree',
1689                                 work.read(True)+os.path.basename(result_pwd),
1690                                 True)
1691         if act.ah['dsc_tests']:
1692                 testbed.register_ephemeral(act.work)
1693                 testbed.register_ephemeral(act.tests_tree)
1694
1695         if not built:
1696                 act.blamed = []
1697                 return
1698
1699         act.blamed = copy.copy(testbed.blamed)
1700
1701         debug_b('filter=%s' % filter)
1702         if filter != '_':
1703                 script = tmpdir_script + [
1704                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1705                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1706                         'cd ..',
1707                         'echo *.deb >&3',
1708                         ]
1709                 result_debs = source_rules_command(act,script,what,
1710                                 'binary',work,work.write(True),
1711                                 results_lines=1, xargs=['x',tmpdir])
1712                 if result_debs == '*.deb': debs = []
1713                 else: debs = result_debs.split(' ')
1714                 debug_b('debs='+`debs`)
1715                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1716                 for deb in debs:
1717                         m = re.match(deb)
1718                         if not m: badpkg("badly-named binary `%s'" % deb)
1719                         pkg = m.groups()[0]
1720                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1721                         for pat in filter.split(','):
1722                                 debug_b('  pat=%s' % pat)
1723                                 if not fnmatch.fnmatchcase(pkg,pat):
1724                                         debug_b('   no match')
1725                                         continue
1726                                 deb_what = pkg+'_'+what+'.deb'
1727                                 bin = RelativeInputFile(deb_what,work,deb,True)
1728                                 debug_b('  deb_what=%s, bin=%s' %
1729                                         (deb_what, str(bin)))
1730                                 binaries.register(act,pkg,bin,
1731                                         'forbuilds',testbed.blamed)
1732                                 act.binaries.append((pkg,bin))
1733                                 break
1734                 debug_b('all done.')
1735
1736 #---------- main processing loop and main program
1737
1738 def process_actions():
1739         global binaries
1740
1741         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1742         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1743         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1744
1745         debug_a1('starting')
1746         testbed.open()
1747         binaries = Binaries()
1748
1749         for act in opts.actions:
1750                 if act.af is not None and not act.af.spec_tbp:
1751                         testbed.register_ephemeral(act.af)
1752
1753         binaries.reset()
1754         control_override = None
1755
1756         debug_a1('builds ...')
1757         for act in opts.actions:
1758                 debug_a2('%s %s' %
1759                         (act.kind, act.what))
1760
1761                 if act.kind == 'control':
1762                         control_override = act.af
1763                 if act.kind == 'deb':
1764                         testbed.blame('arg:'+act.af.spec)
1765                         determine_package(act)
1766                         testbed.blame('deb:'+act.pkg)
1767                         binaries.register(act,act.pkg,act.af,
1768                                 'forbuilds',testbed.blamed)
1769                 if act.kind == 'dsc' or act.kind == 'ubtree':
1770                         build_source(act, control_override)
1771                 if act.kind == 'tree':
1772                         act.binaries = []
1773                 if act.kind.endswith('tree') or act.kind == 'dsc':
1774                         control_override = None
1775                 if act.kind == 'instantiate':
1776                         pass
1777
1778         debug_a1('builds done.')
1779
1780         binaries.reset()
1781         control_override = None
1782
1783         debug_a1('tests ...')
1784         for act in opts.actions:
1785                 debug_a2('test %s %s' % (act.kind, act.what))
1786
1787                 testbed.needs_reset()
1788                 if act.kind == 'control':
1789                         control_override = act.af
1790                 if act.kind == 'deb':
1791                         binaries.register(act,act.pkg,act.af,'fortests',
1792                                 ['deb:'+act.pkg])
1793                 if act.kind == 'dsc' or act.kind == 'ubtree':
1794                         for (pkg,bin) in act.binaries:
1795                                 binaries.register(act,pkg,bin,'fortests',
1796                                         act.blamed)
1797                 if act.kind == 'dsc':
1798                         if act.ah['dsc_tests']:
1799                                 debug_a3('read control ...')
1800                                 stanzas = read_control(act, act.tests_tree,
1801                                                 control_override)
1802                                 testbed.blamed += act.blamed
1803                                 debug_a3('run_tests ...')
1804                                 run_tests(stanzas, act.tests_tree)
1805                         control_override = None
1806                 if act.kind == 'tree' or act.kind == 'ubtree':
1807                         testbed.blame('arg:'+act.af.spec)
1808                         stanzas = read_control(act, act.af, control_override)
1809                         debug_a3('run_tests ...')
1810                         run_tests(stanzas, act.af)
1811                         control_override = None
1812                 if act.kind == 'instantiate':
1813                         testbed.prepare([])
1814         debug_a1('tests done.')
1815
1816 def main():
1817         global testbed
1818         global tmpdir
1819         try:
1820                 parse_args()
1821         except SystemExit, se:
1822                 os._exit(20)
1823         try:
1824                 setup_trace()
1825                 testbed = Testbed()
1826                 testbed.start()
1827                 finalise_options()
1828                 process_actions()
1829         except:
1830                 ec = print_exception(sys.exc_info(), '')
1831                 cleanup()
1832                 os._exit(ec)
1833         cleanup()
1834         os._exit(errorcode)
1835
1836 main()