runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':10, 'install':300, 'test':300, 'build':3000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: debug'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = sys.stderr
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         ep._sp.stdin.close()
 128         rc = ep._sp.wait()
 129         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 130         ep._sp = None
 131
 132 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 133         if dbg is not None:
 134                 if isinstance(dbg,tuple): (what,script) = dbg
 135                 else: (what,script) = (dbg,None)
 136                 debug_subprocess(what, cmdl, script=script)
 137         ep = Errplumb(critical)
 138         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 139         output = running.communicate()[0]
 140         rc = running.wait()
 141         ep.wait()
 142         return (rc, output)
 143
 144 def psummary(m):
 145         if summary_stream is not None: print >>summary_stream, m
 146
 147 def preport(m):
 148         print m
 149         sys.stdout.flush()
 150         if trace_stream is not None: print >>trace_stream, m
 151         psummary(m)
 152
 153 def report(tname, result):
 154         preport('%-20s %s' % (tname, result))
 155
 156 #---------- errors we define
 157
 158 class Quit:
 159         def __init__(q,ec,m): q.ec = ec; q.m = m
 160
 161 def bomb(m):
 162         raise Quit(20, "unexpected error: %s" % m)
 163
 164 def badpkg(m):
 165         preport('blame: ' + ' '.join(testbed.blamed))
 166         preport('badpkg: ' + m)
 167         raise Quit(12, "erroneous package: %s" % m)
 168
 169 class Unsupported:
 170  def __init__(u, lno, m):
 171         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 172         else: u.m = m
 173  def report(u, tname):
 174         global errorcode
 175         errorcode != 2
 176         report(tname, 'SKIP %s' % u.m)
 177
 178 #---------- convenience function
 179
 180 def mkdir_okexist(pathname, mode=02755):
 181         try:
 182                 os.mkdir(pathname, mode)
 183         except OSError, oe:
 184                 if oe.errno != errno.EEXIST: raise
 185
 186 def rmtree(what, pathname):
 187         debug('/ %s rmtree %s' % (what, pathname), 2)
 188         shutil.rmtree(pathname)
 189
 190 def debug_subprocess(what, cmdl=None, script=None):
 191         o = '$ '+what+':'
 192         if cmdl is not None:
 193                 ol = []
 194                 for x in cmdl:
 195                         if x is script: x = '<SCRIPT>'
 196                         ol.append(x.    replace('\\','\\\\').
 197                                         replace(' ','\\ ')      )
 198                 o += ' '+ ' '.join(ol)
 199         debug(o)
 200         if script is not None and opts.debuglevel >= 1:
 201                 o = ''
 202                 for l in script.rstrip('\n').split('\n'):
 203                         o += '$     '+l+'\n'
 204                 debug(o,1)
 205
 206 def flatten(l):
 207         return reduce((lambda a,b: a + b), l, [])
 208
 209 #---------- fancy automatic file-copying class
 210
 211 class AutoFile:
 212         # p.what
 213         # p.path[tb]    None or path    not None => path known
 214         # p.file[tb]    None or path    not None => file exists
 215         # p.spec        string or None
 216         # p.spec_tbp    True or False, or not set if spec is None
 217         # p.dir         '' or '/'
 218
 219  def __init__(p, what):
 220         p.what = what
 221         p.path = [None,None]
 222         p.file = [None,None]
 223         p.spec = None
 224         p.dir = ''
 225
 226  def __repr__(p):
 227         return "<AF@%s>" % p.__str__()
 228  def __str__(p):
 229         def ptbp(tbp):
 230                 if p.path[tbp] is None: return '-'+p.dir
 231                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 232                 else: return p.path[tbp]+p.dir+'!'
 233         out = p.what
 234         if p.spec is not None:
 235                 if not hasattr(p,'spec_tb'): out += '~'
 236                 elif p.spec_tbp: out += '#'
 237                 else: out += '='
 238                 out += p.spec
 239         out += ':'
 240         out += ptbp(False)
 241         out += '|'
 242         out += ptbp(True)
 243         return out
 244
 245  def _wrong(p, how):
 246         xtra = ''
 247         if p.spec is not None:
 248                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 249         raise ("internal error: %s (%s)" % (how, str(p)))
 250
 251  def _ensure_path(p, tbp):
 252         if p.path[tbp] is None:
 253                 if '/' in p.what:
 254                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 255                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 256                 if not tbp:
 257                         p.path[tbp] = tmpdir+'/'+p.what
 258                 else:
 259                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 260
 261  def write(p, tbp=False):
 262         p._debug('write %s...' % 'HT'[tbp])
 263         p._ensure_path(tbp)
 264
 265         if p.dir and not p.file[tbp]:
 266                 if not tbp:
 267                         p._debug('mkdir H')
 268                         mkdir_okexist(p.path[tbp])
 269                 else:
 270                         cmdl = ['sh','-ec',
 271                                 'test -d "$1" || mkdir -p "$1"',
 272                                 'x', p.path[tbp]]
 273                         tf_what = urllib.quote(p.what).replace('/','%2F')
 274                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 275                         if rc: bomb('failed to create directory %s' %
 276                                 p.path[tbp])
 277
 278         p.file[tbp] = p.path[tbp]
 279         return p.path[tbp]
 280
 281  def read(p, tbp=False):
 282         p._debug('read %s...' % 'HT'[tbp])
 283         p._ensure_path(tbp)
 284
 285         if p.file[tbp] is None:
 286                 if p.file[not tbp] is None:
 287                         p._wrong("requesting read but nonexistent")
 288                 cud = ['copyup','copydown'][tbp]
 289                 src = p.file[not tbp] + p.dir
 290                 dst = p.path[tbp] + p.dir
 291                 testbed.command(cud, (src, dst))
 292                 p.file[tbp] = p.path[tbp]
 293
 294         return p.file[tbp] + p.dir
 295
 296  def invalidate(p, tbp=False):
 297         p.file[tbp] = None
 298         p._debug('invalidated %s' % 'HT'[tbp])
 299
 300  def _debug(p, m):
 301         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 302
 303  def _constructed(p):
 304         p._debug('constructed: '+str(p))
 305         p._check()
 306
 307  def _check(p):
 308         for tbp in [False,True]:
 309          for pf in [p.path, p.file]:
 310                 if pf[tbp] is None: continue
 311                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 312                 if p.dir and pf[tbp].endswith('/'):
 313                         pf[tbp] = pf[tbp].rstrip('/')
 314                         if not pf[tbp]: pf[tbp] = '/'
 315                 if not p.dir and pf[tbp].endswith('/'):
 316                         bomb("directory `%s' specified for "
 317                              "non-directory %s" % (pf[tbp], p.what))
 318
 319  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 320         AutoFile.__init__(p,what)
 321         sh_on = ''; sh_sibl = ''
 322         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 323         if sibling: sh_sibl=' (sibling)'
 324         parent._debug('using as base: %s: %s%s%s' %
 325                         (str(parent), leaf, sh_on, sh_sibl))
 326         if not sibling and not parent.dir:
 327                 parent._wrong('asked for non-sibling relative path of non-dir')
 328         if sibling: trim = os.path.dirname
 329         else: trim = lambda x: x
 330         for tbp in [False,True]:
 331                 if parent.path[tbp] is None: continue
 332                 trimmed = trim(parent.path[tbp])
 333                 if trimmed: trimmed += '/'
 334                 p.path[tbp] = trimmed + leaf
 335                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 336                         p.file[tbp] = p.path[tbp]
 337
 338 class InputFile(AutoFile):
 339  def _init(p, what, spec, spec_tbp=False):
 340         AutoFile.__init__(p, what)
 341         p.spec = spec
 342         p.spec_tbp = spec_tbp
 343         p.path[spec_tbp] = spec
 344         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 345  def __init__(p, what, spec, spec_tbp=False):
 346         p._init(what,spec,spec_tbp)
 347         p._constructed()
 348
 349 class InputDir(InputFile):
 350  def __init__(p, what, spec, spec_tbp=False):
 351         InputFile._init(p,what,spec,spec_tbp)
 352         p.dir = '/'
 353         p._constructed()
 354
 355 class OutputFile(AutoFile):
 356  def _init(p, what, spec, spec_tbp=False):
 357         AutoFile.__init__(p, what)
 358         p.spec = spec
 359         p.spec_tbp = spec_tbp
 360         p.path[spec_tbp] = spec
 361  def __init__(p, what, spec, spec_tbp=False):
 362         p._init(what,spec,spec_tbp)
 363         p._constructed()
 364
 365 class OutputDir(OutputFile):
 366  def __init__(p, what, spec, spec_tbp=False):
 367         OutputFile._init(p,what,spec,spec_tbp)
 368         p.dir = '/'
 369         p._constructed()
 370
 371 class RelativeInputFile(AutoFile):
 372  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 373         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 374         p._constructed()
 375
 376 class RelativeOutputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, sibling=False):
 378         p._relative_init(what, parent, leaf, None, False, sibling)
 379         p._constructed()
 380
 381 class TemporaryFile(AutoFile):
 382  def __init__(p, what):
 383         AutoFile.__init__(p, what)
 384         p._constructed()
 385
 386 class TemporaryDir(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p.dir = '/'
 390         p._constructed()
 391
 392 #---------- parsing and representation of the arguments
 393
 394 class Action:
 395  def __init__(a, kind, af, arghandling, what):
 396         # extra attributes get added during processing
 397         a.kind = kind
 398         a.af = af
 399         a.ah = arghandling
 400         a.what = what
 401         a.missing_tests_control = False
 402  def __repr__(a):
 403         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 404
 405 def parse_args():
 406         global opts
 407         global n_non_actions # argh, stupid python scoping rules
 408         usage = "%prog <options> --- <virt-server>..."
 409         parser = OptionParser(usage=usage)
 410         pa = parser.add_option
 411         pe = parser.add_option
 412
 413         arghandling = {
 414                 'dsc_tests': True,
 415                 'dsc_filter': '*',
 416                 'deb_forbuilds': 'auto',
 417                 'deb_fortests': 'auto',
 418                 'tb': False,
 419                 'override_control': None
 420         }
 421         initial_arghandling = arghandling.copy()
 422         n_non_actions = 0
 423
 424         #----------
 425         # actions (ie, test sets to run, sources to build, binaries to use):
 426
 427         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 428                 global n_non_actions
 429                 parser.largs.append((value,kindpath))
 430                 n_non_actions += not(is_act)
 431
 432         def pa_action(long, metavar, kindpath, help, is_act=True):
 433                 pa('','--'+long, action='callback', callback=cb_action,
 434                         nargs=1, type='string',
 435                         callback_args=(long,kindpath,is_act), help=help)
 436
 437         pa_action('built-tree',         'TREE', '@/',
 438                 help='run tests from build tree TREE')
 439
 440         pa_action('unbuilt-tree',       'TREE', '@//',
 441                 help='run tests from build tree TREE')
 442
 443         pa_action('source',             'DSC', '@.dsc',
 444                 help='build DSC and use its tests and/or'
 445                     ' generated binary packages')
 446
 447         pa_action('binary',             'DEB', '@.deb',
 448                help='use binary package DEB according'
 449                     ' to most recent --binaries-* settings')
 450
 451         def cb_actnoarg(op,optstr,value,parser, largsentry):
 452                 parser.largs.append(largsentry)
 453         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 454                 callback_args=((None, ('instantiate',)),),
 455                 help='instantiate testbed now (during testing phase)'
 456                      ' and install packages'
 457                      ' selected for automatic installation, even'
 458                      ' if this might apparently not be required otherwise')
 459
 460         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 461                help='run tests from control file CONTROL instead,'
 462                     ' (applies to next test suite only)')
 463
 464         #----------
 465         # argument handling settings (what ways to use action
 466         #  arguments, and pathname processing):
 467
 468         def cb_setah(option, opt_str, value, parser, toset,setval):
 469                 if type(setval) == list:
 470                         if not value in setval:
 471                                 parser.error('value for %s option (%s) is not '
 472                                  'one of the permitted values (%s)' %
 473                                  (value, opt_str, setval.join(' ')))
 474                 elif setval is not None:
 475                         value = setval
 476                 for v in toset:
 477                         arghandling[v] = value
 478                 parser.largs.append(arghandling.copy())
 479
 480         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 481                 type = metavar
 482                 if type is not None: type = 'string'
 483                 pa('',long, action='callback', callback=cb_setah,
 484                    callback_args=(affected,effect), **kwargs)
 485
 486         #---- paths: host or testbed:
 487         #
 488         pa_setah('--paths-testbed', ['tb'],True,
 489                 help='subsequent path specifications refer to the testbed')
 490         pa_setah('--paths-host', ['tb'],False,
 491                 help='subsequent path specifications refer to the host')
 492
 493         #---- source processing settings:
 494
 495         pa_setah('--sources-tests', ['dsc_tests'],True,
 496                 help='run tests from builds of subsequent sources')
 497         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 498                 help='do not run tests from builds of subsequent sources')
 499
 500         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 501                 type='string', metavar='PATTERN-LIST',
 502                 help='from subsequent sources, use binaries matching'
 503                      ' PATTERN-LIST (comma-separated glob patterns)'
 504                      ' according to most recent --binaries-* settings')
 505         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 506                 help='from subsequent sources, do not use any binaries')
 507
 508         #---- binary package processing settings:
 509
 510         def pa_setahbins(long,toset,how):
 511          pa_setah(long, toset,['ignore','auto','install'],
 512                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 513                 help=how+' ignore binaries, install them as needed'
 514                         ' for dependencies, or unconditionally install'
 515                         ' them, respectively')
 516         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 517         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 518         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 519
 520         #----------
 521         # general options:
 522
 523         def cb_vserv(op,optstr,value,parser):
 524                 parser.values.vserver = list(parser.rargs)
 525                 del parser.rargs[:]
 526
 527         def cb_path(op,optstr,value,parser, constructor,long,dir):
 528                 name = long.replace('-','_')
 529                 af = constructor(long, value,arghandling['tb'])
 530                 setattr(parser.values, name, af)
 531
 532         def pa_path(long, constructor, help, dir=False):
 533                 pa('','--'+long, action='callback', callback=cb_path,
 534                         callback_args=(constructor,long,dir),
 535                         nargs=1, type='string',
 536                         help=help, metavar='PATH')
 537
 538         pa_path('output-dir', OutputDir, dir=True,
 539                 help='write stderr/out files in PATH')
 540
 541         pa('','--tmp-dir',              type='string', dest='tmpdir',
 542                 help='write temporary files to TMPDIR, emptying it'
 543                      ' beforehand and leaving it behind at the end')
 544         pa('','--log-file',             type='string', dest='logfile',
 545                 help='write the log LOGFILE, emptying it beforehand,'
 546                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 547         pa('','--summary-file',         type='string', dest='summary',
 548                 help='write a summary report to SUMMARY,'
 549                      ' emptying it beforehand')
 550
 551         pa('','--user',                 type='string', dest='user',
 552                 help='run tests as USER (needs root on testbed)')
 553         pa('','--gain-root',            type='string', dest='gainroot',
 554                 help='prefix debian/rules binary with GAINROOT')
 555         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 556         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 557         pa('','--gnupg-home',           type='string', dest='gnupghome',
 558                 default='~/.autopkgtest/gpg',
 559                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 560                         " signing private apt archive);"
 561                         " `fresh' means generate new key each time.")
 562
 563         #----------
 564         # actual meat:
 565
 566         class SpecialOption(optparse.Option): pass
 567         vs_op = SpecialOption('','--VSERVER-DUMMY')
 568         vs_op.action = 'callback'
 569         vs_op.type = None
 570         vs_op.default = None
 571         vs_op.nargs = 0
 572         vs_op.callback = cb_vserv
 573         vs_op.callback_args = ( )
 574         vs_op.callback_kwargs = { }
 575         vs_op.help = 'introduces virtualisation server and args'
 576         vs_op._short_opts = []
 577         vs_op._long_opts = ['---']
 578
 579         pa(vs_op)
 580
 581         (opts,args) = parser.parse_args()
 582         if not hasattr(opts,'vserver'):
 583                 parser.error('you must specifiy --- <virt-server>...')
 584         if n_non_actions >= len(parser.largs):
 585                 parser.error('nothing to do specified')
 586
 587         arghandling = initial_arghandling
 588         opts.actions = []
 589         ix = 0
 590         for act in args:
 591                 if type(act) == dict:
 592                         arghandling = act
 593                         continue
 594                 elif type(act) == tuple:
 595                         pass
 596                 elif type(act) == str:
 597                         act = (act,act)
 598                 else:
 599                         raise ("unknown action in list `%s' having"
 600                               " type `%s'" % (act, type(act)))
 601                 (pathstr, kindpath) = act
 602
 603                 constructor = InputFile
 604                 if type(kindpath) is tuple:             kind = kindpath[0]
 605                 elif kindpath.endswith('.deb'):         kind = 'deb'
 606                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 607                 elif kindpath.endswith('//'):
 608                         kind = 'ubtree'
 609                         constructor = InputDir
 610                 elif kindpath.endswith('/'):
 611                         kind = 'tree'
 612                         constructor = InputDir
 613                 else: parser.error("do not know how to handle filename \`%s';"
 614                         " specify --source --binary or --build-tree")
 615
 616                 what = '%s%s' % (kind,ix); ix += 1
 617
 618                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 619                 else: fwhatx = '-'+kind
 620
 621                 if pathstr is None: af = None
 622                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 623                 opts.actions.append(Action(kind, af, arghandling, what))
 624
 625 def setup_trace():
 626         global trace_stream, tmpdir, summary_stream
 627
 628         if opts.tmpdir is not None:
 629                 rmtree('tmpdir(specified)',opts.tmpdir)
 630                 mkdir_okexist(opts.tmpdir, 0700)
 631                 tmpdir = opts.tmpdir
 632         else:
 633                 assert(tmpdir is None)
 634                 tmpdir = tempfile.mkdtemp()
 635
 636         if opts.logfile is None:
 637                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 638                         opts.logfile = opts.output_dir.spec + '/log'
 639                 elif opts.tmpdir is not None:
 640                         opts.logfile = opts.tmpdir + '/log'
 641         if opts.logfile is not None:
 642                 trace_stream = open(opts.logfile, 'w', 0)
 643         if opts.summary is not None:
 644                 summary_stream = open(opts.summary, 'w', 0)
 645
 646         debug('options: '+`opts`, 1)
 647
 648 def finalise_options():
 649         global opts, tb, build_essential
 650
 651         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 652                 opts.user = ''
 653
 654         if opts.user is None:
 655                 su = 'suggested-normal-user='
 656                 ul = [
 657                         e[len(su):]
 658                         for e in testbed.caps
 659                         if e.startswith(su)
 660                         ]
 661                 if ul:
 662                         opts.user = ul[0]
 663                 else:
 664                         opts.user = ''
 665
 666         if opts.user:
 667                 if 'root-on-testbed' not in testbed.caps:
 668                         pstderr("warning: virtualisation"
 669                                 " system does not offer root on testbed,"
 670                                 " but --user option specified: failure likely")
 671                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 672         else:
 673                 opts.user_wrap = lambda x: x
 674
 675         if opts.gainroot is None:
 676                 opts.gainroot = ''
 677                 if (opts.user or
 678                     'root-on-testbed' not in testbed.caps):
 679                         opts.gainroot = 'fakeroot'
 680                         build_essential += ['fakeroot']
 681
 682         if opts.gnupghome.startswith('~/'):
 683                 try: home = os.environ['HOME']
 684                 except KeyError:
 685                         parser.error("HOME environment variable"
 686                                 " not set, needed for --gnupghome=`%s"
 687                                 % opts.gnupghome)
 688                 opts.gnupghome = home + opts.gnupghome[1:]
 689         elif opts.gnupghome == 'fresh':
 690                 opts.gnupghome = None
 691
 692 #---------- testbed management - the Testbed class
 693
 694 class Testbed:
 695  def __init__(tb):
 696         tb.sp = None
 697         tb.lastsend = None
 698         tb.scratch = None
 699         tb.modified = False
 700         tb.blamed = []
 701         tb._ephemeral = []
 702         tb._debug('init')
 703         tb._need_reset_apt = False
 704  def _debug(tb, m, minlevel=0):
 705         debug('** '+m, minlevel)
 706  def start(tb):
 707         tb._debug('start')
 708         p = subprocess.PIPE
 709         debug_subprocess('vserver', opts.vserver)
 710         tb._errplumb = Errplumb(True)
 711         tb.sp = subprocess.Popen(opts.vserver,
 712                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 713         tb.expect('ok')
 714         tb.caps = tb.commandr('capabilities')
 715  def stop(tb):
 716         tb._debug('stop')
 717         tb.close()
 718         if tb.sp is None: return
 719         ec = tb.sp.returncode
 720         if ec is None:
 721                 tb.sp.stdout.close()
 722                 tb.send('quit')
 723                 tb.sp.stdin.close()
 724                 ec = tb.sp.wait()
 725         if ec:
 726                 tb.bomb('testbed gave exit status %d after quit' % ec)
 727         tb._errplumb.wait()
 728  def open(tb):
 729         tb._debug('open, scratch=%s' % tb.scratch)
 730         if tb.scratch is not None: return
 731         pl = tb.commandr('open')
 732         tb.scratch = InputDir('tb-scratch', pl[0], True)
 733         tb.deps_processed = []
 734  def mungeing_apt(tb):
 735         if not 'revert' in tb.caps:
 736                 tb._need_reset_apt = True
 737  def reset_apt(tb):
 738         if not tb._need_reset_apt: return
 739         what = 'aptget-update-reset'
 740         cmdl = ['apt-get','-qy','update']
 741         rc = tb.execute(what, cmdl, timeout=timeouts['install'])
 742         if rc:
 743                 pstderr("\n" "warning: failed to restore"
 744                         " testbed apt cache, exit code %d" % rc)
 745         tb._need_reset_apt = False
 746  def close(tb):
 747         tb._debug('close, scratch=%s' % tb.scratch)
 748         if tb.scratch is None: return
 749         tb.scratch = None
 750         if tb.sp is None: return
 751         tb.command('close')
 752  def prepare1(tb, deps_new):
 753         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 754                 (tb.modified, tb.deps_processed, deps_new), 1)
 755         if 'revert' in tb.caps and (tb.modified or
 756             [d for d in tb.deps_processed if d not in deps_new]):
 757                 for af in tb._ephemeral: af.read(False)
 758                 tb._debug('reset **')
 759                 tb.command('revert')
 760                 tb.blamed = []
 761                 for af in tb._ephemeral: af.invalidate(True)
 762         tb.modified = False
 763  def prepare2(tb, deps_new):
 764         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 765         binaries.publish()
 766         tb._install_deps(deps_new)
 767  def prepare(tb, deps_new):
 768         tb.prepare1(deps_new)
 769         tb.prepare2(deps_new)
 770  def register_ephemeral(tb, af):
 771         tb._ephemeral.append(af)
 772  def _install_deps(tb, deps_new):
 773         tb._debug(' installing dependencies '+`deps_new`, 1)
 774         tb.deps_processed = deps_new
 775         if not deps_new: return
 776         dstr = ', '.join(deps_new)
 777         script = binaries.apt_pkg_gdebi_script(
 778                 dstr, [[
 779                 'from GDebi.DebPackage import DebPackage',
 780                 'd = DebPackage(cache)',
 781                 'res = d.satisfyDependsStr(arg)',
 782                 ]])
 783         cmdl = ['python','-c',script]
 784         what = 'install-deps'
 785         rc = testbed.execute(what+'-debinstall', cmdl, script=script,
 786                                 timeout=timeouts['install'])
 787         if rc: badpkg('dependency install failed, exit code %d' % rc)
 788  def needs_reset(tb):
 789         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 790         tb.modified = True
 791  def blame(tb, m):
 792         tb._debug('blame += %s' % m, 1)
 793         tb.blamed.append(m)
 794  def bomb(tb, m):
 795         tb._debug('bomb %s' % m)
 796         if tb.sp is not None:
 797                 tb.sp.stdout.close()
 798                 tb.sp.stdin.close()
 799                 ec = tb.sp.wait()
 800                 if ec: pstderr('adt-run: testbed failing,'
 801                                 ' exit status %d' % ec)
 802         tb.sp = None
 803         raise Quit(16, 'testbed failed: %s' % m)
 804  def send(tb, string):
 805         tb.sp.stdin
 806         try:
 807                 debug('>> '+string, 2)
 808                 print >>tb.sp.stdin, string
 809                 tb.sp.stdin.flush()
 810                 tb.lastsend = string
 811         except:
 812                 (type, value, dummy) = sys.exc_info()
 813                 tb.bomb('cannot send to testbed: %s' % traceback.
 814                         format_exception_only(type, value))
 815  def expect(tb, keyword, nresults=None):
 816         l = tb.sp.stdout.readline()
 817         if not l: tb.bomb('unexpected eof from the testbed')
 818         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 819         l = l.rstrip('\n')
 820         debug('<< '+l, 2)
 821         ll = l.split()
 822         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 823         if ll[0] != keyword:
 824                 if tb.lastsend is None:
 825                         tb.bomb("got banner `%s', expected `%s...'" %
 826                                 (l, keyword))
 827                 else:
 828                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 829                                 (tb.lastsend, l, keyword))
 830         ll = ll[1:]
 831         if nresults is not None and len(ll) != nresults:
 832                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 833                         " expected %d result parameters" %
 834                         (string, l, len(ll), nresults))
 835         return ll
 836  def commandr(tb, cmd, args=(), nresults=None):
 837         # pass args=[None,...] or =(None,...) to avoid more url quoting
 838         if type(cmd) is str: cmd = [cmd]
 839         if len(args) and args[0] is None: args = args[1:]
 840         else: args = map(urllib.quote, args)
 841         al = cmd + args
 842         tb.send(string.join(al))
 843         ll = tb.expect('ok', nresults)
 844         rl = map(urllib.unquote, ll)
 845         return rl
 846  def command(tb, cmd, args=()):
 847         tb.commandr(cmd, args, 0)
 848  def commandr1(tb, cmd, args=()):
 849         rl = tb.commandr(cmd, args, 1)
 850         return rl[0]
 851  def execute(tb, what, cmdl,
 852                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 853                 script=False, tmpdir=None, timeout=timeouts['short']):
 854         # Options for script:
 855         #   False - do not call debug_subprocess, no synch. reporting required
 856         #   None or string - call debug_subprocess with that value,
 857         #                       plumb stderr through synchronously if possible
 858         # Options for se:
 859         #   None - usual Errplumb (output is of kind 2c)
 860         #   string - path on testbed (output is of kind 2d)
 861
 862         if script is not False: debug_subprocess(what, cmdl, script=script)
 863         if cwd is None: cwd = tb.scratch.write(True)
 864
 865         xdump = None
 866         if se is None:
 867                 ep = Errplumb()
 868                 se_catch = TemporaryFile(what+'-xerr')
 869                 se_use = se_catch.write(True)
 870                 if not opts.quiet: xdump = 'debug=2-2'
 871                 elif trace_stream is not None:
 872                         xdump = 'debug=2-%d' % trace_stream.fileno()
 873         else:
 874                 ep = None
 875                 se_catch = None
 876                 se_use = se
 877
 878         cmdl = [None,
 879                 ','.join(map(urllib.quote, cmdl)),
 880                 si, so, se_use, cwd]
 881
 882         if timeout is not None and timeout > 0:
 883                 cmdl.append('timeout=%d' % timeout)
 884
 885         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 886         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 887
 888         rc = tb.commandr1('execute', cmdl)
 889         try: rc = int(rc)
 890         except ValueError: bomb("execute for %s gave invalid response `%s'"
 891                                         % (what,rc))
 892
 893         if se_catch is not None:
 894                 debug_file(se_catch.read())
 895         if ep is not None:
 896                 ep.wait()
 897
 898         return rc
 899
 900 #---------- representation of test control files: Field*, Test, etc.
 901
 902 class FieldBase:
 903  def __init__(f, fname, stz, base, tnames, vl):
 904         assert(vl)
 905         f.stz = stz
 906         f.base = base
 907         f.tnames = tnames
 908         f.vl = vl
 909  def words(f):
 910         def distribute(vle):
 911                 (lno, v) = vle
 912                 r = v.split()
 913                 r = map((lambda w: (lno, w)), r)
 914                 return r
 915         return flatten(map(distribute, f.vl))
 916  def atmostone(f):
 917         if len(vl) == 1:
 918                 (f.lno, f.v) = vl[0]
 919         else:
 920                 raise Unsupported(f.vl[1][0],
 921                         'only one %s field allowed' % fn)
 922         return f.v
 923
 924 class FieldIgnore(FieldBase):
 925  def parse(f): pass
 926
 927 class Restriction:
 928  def __init__(r,rname,base): pass
 929
 930 class Restriction_rw_build_tree(Restriction): pass
 931 class Restriction_breaks_testbed(Restriction):
 932  def __init__(r, rname, base):
 933         if 'revert' not in testbed.caps:
 934                 raise Unsupported(f.lno,
 935                         'Test breaks testbed but testbed cannot revert')
 936 class Restriction_needs_root(Restriction):
 937  def __init__(r, rname, base):
 938         if 'root-on-testbed' not in testbed.caps:
 939                 raise Unsupported(f.lno,
 940                         'Test needs root on testbed which is not available')
 941
 942 class Field_Restrictions(FieldBase):
 943  def parse(f):
 944         for wle in f.words():
 945                 (lno, rname) = wle
 946                 nrname = rname.replace('-','_')
 947                 try: rclass = globals()['Restriction_'+nrname]
 948                 except KeyError: raise Unsupported(lno,
 949                         'unknown restriction %s' % rname)
 950                 r = rclass(nrname, f.base)
 951                 f.base['restriction_names'].append(rname)
 952                 f.base['restrictions'].append(r)
 953
 954 class Field_Features(FieldIgnore):
 955  def parse(f):
 956         for wle in f.words():
 957                 (lno, fname) = wle
 958                 f.base['feature_names'].append(fname)
 959                 nfname = fname.replace('-','_')
 960                 try: fclass = globals()['Feature_'+nfname]
 961                 except KeyError: continue
 962                 ft = fclass(nfname, f.base)
 963                 f.base['features'].append(ft)
 964
 965 class Field_Tests(FieldIgnore): pass
 966
 967 class Field_Depends(FieldBase):
 968  def parse(f):
 969         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 970         dl = map(lambda x: x.strip(),
 971                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 972         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
 973         for d in dl:
 974                 if re.search(d):
 975                         badpkg("Test Depends field contains dependency"
 976                                " `%s' with invalid characters" % d)
 977         f.base['depends'] = dl
 978
 979 class Field_Tests_directory(FieldBase):
 980  def parse(f):
 981         td = atmostone(f)
 982         if td.startswith('/'): raise Unspported(f.lno,
 983                 'Tests-Directory may not be absolute')
 984         f.base['testsdir'] = td
 985
 986 def run_tests(stanzas, tree):
 987         global errorcode, testbed
 988         if stanzas == ():
 989                 report('*', 'SKIP no tests in this package')
 990                 errorcode |= 8
 991         for stanza in stanzas:
 992                 tests = stanza[' tests']
 993                 if not tests:
 994                         report('*', 'SKIP package has metadata but no tests')
 995                         errorcode |= 8
 996                 for t in tests:
 997                         t.prepare()
 998                         t.run(tree)
 999                         if 'breaks-testbed' in t.restriction_names:
1000                                 testbed.needs_reset()
1001                 testbed.needs_reset()
1002
1003 class Test:
1004  def __init__(t, tname, base, act):
1005         if '/' in tname: raise Unsupported(base[' lno'],
1006                 'test name may not contain / character')
1007         for k in base: setattr(t,k,base[k])
1008         t.tname = tname
1009         t.act = act
1010         t.what = act.what+'t-'+tname
1011         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1012         else: t.path = tname
1013         t._debug('constructed; path=%s' % t.path)
1014         t._debug(' .depends=%s' % t.depends)
1015  def _debug(t, m):
1016         debug('& %s: %s' % (t.what, m))
1017  def report(t, m):
1018         report(t.what, m)
1019  def reportfail(t, m):
1020         global errorcode
1021         errorcode |= 4
1022         report(t.what, 'FAIL ' + m)
1023  def prepare(t):
1024         t._debug('preparing')
1025         dn = []
1026         for d in t.depends:
1027                 t._debug(' processing dependency '+d)
1028                 if not '@' in d:
1029                         t._debug('  literal dependency '+d)
1030                         dn.append(d)
1031                 else:
1032                         for (pkg,bin) in t.act.binaries:
1033                                 d = d.replace('@',pkg)
1034                                 t._debug('  synthesised dependency '+d)
1035                                 dn.append(d)
1036         testbed.prepare(dn)
1037  def run(t, tree):
1038         t._debug('[----------------------------------------')
1039         def stdouterr(oe):
1040                 idstr = t.what + '-' + oe
1041                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1042                         use_dir = opts.output_dir
1043                 else:
1044                         use_dir = testbed.scratch
1045                 return RelativeOutputFile(idstr, use_dir, idstr)
1046
1047         t.act.work.write(True)
1048
1049         af = RelativeInputFile(t.what, tree, t.path)
1050         so = stdouterr('stdout')
1051         se = stdouterr('stderr')
1052
1053         tf = af.read(True)
1054         tmpdir = None
1055         tree.read(True)
1056
1057         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1058         if rc: bomb('failed to chmod +x %s' % tf)
1059
1060         if 'needs-root' not in t.restriction_names and opts.user is not None:
1061                 tfl = ['su',opts.user,'-c',tf]
1062                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1063                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1064                 if opts.user: script += '; chown %s "$1"' % opts.user
1065                 if 'rw-build-tree' in t.restriction_names:
1066                         script += '; chown -R %s "$2"' % opts.user
1067                 rc = testbed.execute('mktmpdir-'+t.what,
1068                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1069                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1070                                 % (tmpdir, rc))
1071         else:
1072                 tfl = [tf]
1073
1074         rc = testbed.execute('test-'+t.what, tfl,
1075                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1076                 tmpdir=tmpdir, timeout=timeouts['test'])
1077
1078         so_read = so.read()
1079         se_read = se.read()
1080
1081         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1082         stab = os.stat(se_read)
1083         if stab.st_size != 0:
1084                 l = open(se_read).readline()
1085                 l = l.rstrip('\n \t\r')
1086                 if len(l) > 35: l = l[:35] + '...'
1087                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1088                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1089                 debug_file(se_read)
1090         elif rc != 0:
1091                 t.reportfail('non-zero exit status %d' % rc)
1092         else:
1093                 t.report('PASS')
1094
1095         stab = os.stat(so_read)
1096         if stab.st_size != 0:
1097                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1098                 debug_file(so_read)
1099
1100         t._debug('----------------------------------------]')
1101
1102 def read_control(act, tree, control_override):
1103         stanzas = [ ]
1104
1105         if control_override is not None:
1106                 control_af = control_override
1107                 testbed.blame('arg:'+control_override.spec)
1108         else:
1109                 if act.missing_tests_control:
1110                         return ()
1111                 control_af = RelativeInputFile(act.what+'-testcontrol',
1112                         tree, 'debian/tests/control')
1113         try:
1114                 control = open(control_af.read(), 'r')
1115         except OSError, oe:
1116                 if oe[0] != errno.ENOENT: raise
1117                 return []
1118
1119         lno = 0
1120         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1121         stz = None      # stz[field_name][index] = (lno, value)
1122                         # special field names:
1123                         # stz[' lno'] = number
1124                         # stz[' tests'] = list of Test objects
1125         def end_stanza(stz):
1126                 if stz is None: return
1127                 stz[' errs'] = 0
1128                 stanzas.append(stz)
1129                 stz = None
1130                 hcurrent = None
1131
1132         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1133         while 1:
1134                 l = control.readline()
1135                 if not l: break
1136                 lno += 1
1137                 if not l.endswith('\n'): badctrl('unterminated line')
1138                 if regexp.compile('\s*\#').match(l): continue
1139                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1140                 initmat = initre.match(l)
1141                 if initmat:
1142                         (fname, l) = initmat.groups()
1143                         fname = string.capwords(fname)
1144                         if stz is None:
1145                                 stz = { ' lno': lno, ' tests': [] }
1146                         if not stz.has_key(fname): stz[fname] = [ ]
1147                         hcurrent = stz[fname]
1148                 elif regexp.compile('\s').match(l):
1149                         if not hcurrent: badctrl('unexpected continuation')
1150                 else:
1151                         badctrl('syntax error')
1152                 hcurrent.append((lno, l))
1153         end_stanza(stz)
1154
1155         def testbadctrl(stz, lno, m):
1156                 report_badctrl(lno, m)
1157                 stz[' errs'] += 1
1158
1159         for stz in stanzas:
1160                 try:
1161                         try: tnames = stz['Tests']
1162                         except KeyError:
1163                                 tnames = ['*']
1164                                 raise Unsupported(stz[' lno'],
1165                                         'no Tests field')
1166                         tnames = map((lambda lt: lt[1]), tnames)
1167                         tnames = string.join(tnames).split()
1168                         base = {
1169                                 'restriction_names': [],
1170                                 'restrictions': [],
1171                                 'feature_names': [],
1172                                 'features': [],
1173                                 'testsdir': 'debian/tests',
1174                                 'depends' : '@'
1175                         }
1176                         for fname in stz.keys():
1177                                 if fname.startswith(' '): continue
1178                                 vl = stz[fname]
1179                                 try: fclass = globals()['Field_'+
1180                                         fname.replace('-','_')]
1181                                 except KeyError: raise Unsupported(vl[0][0],
1182                                         'unknown metadata field %s' % fname)
1183                                 f = fclass(stz, fname, base, tnames, vl)
1184                                 f.parse()
1185                         for tname in tnames:
1186                                 t = Test(tname, base, act)
1187                                 stz[' tests'].append(t)
1188                 except Unsupported, u:
1189                         for tname in tnames: u.report(tname)
1190                         continue
1191
1192         return stanzas
1193
1194 def print_exception(ei, msgprefix=''):
1195         if msgprefix: pstderr(msgprefix)
1196         (et, q, tb) = ei
1197         if et is Quit:
1198                 pstderr('adt-run: ' + q.m)
1199                 psummary('quitting: '+q.m)
1200                 return q.ec
1201         else:
1202                 pstderr("adt-run: unexpected, exceptional, error:")
1203                 psummary('quitting: unexpected error, consult transcript')
1204                 traceback.print_exc(None, sys.stderr)
1205                 if trace_stream is not None:
1206                         traceback.print_exc(None, trace_stream)
1207                 return 20
1208
1209 def cleanup():
1210         global trace_stream
1211         try:
1212                 rm_ec = 0
1213                 if opts.tmpdir is None and tmpdir is not None:
1214                         rmtree('tmpdir', tmpdir)
1215                 if testbed is not None:
1216                         testbed.reset_apt()
1217                         testbed.stop()
1218                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1219                 if trace_stream is not None:
1220                         trace_stream.close()
1221                         trace_stream = None
1222         except:
1223                 print_exception(sys.exc_info(),
1224                         '\nadt-run: error cleaning up:\n')
1225                 os._exit(20)
1226
1227 #---------- registration, installation etc. of .deb's: Binaries
1228
1229 def determine_package(act):
1230         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1231         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1232         if rc: badpkg('failed to parse binary package, code %d' % rc)
1233         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1234         act.pkg = None
1235         for l in output.split('\n'):
1236                 m = re.match(l)
1237                 if not m: continue
1238                 if act.pkg: badpkg('two Package: lines in control file')
1239                 act.pkg = m.groups()[0]
1240         if not act.pkg: badpkg('no good Package: line in control file')
1241
1242 class Binaries:
1243  def __init__(b):
1244         b.dir = TemporaryDir('binaries')
1245         b.dir.write()
1246         ok = False
1247
1248         if opts.gnupghome is None:
1249                 opts.gnupghome = tmpdir+'/gnupg'
1250
1251         b._debug('initialising')
1252         try:
1253                 for x in ['pubring','secring']:
1254                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1255                 ok = True
1256         except OSError, oe:
1257                 if oe.errno != errno.ENOENT: raise
1258
1259         if ok: b._debug('no key generation needed')
1260         else: b.genkey()
1261
1262  def _debug(b, s):
1263         debug('* '+s)
1264
1265  def genkey(b):
1266         b._debug('preparing for key generation')
1267
1268         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1269         mkdir_okexist(opts.gnupghome, 0700)
1270
1271         script = '''
1272   exec >&2
1273   cd "$1"
1274   cat <<"END" >key-gen-params
1275 Key-Type: DSA
1276 Key-Length: 1024
1277 Key-Usage: sign
1278 Name-Real: autopkgtest per-run key
1279 Name-Comment: do not trust this key
1280 Name-Email: autopkgtest@example.com
1281 END
1282   set -x
1283   gpg --homedir="$1" --batch --gen-key key-gen-params
1284                 '''
1285         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1286         rc = subprocess_cooked(cmdl, dbg=(genkey,script))[0]
1287         if rc:
1288                 try:
1289                         f = open(opts.gnupghome+'/key-gen-log')
1290                         tp = file.read()
1291                 except OSError, e: tp = e
1292                 pstderr(tp)
1293                 bomb('key generation failed, code %d' % rc)
1294
1295  def apt_configs(b):
1296         return {
1297                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1298         }
1299
1300  def apt_pkg_gdebi_script(b, arg, middle):
1301         script = [
1302                 'import apt_pkg',
1303                 'import urllib',
1304                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1305                 ]
1306         for (k,v) in b.apt_configs().iteritems():
1307                 v = urllib.quote(v)
1308                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1309                                 % (k, v))
1310         script += [
1311                 'from GDebi.Cache import Cache',
1312                 'cache = Cache()',
1313                 ]
1314         for m in middle:
1315                 script += m + [
1316                 'print res',
1317                 'print d.missingDeps',
1318                 'print d.requiredChanges',
1319                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1320                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1321                           'd._failureString)',
1322                 'cache.commit()',
1323                 ''
1324                 ]
1325         return '\n'.join(script)
1326  def apt_get(b):
1327         ag = ['apt-get','-qy']
1328         for kv in b.apt_configs().iteritems():
1329                 ag += ['-o', '%s=%s' % kv]
1330         return ag
1331
1332  def reset(b):
1333         b._debug('reset')
1334         rmtree('binaries', b.dir.read())
1335         b.dir.invalidate()
1336         b.dir.write()
1337         b.install = []
1338         b.blamed = []
1339         b.registered = set()
1340
1341  def register(b, act, pkg, af, forwhat, blamed):
1342         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1343                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1344
1345         if act.ah['deb_'+forwhat] == 'ignore': return
1346
1347         b.blamed += testbed.blamed
1348
1349         leafname = pkg+'.deb'
1350         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1351
1352         try: os.remove(dest.write())
1353         except OSError, oe:
1354                 if oe.errno != errno.ENOENT: raise e
1355
1356         try: os.link(af.read(), dest.write())
1357         except OSError, oe:
1358                 if oe.errno != errno.EXDEV: raise e
1359                 shutil.copy(af.read(), dest)
1360
1361         if act.ah['deb_'+forwhat] == 'install':
1362                 b.install.append(pkg)
1363
1364         b.registered.add(pkg)
1365
1366  def publish(b):
1367         b._debug('publish')
1368
1369         script = '''
1370   exec >&2
1371   cd "$1"
1372   apt-ftparchive packages . >Packages
1373   gzip <Packages >Packages.gz
1374   apt-ftparchive release . >Release
1375   rm -f Release.gpg
1376   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1377   gpg --homedir="$2" --batch --export >archive-key.pgp
1378                 '''
1379         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1380         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1381         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1382
1383         b.dir.invalidate(True)
1384         apt_source = b.dir.read(True)
1385
1386         so = TemporaryFile('vlds')
1387         script = '''
1388   exec 3>&1 >&2
1389   apt-key add archive-key.pgp
1390   echo "deb file://'''+apt_source+''' /" >sources.list
1391   cat /etc/apt/sources.list >>sources.list
1392   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1393     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1394   fi
1395   '''+ ' '.join(b.apt_get()) +''' update >&2
1396   cat /var/lib/dpkg/status >&3
1397                 '''
1398         testbed.mungeing_apt()
1399         rc = testbed.execute('apt-key', ['sh','-ec',script],
1400                                 so=so.write(True), cwd=b.dir.write(True),
1401                                 script=script)
1402         if rc: bomb('apt setup failed with exit code %d' % rc)
1403
1404         testbed.blamed += b.blamed
1405
1406         b._debug('publish reinstall checking...')
1407         pkgs_reinstall = set()
1408         pkg = None
1409         for l in open(so.read()):
1410                 if l.startswith('Package: '):
1411                         pkg = l[9:].rstrip()
1412                 elif l.startswith('Status: install '):
1413                         if pkg in b.registered:
1414                                 pkgs_reinstall.add(pkg)
1415                                 b._debug(' publish reinstall needs '+pkg)
1416
1417         if pkgs_reinstall:
1418                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1419                 what = 'apt-get-reinstall'
1420                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1421                         [pkg for pkg in pkgs_reinstall])
1422                 rc = testbed.execute(what, cmdl, script=None,
1423                                 timeout=timeouts['install'])
1424                 if rc: badpkg("installation of basic binarries failed,"
1425                                 " exit code %d" % rc)
1426
1427         b._debug('publish install...')
1428         for pkg in b.install:
1429                 what = 'apt-get-install-%s' % pkg
1430                 testbed.blame(pkg)
1431                 cmdl = b.apt_get() + ['install',pkg]
1432                 rc = testbed.execute(what, cmdl, script=None,
1433                                 timeout=timeouts['install'])
1434                 if rc: badpkg("installation of %s failed, exit code %d"
1435                                 % (pkg, rc))
1436
1437         b._debug('publish done')
1438
1439 #---------- processing of sources (building)
1440
1441 def source_rules_command(act,script,what,which,work,cwd,
1442                                 results_lines=0,xargs=[]):
1443         script = [      "exec 3>&1 >&2",
1444                         "set -x"        ] + script
1445         script = '\n'.join(script)
1446         so = TemporaryFile('%s-%s-results' % (what,which))
1447         rc = testbed.execute('%s-%s' % (what,which),
1448                         ['sh','-ec',script]+xargs, script=script,
1449                         so=so.write(True), cwd=cwd,
1450                         timeout=timeouts['build'])
1451         results = open(so.read()).read().rstrip('\n')
1452         if len(results): results = results.split("\n")
1453         else: results = []
1454         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1455         if results_lines is not None and len(results) != results_lines:
1456                 badpkg("got %d lines of results from %s where %d expected"
1457                         % (len(results), which, results_lines))
1458         if results_lines==1: return results[0]
1459         return results
1460
1461 def build_source(act, control_override):
1462         act.blame = 'arg:'+act.af.spec
1463         testbed.blame(act.blame)
1464         testbed.prepare1([])
1465         testbed.needs_reset()
1466
1467         what = act.what
1468         basename = act.af.spec
1469         debiancontrol = None
1470         act.binaries = []
1471
1472         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1473
1474         if act.kind == 'dsc':
1475                 dsc = act.af
1476                 dsc_file = open(dsc.read())
1477                 in_files = False
1478                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1479                 for l in dsc_file:
1480                         l = l.rstrip('\n')
1481                         if l.startswith('Files:'): in_files = True; continue
1482                         elif l.startswith('#'): pass
1483                         elif not l.startswith(' '):
1484                                 in_files = False
1485                                 if l.startswith('Source:'):
1486                                         act.blame = 'dsc:'+l[7:].strip()
1487                                         testbed.blame(act.blame)
1488                         if not in_files: continue
1489
1490                         m = fre.match(l)
1491                         if not m: badpkg(".dsc contains unparseable line"
1492                                         " in Files: `%s'" % l)
1493                         leaf = m.groups(0)[0]
1494                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1495                                         sibling=True)
1496                         subfile.read(True)
1497                 dsc.read(True)
1498
1499         if act.kind == 'ubtree':
1500                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1501                         act.af, 'debian/control')
1502                 dsc = TemporaryFile(what+'-fakedsc')
1503                 dsc_w = open(dsc.write(), 'w')
1504                 for l in open(debiancontrol.read()):
1505                         l = l.rstrip('\n')
1506                         if not len(l): break
1507                         print >>dsc_w, l
1508                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1509                 dsc_w.close()
1510
1511         if act.kind == 'dsc':
1512                 testbed.prepare2([])
1513                 script = binaries.apt_pkg_gdebi_script('', [[
1514                                 'from GDebi.DebPackage import DebPackage',
1515                                 'd = DebPackage(cache)',
1516                                 'res = d.satisfyDependsStr("dpkg-dev")',
1517                         ]])
1518                 cmdl = ['python','-c',script]
1519                 whatp = what+'-dpkgsource'
1520                 rc = testbed.execute(what, cmdl, script=script,
1521                                 timeout=timeouts['install'])
1522                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1523
1524         work = TemporaryDir(what+'-build')
1525         act.work = work
1526
1527         tmpdir = work.write(True)+'/tmpdir'
1528         tmpdir_script = [
1529                         'TMPDIR="$1"',
1530                         'rm -rf -- "$TMPDIR"',
1531                         'export TMPDIR',
1532                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1533                 ]
1534
1535         if act.kind == 'ubtree':
1536                 spec = '%s/real-tree' % work.write(True)
1537                 create_command = '''
1538                         rm -rf "$spec"
1539                         mkdir "$spec"
1540                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1541                         '''
1542                 initcwd = act.af.read(True)
1543
1544         if act.kind == 'dsc':
1545                 spec = dsc.read(True)
1546                 create_command = '''
1547                         dpkg-source -x $spec
1548                         '''
1549                 initcwd = work.write(True)
1550
1551         script = [
1552                 'spec="$2"',
1553                 'origpwd=`pwd`',
1554                 'cd '+work.write(True)
1555         ]
1556
1557         if opts.user:
1558                 script += ([ 'chown '+opts.user+' .' ] +
1559                         tmpdir_script +
1560                         [ 'spec="$spec" origpwd="$origpwd" '
1561                                 +opts.user_wrap(create_command) ])
1562         else:
1563                 script += (tmpdir_script +
1564                         [ create_command ])
1565
1566         script += [
1567                         'cd */.',
1568                         'pwd >&3',
1569                         'set +e; test -f debian/tests/control; echo $? >&3'
1570                 ]
1571         (result_pwd, control_test_rc) = source_rules_command(
1572                         act,script,what,'extract',work,
1573                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1574
1575         filter = act.ah['dsc_filter']
1576
1577         if control_test_rc == '1': act.missing_tests_control = True
1578
1579         # For optional builds:
1580         #
1581         # We might need to build the package because:
1582         #   - we want its binaries (filter isn't _ and at least one of the
1583         #       deb_... isn't ignore)
1584         #   - the test control file says so
1585         #       (assuming we have any tests)
1586
1587         class NeedBuildException: pass
1588         def build_needed(m):
1589                 debug_b('build needed for %s' % m)
1590                 raise NeedBuildException()
1591
1592         try:
1593                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1594                                       act.ah['deb_fortests'] != 'ignore'):
1595                         build_needed('binaries')
1596
1597                 result_pwd_af = InputDir(what+'-treeforcontrol',
1598                         result_pwd, True)
1599                 stanzas = read_control(act, result_pwd_af, control_override)
1600                 for stanza in stanzas:
1601                         for t in stanza[' tests']:
1602                                 if 'no-build-needed' not in t.feature_names:
1603                                         build_needed('test %s' % t.tname)
1604                                 for d in t.depends:
1605                                         if '@' in d:
1606                                                 build_needed('test %s '
1607                                                  'dependency %s' % (t.tname,d))
1608
1609                 debug_b('build not needed')
1610                 built = False
1611
1612         except NeedBuildException:
1613
1614                 if act.kind != 'dsc':
1615                         testbed.prepare2([])
1616
1617                 script = binaries.apt_pkg_gdebi_script(
1618                         dsc.read(True), [[
1619                         'from GDebi.DscSrcPackage import DscSrcPackage',
1620                         'd = DscSrcPackage(cache, arg)',
1621                         'res = d.checkDeb()',
1622                          ],[
1623                         'from GDebi.DebPackage import DebPackage',
1624                         'd = DebPackage(cache)',
1625                         'res = d.satisfyDependsStr("'+
1626                                         ','.join(build_essential)+
1627                                 '")',
1628                         ]])
1629
1630                 cmdl = ['python','-c',script]
1631                 whatp = what+'-builddeps'
1632                 rc = testbed.execute(what, cmdl, script=script,
1633                                 timeout=timeouts['install'])
1634                 if rc: badpkg('build-depends install failed,'
1635                               ' exit code %d' % rc)
1636
1637                 script = tmpdir_script + [
1638                         'cd "$2"',
1639                         'dpkg-checkbuilddeps',
1640                         opts.user_wrap('debian/rules build'),
1641                         ]
1642                 source_rules_command(act,script,what,'build',work,
1643                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1644
1645                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1646                         badpkg("results dir `%s' is not in expected parent"
1647                                " dir `%s'" % (result_pwd, work.read(True)))
1648
1649                 built = True
1650
1651         act.tests_tree = InputDir(what+'-tests-tree',
1652                                 work.read(True)+os.path.basename(result_pwd),
1653                                 True)
1654         if act.ah['dsc_tests']:
1655                 testbed.register_ephemeral(act.work)
1656                 testbed.register_ephemeral(act.tests_tree)
1657
1658         if not built:
1659                 act.blamed = []
1660                 return
1661
1662         act.blamed = copy.copy(testbed.blamed)
1663
1664         debug_b('filter=%s' % filter)
1665         if filter != '_':
1666                 script = tmpdir_script + [
1667                         'cd '+work.write(True)+'/*/.',
1668                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1669                         'cd ..',
1670                         'echo *.deb >&3',
1671                         ]
1672                 result_debs = source_rules_command(act,script,what,
1673                                 'binary',work,work.write(True),
1674                                 results_lines=1, xargs=['x',tmpdir])
1675                 if result_debs == '*': debs = []
1676                 else: debs = result_debs.split(' ')
1677                 debug_b('debs='+`debs`)
1678                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1679                 for deb in debs:
1680                         m = re.match(deb)
1681                         if not m: badpkg("badly-named binary `%s'" % deb)
1682                         pkg = m.groups()[0]
1683                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1684                         for pat in filter.split(','):
1685                                 debug_b('  pat=%s' % pat)
1686                                 if not fnmatch.fnmatchcase(pkg,pat):
1687                                         debug_b('   no match')
1688                                         continue
1689                                 deb_what = pkg+'_'+what+'.deb'
1690                                 bin = RelativeInputFile(deb_what,work,deb,True)
1691                                 debug_b('  deb_what=%s, bin=%s' %
1692                                         (deb_what, str(bin)))
1693                                 binaries.register(act,pkg,bin,
1694                                         'forbuilds',testbed.blamed)
1695                                 act.binaries.append((pkg,bin))
1696                                 break
1697                 debug_b('all done.')
1698
1699 #---------- main processing loop and main program
1700
1701 def process_actions():
1702         global binaries
1703
1704         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1705         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1706         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1707
1708         debug_a1('starting')
1709         testbed.open()
1710         binaries = Binaries()
1711
1712         for act in opts.actions:
1713                 if act.af is not None and not act.af.spec_tbp:
1714                         testbed.register_ephemeral(act.af)
1715
1716         binaries.reset()
1717         control_override = None
1718
1719         debug_a1('builds ...')
1720         for act in opts.actions:
1721                 debug_a2('%s %s' %
1722                         (act.kind, act.what))
1723
1724                 if act.kind == 'control':
1725                         control_override = act.af
1726                 if act.kind == 'deb':
1727                         testbed.blame('arg:'+act.af.spec)
1728                         determine_package(act)
1729                         testbed.blame('deb:'+act.pkg)
1730                         binaries.register(act,act.pkg,act.af,
1731                                 'forbuilds',testbed.blamed)
1732                 if act.kind == 'dsc' or act.kind == 'ubtree':
1733                         build_source(act, control_override)
1734                 if act.kind == 'tree':
1735                         act.binaries = []
1736                 if act.kind.endswith('tree') or act.kind == 'dsc':
1737                         control_override = None
1738                 if act.kind == 'instantiate':
1739                         pass
1740
1741         debug_a1('builds done.')
1742
1743         binaries.reset()
1744         control_override = None
1745
1746         debug_a1('tests ...')
1747         for act in opts.actions:
1748                 debug_a2('test %s %s' % (act.kind, act.what))
1749
1750                 testbed.needs_reset()
1751                 if act.kind == 'control':
1752                         control_override = act.af
1753                 if act.kind == 'deb':
1754                         binaries.register(act,act.pkg,act.af,'fortests',
1755                                 ['deb:'+act.pkg])
1756                 if act.kind == 'dsc' or act.kind == 'ubtree':
1757                         for (pkg,bin) in act.binaries:
1758                                 binaries.register(act,pkg,bin,'fortests',
1759                                         act.blamed)
1760                 if act.kind == 'dsc':
1761                         if act.ah['dsc_tests']:
1762                                 debug_a3('read control ...')
1763                                 stanzas = read_control(act, act.tests_tree,
1764                                                 control_override)
1765                                 testbed.blamed += act.blamed
1766                                 debug_a3('run_tests ...')
1767                                 run_tests(stanzas, act.tests_tree)
1768                         control_override = None
1769                 if act.kind == 'tree' or act.kind == 'ubtree':
1770                         testbed.blame('arg:'+act.af.spec)
1771                         stanzas = read_control(act, act.af, control_override)
1772                         debug_a3('run_tests ...')
1773                         run_tests(stanzas, act.af)
1774                         control_override = None
1775                 if act.kind == 'instantiate':
1776                         testbed.prepare([])
1777         debug_a1('tests done.')
1778
1779 def main():
1780         global testbed
1781         global tmpdir
1782         try:
1783                 parse_args()
1784         except SystemExit, se:
1785                 os._exit(20)
1786         try:
1787                 setup_trace()
1788                 testbed = Testbed()
1789                 testbed.start()
1790                 finalise_options()
1791                 process_actions()
1792         except:
1793                 ec = print_exception(sys.exc_info(), '')
1794                 cleanup()
1795                 os._exit(ec)
1796         cleanup()
1797         os._exit(errorcode)
1798
1799 main()