runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':10, 'install':900, 'test':600, 'build':3000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         shutil.rmtree(pathname)
 192
 193 def debug_subprocess(what, cmdl=None, script=None):
 194         o = '$ '+what+':'
 195         if cmdl is not None:
 196                 ol = []
 197                 for x in cmdl:
 198                         if x is script: x = '<SCRIPT>'
 199                         ol.append(x.    replace('\\','\\\\').
 200                                         replace(' ','\\ ')      )
 201                 o += ' '+ ' '.join(ol)
 202         debug(o)
 203         if script is not None and opts.debuglevel >= 1:
 204                 o = ''
 205                 for l in script.rstrip('\n').split('\n'):
 206                         o += '$     '+l+'\n'
 207                 debug(o,1)
 208
 209 def flatten(l):
 210         return reduce((lambda a,b: a + b), l, [])
 211
 212 #---------- fancy automatic file-copying class
 213
 214 class AutoFile:
 215         # p.what
 216         # p.path[tb]    None or path    not None => path known
 217         # p.file[tb]    None or path    not None => file exists
 218         # p.spec        string or None
 219         # p.spec_tbp    True or False, or not set if spec is None
 220         # p.dir         '' or '/'
 221
 222  def __init__(p, what):
 223         p.what = what
 224         p.path = [None,None]
 225         p.file = [None,None]
 226         p.spec = None
 227         p.dir = ''
 228
 229  def __repr__(p):
 230         return "<AF@%s>" % p.__str__()
 231  def __str__(p):
 232         def ptbp(tbp):
 233                 if p.path[tbp] is None: return '-'+p.dir
 234                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 235                 else: return p.path[tbp]+p.dir+'!'
 236         out = p.what
 237         if p.spec is not None:
 238                 if not hasattr(p,'spec_tb'): out += '~'
 239                 elif p.spec_tbp: out += '#'
 240                 else: out += '='
 241                 out += p.spec
 242         out += ':'
 243         out += ptbp(False)
 244         out += '|'
 245         out += ptbp(True)
 246         return out
 247
 248  def _wrong(p, how):
 249         xtra = ''
 250         if p.spec is not None:
 251                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 252         raise ("internal error: %s (%s)" % (how, str(p)))
 253
 254  def _ensure_path(p, tbp):
 255         if p.path[tbp] is None:
 256                 if '/' in p.what:
 257                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 258                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 259                 if not tbp:
 260                         p.path[tbp] = tmpdir+'/'+p.what
 261                 else:
 262                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 263
 264  def write(p, tbp=False):
 265         p._debug('write %s...' % 'HT'[tbp])
 266         p._ensure_path(tbp)
 267
 268         if p.dir and not p.file[tbp]:
 269                 if not tbp:
 270                         p._debug('mkdir H')
 271                         mkdir_okexist(p.path[tbp])
 272                 else:
 273                         cmdl = ['sh','-ec',
 274                                 'test -d "$1" || mkdir -p "$1"',
 275                                 'x', p.path[tbp]]
 276                         tf_what = urllib.quote(p.what).replace('/','%2F')
 277                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 278                         if rc: bomb('failed to create directory %s' %
 279                                 p.path[tbp])
 280
 281         p.file[tbp] = p.path[tbp]
 282         return p.path[tbp]
 283
 284  def read(p, tbp=False):
 285         p._debug('read %s...' % 'HT'[tbp])
 286         p._ensure_path(tbp)
 287
 288         if p.file[tbp] is None:
 289                 if p.file[not tbp] is None:
 290                         p._wrong("requesting read but nonexistent")
 291                 cud = ['copyup','copydown'][tbp]
 292                 src = p.file[not tbp] + p.dir
 293                 dst = p.path[tbp] + p.dir
 294                 testbed.command(cud, (src, dst))
 295                 p.file[tbp] = p.path[tbp]
 296
 297         return p.file[tbp] + p.dir
 298
 299  def invalidate(p, tbp=False):
 300         p.file[tbp] = None
 301         p._debug('invalidated %s' % 'HT'[tbp])
 302
 303  def _debug(p, m):
 304         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 305
 306  def _constructed(p):
 307         p._debug('constructed: '+str(p))
 308         p._check()
 309
 310  def _check(p):
 311         for tbp in [False,True]:
 312          for pf in [p.path, p.file]:
 313                 if pf[tbp] is None: continue
 314                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 315                 if p.dir and pf[tbp].endswith('/'):
 316                         pf[tbp] = pf[tbp].rstrip('/')
 317                         if not pf[tbp]: pf[tbp] = '/'
 318                 if not p.dir and pf[tbp].endswith('/'):
 319                         bomb("directory `%s' specified for "
 320                              "non-directory %s" % (pf[tbp], p.what))
 321
 322  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 323         AutoFile.__init__(p,what)
 324         sh_on = ''; sh_sibl = ''
 325         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 326         if sibling: sh_sibl=' (sibling)'
 327         parent._debug('using as base: %s: %s%s%s' %
 328                         (str(parent), leaf, sh_on, sh_sibl))
 329         if not sibling and not parent.dir:
 330                 parent._wrong('asked for non-sibling relative path of non-dir')
 331         if sibling: trim = os.path.dirname
 332         else: trim = lambda x: x
 333         for tbp in [False,True]:
 334                 if parent.path[tbp] is None: continue
 335                 trimmed = trim(parent.path[tbp])
 336                 if trimmed: trimmed += '/'
 337                 p.path[tbp] = trimmed + leaf
 338                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 339                         p.file[tbp] = p.path[tbp]
 340
 341 class InputFile(AutoFile):
 342  def _init(p, what, spec, spec_tbp=False):
 343         AutoFile.__init__(p, what)
 344         p.spec = spec
 345         p.spec_tbp = spec_tbp
 346         p.path[spec_tbp] = spec
 347         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 348  def __init__(p, what, spec, spec_tbp=False):
 349         p._init(what,spec,spec_tbp)
 350         p._constructed()
 351
 352 class InputDir(InputFile):
 353  def __init__(p, what, spec, spec_tbp=False):
 354         InputFile._init(p,what,spec,spec_tbp)
 355         p.dir = '/'
 356         p._constructed()
 357
 358 class OutputFile(AutoFile):
 359  def _init(p, what, spec, spec_tbp=False):
 360         AutoFile.__init__(p, what)
 361         p.spec = spec
 362         p.spec_tbp = spec_tbp
 363         p.path[spec_tbp] = spec
 364  def __init__(p, what, spec, spec_tbp=False):
 365         p._init(what,spec,spec_tbp)
 366         p._constructed()
 367
 368 class OutputDir(OutputFile):
 369  def __init__(p, what, spec, spec_tbp=False):
 370         OutputFile._init(p,what,spec,spec_tbp)
 371         p.dir = '/'
 372         p._constructed()
 373
 374 class RelativeInputFile(AutoFile):
 375  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 376         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 377         p._constructed()
 378
 379 class RelativeOutputFile(AutoFile):
 380  def __init__(p, what, parent, leaf, sibling=False):
 381         p._relative_init(what, parent, leaf, None, False, sibling)
 382         p._constructed()
 383
 384 class TemporaryFile(AutoFile):
 385  def __init__(p, what):
 386         AutoFile.__init__(p, what)
 387         p._constructed()
 388
 389 class TemporaryDir(AutoFile):
 390  def __init__(p, what):
 391         AutoFile.__init__(p, what)
 392         p.dir = '/'
 393         p._constructed()
 394
 395 #---------- parsing and representation of the arguments
 396
 397 class Action:
 398  def __init__(a, kind, af, arghandling, what):
 399         # extra attributes get added during processing
 400         a.kind = kind
 401         a.af = af
 402         a.ah = arghandling
 403         a.what = what
 404         a.missing_tests_control = False
 405  def __repr__(a):
 406         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 407
 408 def parse_args():
 409         global opts
 410         global n_non_actions # argh, stupid python scoping rules
 411         usage = "%prog <options> --- <virt-server>..."
 412         parser = OptionParser(usage=usage)
 413         pa = parser.add_option
 414         pe = parser.add_option
 415
 416         arghandling = {
 417                 'dsc_tests': True,
 418                 'dsc_filter': '*',
 419                 'deb_forbuilds': 'auto',
 420                 'deb_fortests': 'auto',
 421                 'tb': False,
 422                 'override_control': None
 423         }
 424         initial_arghandling = arghandling.copy()
 425         n_non_actions = 0
 426
 427         #----------
 428         # actions (ie, test sets to run, sources to build, binaries to use):
 429
 430         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 431                 global n_non_actions
 432                 parser.largs.append((value,kindpath))
 433                 n_non_actions += not(is_act)
 434
 435         def pa_action(long, metavar, kindpath, help, is_act=True):
 436                 pa('','--'+long, action='callback', callback=cb_action,
 437                         nargs=1, type='string',
 438                         callback_args=(long,kindpath,is_act), help=help)
 439
 440         pa_action('built-tree',         'TREE', '@/',
 441                 help='run tests from build tree TREE')
 442
 443         pa_action('unbuilt-tree',       'TREE', '@//',
 444                 help='run tests from build tree TREE')
 445
 446         pa_action('source',             'DSC', '@.dsc',
 447                 help='build DSC and use its tests and/or'
 448                     ' generated binary packages')
 449
 450         pa_action('binary',             'DEB', '@.deb',
 451                help='use binary package DEB according'
 452                     ' to most recent --binaries-* settings')
 453
 454         def cb_actnoarg(op,optstr,value,parser, largsentry):
 455                 parser.largs.append(largsentry)
 456         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 457                 callback_args=((None, ('instantiate',)),),
 458                 help='instantiate testbed now (during testing phase)'
 459                      ' and install packages'
 460                      ' selected for automatic installation, even'
 461                      ' if this might apparently not be required otherwise')
 462
 463         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 464                help='run tests from control file CONTROL instead,'
 465                     ' (applies to next test suite only)')
 466
 467         #----------
 468         # argument handling settings (what ways to use action
 469         #  arguments, and pathname processing):
 470
 471         def cb_setah(option, opt_str, value, parser, toset,setval):
 472                 if type(setval) == list:
 473                         if not value in setval:
 474                                 parser.error('value for %s option (%s) is not '
 475                                  'one of the permitted values (%s)' %
 476                                  (value, opt_str, setval.join(' ')))
 477                 elif setval is not None:
 478                         value = setval
 479                 for v in toset:
 480                         arghandling[v] = value
 481                 parser.largs.append(arghandling.copy())
 482
 483         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 484                 type = metavar
 485                 if type is not None: type = 'string'
 486                 pa('',long, action='callback', callback=cb_setah,
 487                    callback_args=(affected,effect), **kwargs)
 488
 489         #---- paths: host or testbed:
 490         #
 491         pa_setah('--paths-testbed', ['tb'],True,
 492                 help='subsequent path specifications refer to the testbed')
 493         pa_setah('--paths-host', ['tb'],False,
 494                 help='subsequent path specifications refer to the host')
 495
 496         #---- source processing settings:
 497
 498         pa_setah('--sources-tests', ['dsc_tests'],True,
 499                 help='run tests from builds of subsequent sources')
 500         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 501                 help='do not run tests from builds of subsequent sources')
 502
 503         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 504                 type='string', metavar='PATTERN-LIST',
 505                 help='from subsequent sources, use binaries matching'
 506                      ' PATTERN-LIST (comma-separated glob patterns)'
 507                      ' according to most recent --binaries-* settings')
 508         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 509                 help='from subsequent sources, do not use any binaries')
 510
 511         #---- binary package processing settings:
 512
 513         def pa_setahbins(long,toset,how):
 514          pa_setah(long, toset,['ignore','auto','install'],
 515                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 516                 help=how+' ignore binaries, install them as needed'
 517                         ' for dependencies, or unconditionally install'
 518                         ' them, respectively')
 519         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 520         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 521         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 522
 523         #----------
 524         # general options:
 525
 526         def cb_vserv(op,optstr,value,parser):
 527                 parser.values.vserver = list(parser.rargs)
 528                 del parser.rargs[:]
 529
 530         def cb_path(op,optstr,value,parser, constructor,long,dir):
 531                 name = long.replace('-','_')
 532                 af = constructor(long, value,arghandling['tb'])
 533                 setattr(parser.values, name, af)
 534
 535         def pa_path(long, constructor, help, dir=False):
 536                 pa('','--'+long, action='callback', callback=cb_path,
 537                         callback_args=(constructor,long,dir),
 538                         nargs=1, type='string',
 539                         help=help, metavar='PATH')
 540
 541         pa_path('output-dir', OutputDir, dir=True,
 542                 help='write stderr/out files in PATH')
 543
 544         pa('','--tmp-dir',              type='string', dest='tmpdir',
 545                 help='write temporary files to TMPDIR, emptying it'
 546                      ' beforehand and leaving it behind at the end')
 547         pa('','--log-file',             type='string', dest='logfile',
 548                 help='write the log LOGFILE, emptying it beforehand,'
 549                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 550         pa('','--summary-file',         type='string', dest='summary',
 551                 help='write a summary report to SUMMARY,'
 552                      ' emptying it beforehand')
 553
 554         pa('','--user',                 type='string', dest='user',
 555                 help='run tests as USER (needs root on testbed)')
 556         pa('','--gain-root',            type='string', dest='gainroot',
 557                 help='prefix debian/rules binary with GAINROOT')
 558         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 559         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 560         pa('','--gnupg-home',           type='string', dest='gnupghome',
 561                 default='~/.autopkgtest/gpg',
 562                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 563                         " signing private apt archive);"
 564                         " `fresh' means generate new key each time.")
 565
 566         #----------
 567         # actual meat:
 568
 569         class SpecialOption(optparse.Option): pass
 570         vs_op = SpecialOption('','--VSERVER-DUMMY')
 571         vs_op.action = 'callback'
 572         vs_op.type = None
 573         vs_op.default = None
 574         vs_op.nargs = 0
 575         vs_op.callback = cb_vserv
 576         vs_op.callback_args = ( )
 577         vs_op.callback_kwargs = { }
 578         vs_op.help = 'introduces virtualisation server and args'
 579         vs_op._short_opts = []
 580         vs_op._long_opts = ['---']
 581
 582         pa(vs_op)
 583
 584         (opts,args) = parser.parse_args()
 585         if not hasattr(opts,'vserver'):
 586                 parser.error('you must specifiy --- <virt-server>...')
 587         if n_non_actions >= len(parser.largs):
 588                 parser.error('nothing to do specified')
 589
 590         arghandling = initial_arghandling
 591         opts.actions = []
 592         ix = 0
 593         for act in args:
 594                 if type(act) == dict:
 595                         arghandling = act
 596                         continue
 597                 elif type(act) == tuple:
 598                         pass
 599                 elif type(act) == str:
 600                         act = (act,act)
 601                 else:
 602                         raise ("unknown action in list `%s' having"
 603                               " type `%s'" % (act, type(act)))
 604                 (pathstr, kindpath) = act
 605
 606                 constructor = InputFile
 607                 if type(kindpath) is tuple:             kind = kindpath[0]
 608                 elif kindpath.endswith('.deb'):         kind = 'deb'
 609                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 610                 elif kindpath.endswith('//'):
 611                         kind = 'ubtree'
 612                         constructor = InputDir
 613                 elif kindpath.endswith('/'):
 614                         kind = 'tree'
 615                         constructor = InputDir
 616                 else: parser.error("do not know how to handle filename \`%s';"
 617                         " specify --source --binary or --build-tree")
 618
 619                 what = '%s%s' % (kind,ix); ix += 1
 620
 621                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 622                 else: fwhatx = '-'+kind
 623
 624                 if pathstr is None: af = None
 625                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 626                 opts.actions.append(Action(kind, af, arghandling, what))
 627
 628 def setup_trace():
 629         global trace_stream, tmpdir, summary_stream
 630
 631         if opts.tmpdir is not None:
 632                 rmtree('tmpdir(specified)',opts.tmpdir)
 633                 mkdir_okexist(opts.tmpdir, 0700)
 634                 tmpdir = opts.tmpdir
 635         else:
 636                 assert(tmpdir is None)
 637                 tmpdir = tempfile.mkdtemp()
 638
 639         if opts.logfile is None:
 640                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 641                         opts.logfile = opts.output_dir.spec + '/log'
 642                 elif opts.tmpdir is not None:
 643                         opts.logfile = opts.tmpdir + '/log'
 644         if opts.logfile is not None:
 645                 trace_stream = open(opts.logfile, 'w', 0)
 646         if opts.summary is not None:
 647                 summary_stream = open(opts.summary, 'w', 0)
 648
 649         debug('options: '+`opts`, 1)
 650
 651 def finalise_options():
 652         global opts, tb, build_essential
 653
 654         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 655                 opts.user = ''
 656
 657         if opts.user is None:
 658                 su = 'suggested-normal-user='
 659                 ul = [
 660                         e[len(su):]
 661                         for e in testbed.caps
 662                         if e.startswith(su)
 663                         ]
 664                 if ul:
 665                         opts.user = ul[0]
 666                 else:
 667                         opts.user = ''
 668
 669         if opts.user:
 670                 if 'root-on-testbed' not in testbed.caps:
 671                         pstderr("warning: virtualisation"
 672                                 " system does not offer root on testbed,"
 673                                 " but --user option specified: failure likely")
 674                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 675         else:
 676                 opts.user_wrap = lambda x: x
 677
 678         if opts.gainroot is None:
 679                 opts.gainroot = ''
 680                 if (opts.user or
 681                     'root-on-testbed' not in testbed.caps):
 682                         opts.gainroot = 'fakeroot'
 683                         build_essential += ['fakeroot']
 684
 685         if opts.gnupghome.startswith('~/'):
 686                 try: home = os.environ['HOME']
 687                 except KeyError:
 688                         parser.error("HOME environment variable"
 689                                 " not set, needed for --gnupghome=`%s"
 690                                 % opts.gnupghome)
 691                 opts.gnupghome = home + opts.gnupghome[1:]
 692         elif opts.gnupghome == 'fresh':
 693                 opts.gnupghome = None
 694
 695 #---------- testbed management - the Testbed class
 696
 697 class Testbed:
 698  def __init__(tb):
 699         tb.sp = None
 700         tb.lastsend = None
 701         tb.scratch = None
 702         tb.modified = False
 703         tb.blamed = []
 704         tb._ephemeral = []
 705         tb._debug('init')
 706         tb._need_reset_apt = False
 707  def _debug(tb, m, minlevel=0):
 708         debug('** '+m, minlevel)
 709  def start(tb):
 710         tb._debug('start')
 711         p = subprocess.PIPE
 712         debug_subprocess('vserver', opts.vserver)
 713         tb._errplumb = Errplumb(True)
 714         tb.sp = subprocess.Popen(opts.vserver,
 715                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 716         tb.expect('ok')
 717         tb.caps = tb.commandr('capabilities')
 718  def stop(tb):
 719         tb._debug('stop')
 720         tb.close()
 721         if tb.sp is None: return
 722         ec = tb.sp.returncode
 723         if ec is None:
 724                 tb.sp.stdout.close()
 725                 tb.send('quit')
 726                 tb.sp.stdin.close()
 727                 ec = tb.sp.wait()
 728         if ec:
 729                 tb.bomb('testbed gave exit status %d after quit' % ec)
 730         tb._errplumb.wait()
 731  def open(tb):
 732         tb._debug('open, scratch=%s' % tb.scratch)
 733         if tb.scratch is not None: return
 734         pl = tb.commandr('open')
 735         tb.scratch = InputDir('tb-scratch', pl[0], True)
 736         tb.deps_processed = []
 737  def mungeing_apt(tb):
 738         if not 'revert' in tb.caps:
 739                 tb._need_reset_apt = True
 740  def reset_apt(tb):
 741         if not tb._need_reset_apt: return
 742         what = 'aptget-update-reset'
 743         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 744         rc = tb.execute(what, cmdl, kind='install')
 745         if rc:
 746                 pstderr("\n" "warning: failed to restore"
 747                         " testbed apt cache, exit code %d" % rc)
 748         tb._need_reset_apt = False
 749  def close(tb):
 750         tb._debug('close, scratch=%s' % tb.scratch)
 751         if tb.scratch is None: return
 752         tb.scratch = None
 753         if tb.sp is None: return
 754         tb.command('close')
 755  def prepare1(tb, deps_new):
 756         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 757                 (tb.modified, tb.deps_processed, deps_new), 1)
 758         if 'revert' in tb.caps and (tb.modified or
 759             [d for d in tb.deps_processed if d not in deps_new]):
 760                 for af in tb._ephemeral: af.read(False)
 761                 tb._debug('reset **')
 762                 tb.command('revert')
 763                 tb.blamed = []
 764                 for af in tb._ephemeral: af.invalidate(True)
 765         tb.modified = False
 766  def prepare2(tb, deps_new):
 767         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 768         binaries.publish()
 769         tb._install_deps(deps_new)
 770  def prepare(tb, deps_new):
 771         tb.prepare1(deps_new)
 772         tb.prepare2(deps_new)
 773  def register_ephemeral(tb, af):
 774         tb._ephemeral.append(af)
 775  def _install_deps(tb, deps_new):
 776         tb._debug(' installing dependencies '+`deps_new`, 1)
 777         tb.deps_processed = deps_new
 778         if not deps_new: return
 779         dstr = ', '.join(deps_new)
 780         script = binaries.apt_pkg_gdebi_script(
 781                 dstr, [[
 782                 'from GDebi.DebPackage import DebPackage',
 783                 'd = DebPackage(cache)',
 784                 'res = d.satisfyDependsStr(arg)',
 785                 ]])
 786         cmdl = ['python','-c',script]
 787         what = 'install-deps'
 788         rc = testbed.execute(what+'-debinstall', cmdl, script=script,
 789                                 kind='install')
 790         if rc: badpkg('dependency install failed, exit code %d' % rc)
 791  def needs_reset(tb):
 792         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 793         tb.modified = True
 794  def blame(tb, m):
 795         tb._debug('blame += %s' % m, 1)
 796         tb.blamed.append(m)
 797  def bomb(tb, m):
 798         tb._debug('bomb %s' % m)
 799         if tb.sp is not None:
 800                 tb.sp.stdout.close()
 801                 tb.sp.stdin.close()
 802                 ec = tb.sp.wait()
 803                 if ec: pstderr('adt-run: testbed failing,'
 804                                 ' exit status %d' % ec)
 805         tb.sp = None
 806         raise Quit(16, 'testbed failed: %s' % m)
 807  def send(tb, string):
 808         tb.sp.stdin
 809         try:
 810                 debug('>> '+string, 2)
 811                 print >>tb.sp.stdin, string
 812                 tb.sp.stdin.flush()
 813                 tb.lastsend = string
 814         except:
 815                 (type, value, dummy) = sys.exc_info()
 816                 tb.bomb('cannot send to testbed: %s' % traceback.
 817                         format_exception_only(type, value))
 818  def expect(tb, keyword, nresults=None):
 819         l = tb.sp.stdout.readline()
 820         if not l: tb.bomb('unexpected eof from the testbed')
 821         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 822         l = l.rstrip('\n')
 823         debug('<< '+l, 2)
 824         ll = l.split()
 825         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 826         if ll[0] != keyword:
 827                 if tb.lastsend is None:
 828                         tb.bomb("got banner `%s', expected `%s...'" %
 829                                 (l, keyword))
 830                 else:
 831                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 832                                 (tb.lastsend, l, keyword))
 833         ll = ll[1:]
 834         if nresults is not None and len(ll) != nresults:
 835                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 836                         " expected %d result parameters" %
 837                         (string, l, len(ll), nresults))
 838         return ll
 839  def commandr(tb, cmd, args=(), nresults=None):
 840         # pass args=[None,...] or =(None,...) to avoid more url quoting
 841         if type(cmd) is str: cmd = [cmd]
 842         if len(args) and args[0] is None: args = args[1:]
 843         else: args = map(urllib.quote, args)
 844         al = cmd + args
 845         tb.send(string.join(al))
 846         ll = tb.expect('ok', nresults)
 847         rl = map(urllib.unquote, ll)
 848         return rl
 849  def command(tb, cmd, args=()):
 850         tb.commandr(cmd, args, 0)
 851  def commandr1(tb, cmd, args=()):
 852         rl = tb.commandr(cmd, args, 1)
 853         return rl[0]
 854  def execute(tb, what, cmdl,
 855                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 856                 script=False, tmpdir=None, kind='short'):
 857         # Options for script:
 858         #   False - do not call debug_subprocess, no synch. reporting required
 859         #   None or string - call debug_subprocess with that value,
 860         #                       plumb stderr through synchronously if possible
 861         # Options for se:
 862         #   None - usual Errplumb (output is of kind 2c)
 863         #   string - path on testbed (output is of kind 2d)
 864
 865         timeout = timeouts[kind]
 866
 867         if script is not False: debug_subprocess(what, cmdl, script=script)
 868         if cwd is None: cwd = tb.scratch.write(True)
 869
 870         xdump = None
 871         if se is None:
 872                 ep = Errplumb()
 873                 se_catch = TemporaryFile(what+'-xerr')
 874                 se_use = se_catch.write(True)
 875                 if not opts.quiet: xdump = 'debug=2-2'
 876                 elif trace_stream is not None:
 877                         xdump = 'debug=2-%d' % trace_stream.fileno()
 878         else:
 879                 ep = None
 880                 se_catch = None
 881                 se_use = se
 882
 883         cmdl = [None,
 884                 ','.join(map(urllib.quote, cmdl)),
 885                 si, so, se_use, cwd]
 886
 887         if timeout is not None and timeout > 0:
 888                 cmdl.append('timeout=%d' % timeout)
 889
 890         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 891         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 892         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 893
 894         rc = tb.commandr1('execute', cmdl)
 895         try: rc = int(rc)
 896         except ValueError: bomb("execute for %s gave invalid response `%s'"
 897                                         % (what,rc))
 898
 899         if se_catch is not None:
 900                 debug_file(se_catch.read())
 901         if ep is not None:
 902                 ep.wait()
 903
 904         return rc
 905
 906 #---------- representation of test control files: Field*, Test, etc.
 907
 908 class FieldBase:
 909  def __init__(f, fname, stz, base, tnames, vl):
 910         assert(vl)
 911         f.stz = stz
 912         f.base = base
 913         f.tnames = tnames
 914         f.vl = vl
 915  def words(f):
 916         def distribute(vle):
 917                 (lno, v) = vle
 918                 r = v.split()
 919                 r = map((lambda w: (lno, w)), r)
 920                 return r
 921         return flatten(map(distribute, f.vl))
 922  def atmostone(f):
 923         if len(vl) == 1:
 924                 (f.lno, f.v) = vl[0]
 925         else:
 926                 raise Unsupported(f.vl[1][0],
 927                         'only one %s field allowed' % fn)
 928         return f.v
 929
 930 class FieldIgnore(FieldBase):
 931  def parse(f): pass
 932
 933 class Restriction:
 934  def __init__(r,rname,base): pass
 935
 936 class Restriction_rw_build_tree(Restriction): pass
 937 class Restriction_breaks_testbed(Restriction):
 938  def __init__(r, rname, base):
 939         if 'revert' not in testbed.caps:
 940                 raise Unsupported(f.lno,
 941                         'Test breaks testbed but testbed cannot revert')
 942 class Restriction_needs_root(Restriction):
 943  def __init__(r, rname, base):
 944         if 'root-on-testbed' not in testbed.caps:
 945                 raise Unsupported(f.lno,
 946                         'Test needs root on testbed which is not available')
 947
 948 class Field_Restrictions(FieldBase):
 949  def parse(f):
 950         for wle in f.words():
 951                 (lno, rname) = wle
 952                 nrname = rname.replace('-','_')
 953                 try: rclass = globals()['Restriction_'+nrname]
 954                 except KeyError: raise Unsupported(lno,
 955                         'unknown restriction %s' % rname)
 956                 r = rclass(nrname, f.base)
 957                 f.base['restriction_names'].append(rname)
 958                 f.base['restrictions'].append(r)
 959
 960 class Field_Features(FieldIgnore):
 961  def parse(f):
 962         for wle in f.words():
 963                 (lno, fname) = wle
 964                 f.base['feature_names'].append(fname)
 965                 nfname = fname.replace('-','_')
 966                 try: fclass = globals()['Feature_'+nfname]
 967                 except KeyError: continue
 968                 ft = fclass(nfname, f.base)
 969                 f.base['features'].append(ft)
 970
 971 class Field_Tests(FieldIgnore): pass
 972
 973 class Field_Depends(FieldBase):
 974  def parse(f):
 975         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 976         dl = map(lambda x: x.strip(),
 977                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 978         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
 979         for d in dl:
 980                 if re.search(d):
 981                         badpkg("Test Depends field contains dependency"
 982                                " `%s' with invalid characters" % d)
 983         f.base['depends'] = dl
 984
 985 class Field_Tests_directory(FieldBase):
 986  def parse(f):
 987         td = atmostone(f)
 988         if td.startswith('/'): raise Unspported(f.lno,
 989                 'Tests-Directory may not be absolute')
 990         f.base['testsdir'] = td
 991
 992 def run_tests(stanzas, tree):
 993         global errorcode, testbed
 994         if stanzas == ():
 995                 report('*', 'SKIP no tests in this package')
 996                 errorcode |= 8
 997         for stanza in stanzas:
 998                 tests = stanza[' tests']
 999                 if not tests:
1000                         report('*', 'SKIP package has metadata but no tests')
1001                         errorcode |= 8
1002                 for t in tests:
1003                         t.prepare()
1004                         t.run(tree)
1005                         if 'breaks-testbed' in t.restriction_names:
1006                                 testbed.needs_reset()
1007                 testbed.needs_reset()
1008
1009 class Test:
1010  def __init__(t, tname, base, act):
1011         if '/' in tname: raise Unsupported(base[' lno'],
1012                 'test name may not contain / character')
1013         for k in base: setattr(t,k,base[k])
1014         t.tname = tname
1015         t.act = act
1016         t.what = act.what+'t-'+tname
1017         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1018         else: t.path = tname
1019         t._debug('constructed; path=%s' % t.path)
1020         t._debug(' .depends=%s' % t.depends)
1021  def _debug(t, m):
1022         debug('& %s: %s' % (t.what, m))
1023  def report(t, m):
1024         report(t.what, m)
1025  def reportfail(t, m):
1026         global errorcode
1027         errorcode |= 4
1028         report(t.what, 'FAIL ' + m)
1029  def prepare(t):
1030         t._debug('preparing')
1031         dn = []
1032         for d in t.depends:
1033                 t._debug(' processing dependency '+d)
1034                 if not '@' in d:
1035                         t._debug('  literal dependency '+d)
1036                         dn.append(d)
1037                 else:
1038                         for (pkg,bin) in t.act.binaries:
1039                                 d = d.replace('@',pkg)
1040                                 t._debug('  synthesised dependency '+d)
1041                                 dn.append(d)
1042         testbed.prepare(dn)
1043  def run(t, tree):
1044         t._debug('[----------------------------------------')
1045         def stdouterr(oe):
1046                 idstr = t.what + '-' + oe
1047                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1048                         use_dir = opts.output_dir
1049                 else:
1050                         use_dir = testbed.scratch
1051                 return RelativeOutputFile(idstr, use_dir, idstr)
1052
1053         if hasattr(t.act,'work'): t.act.work.write(True)
1054         tree.read(True)
1055
1056         af = RelativeInputFile(t.what, tree, t.path)
1057         so = stdouterr('stdout')
1058         se = stdouterr('stderr')
1059
1060         tf = af.read(True)
1061         tmpdir = None
1062
1063         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1064         if rc: bomb('failed to chmod +x %s' % tf)
1065
1066         if 'needs-root' not in t.restriction_names and opts.user is not None:
1067                 tfl = ['su',opts.user,'-c',tf]
1068                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1069                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1070                 if opts.user: script += '; chown %s "$1"' % opts.user
1071                 if 'rw-build-tree' in t.restriction_names:
1072                         script += '; chown -R %s "$2"' % opts.user
1073                 rc = testbed.execute('mktmpdir-'+t.what,
1074                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1075                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1076                                 % (tmpdir, rc))
1077         else:
1078                 tfl = [tf]
1079
1080         rc = testbed.execute('test-'+t.what, tfl,
1081                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1082                 tmpdir=tmpdir, kind='test')
1083
1084         so_read = so.read()
1085         se_read = se.read()
1086
1087         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1088         stab = os.stat(se_read)
1089         if stab.st_size != 0:
1090                 l = open(se_read).readline()
1091                 l = l.rstrip('\n \t\r')
1092                 if len(l) > 35: l = l[:35] + '...'
1093                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1094                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1095                 debug_file(se_read)
1096         elif rc != 0:
1097                 t.reportfail('non-zero exit status %d' % rc)
1098         else:
1099                 t.report('PASS')
1100
1101         stab = os.stat(so_read)
1102         if stab.st_size != 0:
1103                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1104                 debug_file(so_read)
1105
1106         t._debug('----------------------------------------]')
1107
1108 def read_control(act, tree, control_override):
1109         stanzas = [ ]
1110
1111         if control_override is not None:
1112                 control_af = control_override
1113                 testbed.blame('arg:'+control_override.spec)
1114         else:
1115                 if act.missing_tests_control:
1116                         return ()
1117                 control_af = RelativeInputFile(act.what+'-testcontrol',
1118                         tree, 'debian/tests/control')
1119         try:
1120                 control = open(control_af.read(), 'r')
1121         except (IOError,OSError), oe:
1122                 if oe[0] != errno.ENOENT: raise
1123                 return []
1124
1125         lno = 0
1126         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1127         stz = None      # stz[field_name][index] = (lno, value)
1128                         # special field names:
1129                         # stz[' lno'] = number
1130                         # stz[' tests'] = list of Test objects
1131         def end_stanza(stz):
1132                 if stz is None: return
1133                 stz[' errs'] = 0
1134                 stanzas.append(stz)
1135                 stz = None
1136                 hcurrent = None
1137
1138         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1139         while 1:
1140                 l = control.readline()
1141                 if not l: break
1142                 lno += 1
1143                 if not l.endswith('\n'): badctrl('unterminated line')
1144                 if regexp.compile('\s*\#').match(l): continue
1145                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1146                 initmat = initre.match(l)
1147                 if initmat:
1148                         (fname, l) = initmat.groups()
1149                         fname = string.capwords(fname)
1150                         if stz is None:
1151                                 stz = { ' lno': lno, ' tests': [] }
1152                         if not stz.has_key(fname): stz[fname] = [ ]
1153                         hcurrent = stz[fname]
1154                 elif regexp.compile('\s').match(l):
1155                         if not hcurrent: badctrl('unexpected continuation')
1156                 else:
1157                         badctrl('syntax error')
1158                 hcurrent.append((lno, l))
1159         end_stanza(stz)
1160
1161         def testbadctrl(stz, lno, m):
1162                 report_badctrl(lno, m)
1163                 stz[' errs'] += 1
1164
1165         for stz in stanzas:
1166                 try:
1167                         try: tnames = stz['Tests']
1168                         except KeyError:
1169                                 tnames = ['*']
1170                                 raise Unsupported(stz[' lno'],
1171                                         'no Tests field')
1172                         tnames = map((lambda lt: lt[1]), tnames)
1173                         tnames = string.join(tnames).split()
1174                         base = {
1175                                 'restriction_names': [],
1176                                 'restrictions': [],
1177                                 'feature_names': [],
1178                                 'features': [],
1179                                 'testsdir': 'debian/tests',
1180                                 'depends' : '@'
1181                         }
1182                         for fname in stz.keys():
1183                                 if fname.startswith(' '): continue
1184                                 vl = stz[fname]
1185                                 try: fclass = globals()['Field_'+
1186                                         fname.replace('-','_')]
1187                                 except KeyError: raise Unsupported(vl[0][0],
1188                                         'unknown metadata field %s' % fname)
1189                                 f = fclass(stz, fname, base, tnames, vl)
1190                                 f.parse()
1191                         for tname in tnames:
1192                                 t = Test(tname, base, act)
1193                                 stz[' tests'].append(t)
1194                 except Unsupported, u:
1195                         for tname in tnames: u.report(tname)
1196                         continue
1197
1198         return stanzas
1199
1200 def print_exception(ei, msgprefix=''):
1201         if msgprefix: pstderr(msgprefix)
1202         (et, q, tb) = ei
1203         if et is Quit:
1204                 pstderr('adt-run: ' + q.m)
1205                 psummary('quitting: '+q.m)
1206                 return q.ec
1207         else:
1208                 pstderr("adt-run: unexpected, exceptional, error:")
1209                 psummary('quitting: unexpected error, consult transcript')
1210                 traceback.print_exc(None, sys.stderr)
1211                 if trace_stream is not None:
1212                         traceback.print_exc(None, trace_stream)
1213                 return 20
1214
1215 def cleanup():
1216         global trace_stream
1217         try:
1218                 if testbed is not None:
1219                         testbed.reset_apt()
1220                         testbed.stop()
1221                 if opts.tmpdir is None and tmpdir is not None:
1222                         rmtree('tmpdir', tmpdir)
1223                 if trace_stream is not None:
1224                         trace_stream.close()
1225                         trace_stream = None
1226         except:
1227                 print_exception(sys.exc_info(),
1228                         '\nadt-run: error cleaning up:\n')
1229                 os._exit(20)
1230
1231 #---------- registration, installation etc. of .deb's: Binaries
1232
1233 def determine_package(act):
1234         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1235         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1236         if rc: badpkg('failed to parse binary package, code %d' % rc)
1237         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1238         act.pkg = None
1239         for l in output.split('\n'):
1240                 m = re.match(l)
1241                 if not m: continue
1242                 if act.pkg: badpkg('two Package: lines in control file')
1243                 act.pkg = m.groups()[0]
1244         if not act.pkg: badpkg('no good Package: line in control file')
1245
1246 class Binaries:
1247  def __init__(b):
1248         b.dir = TemporaryDir('binaries')
1249         b.dir.write()
1250         ok = False
1251
1252         if opts.gnupghome is None:
1253                 opts.gnupghome = tmpdir+'/gnupg'
1254
1255         b._debug('initialising')
1256         try:
1257                 for x in ['pubring','secring']:
1258                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1259                 ok = True
1260         except (IOError,OSError), oe:
1261                 if oe.errno != errno.ENOENT: raise
1262
1263         if ok: b._debug('no key generation needed')
1264         else: b.genkey()
1265
1266  def _debug(b, s):
1267         debug('* '+s)
1268
1269  def genkey(b):
1270         b._debug('preparing for key generation')
1271
1272         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1273         mkdir_okexist(opts.gnupghome, 0700)
1274
1275         script = '''
1276   exec >&2
1277   cd "$1"
1278   cat <<"END" >key-gen-params
1279 Key-Type: DSA
1280 Key-Length: 1024
1281 Key-Usage: sign
1282 Name-Real: autopkgtest per-run key
1283 Name-Comment: do not trust this key
1284 Name-Email: autopkgtest@example.com
1285 END
1286   set -x
1287   gpg --homedir="$1" --batch --gen-key key-gen-params
1288                 '''
1289         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1290         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1291         if rc: bomb('key generation failed, code %d' % rc)
1292
1293  def apt_configs(b):
1294         return {
1295                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1296         }
1297
1298  def apt_pkg_gdebi_script(b, arg, middle):
1299         script = [
1300                 'import apt_pkg',
1301                 'import urllib',
1302                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1303                 ]
1304         for (k,v) in b.apt_configs().iteritems():
1305                 v = urllib.quote(v)
1306                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1307                                 % (k, v))
1308         script += [
1309                 'from GDebi.Cache import Cache',
1310                 'cache = Cache()',
1311                 ]
1312         for m in middle:
1313                 script += m + [
1314                 'print res',
1315                 'print d.missingDeps',
1316                 'print d.requiredChanges',
1317                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1318                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1319                           'd._failureString)',
1320                 'cache.commit()',
1321                 ''
1322                 ]
1323         return '\n'.join(script)
1324  def apt_get(b):
1325         ag = ['apt-get','-qy']
1326         for kv in b.apt_configs().iteritems():
1327                 ag += ['-o', '%s=%s' % kv]
1328         return ' '.join(ag)
1329
1330  def reset(b):
1331         b._debug('reset')
1332         rmtree('binaries', b.dir.read())
1333         b.dir.invalidate()
1334         b.dir.write()
1335         b.install = []
1336         b.blamed = []
1337         b.registered = set()
1338
1339  def register(b, act, pkg, af, forwhat, blamed):
1340         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1341                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1342
1343         if act.ah['deb_'+forwhat] == 'ignore': return
1344
1345         b.blamed += testbed.blamed
1346
1347         leafname = pkg+'.deb'
1348         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1349
1350         try: os.remove(dest.write())
1351         except (IOError,OSError), oe:
1352                 if oe.errno != errno.ENOENT: raise e
1353
1354         try: os.link(af.read(), dest.write())
1355         except (IOError,OSError), oe:
1356                 if oe.errno != errno.EXDEV: raise e
1357                 shutil.copy(af.read(), dest)
1358
1359         if act.ah['deb_'+forwhat] == 'install':
1360                 b.install.append(pkg)
1361
1362         b.registered.add(pkg)
1363
1364  def publish(b):
1365         b._debug('publish')
1366
1367         script = '''
1368   exec >&2
1369   cd "$1"
1370   apt-ftparchive packages . >Packages
1371   gzip <Packages >Packages.gz
1372   apt-ftparchive release . >Release
1373   rm -f Release.gpg
1374   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1375   gpg --homedir="$2" --batch --export >archive-key.pgp
1376                 '''
1377         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1378         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1379         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1380
1381         b.dir.invalidate(True)
1382         apt_source = b.dir.read(True)
1383
1384         so = TemporaryFile('vlds')
1385         script = '''
1386   exec 3>&1 >&2
1387   apt-key add archive-key.pgp
1388   echo "deb file://'''+apt_source+''' /" >sources.list
1389   cat /etc/apt/sources.list >>sources.list
1390   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1391     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1392   fi
1393   '''+ b.apt_get() +''' update >&2
1394   cat /var/lib/dpkg/status >&3
1395                 '''
1396         testbed.mungeing_apt()
1397         rc = testbed.execute('apt-key', ['sh','-ec',script],
1398                                 so=so.write(True), cwd=b.dir.write(True),
1399                                 script=script, kind='install')
1400         if rc: bomb('apt setup failed with exit code %d' % rc)
1401
1402         testbed.blamed += b.blamed
1403
1404         b._debug('publish reinstall checking...')
1405         pkgs_reinstall = set()
1406         pkg = None
1407         for l in open(so.read()):
1408                 if l.startswith('Package: '):
1409                         pkg = l[9:].rstrip()
1410                 elif l.startswith('Status: install '):
1411                         if pkg in b.registered:
1412                                 pkgs_reinstall.add(pkg)
1413                                 b._debug(' publish reinstall needs '+pkg)
1414
1415         if pkgs_reinstall:
1416                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1417                 what = 'apt-get-reinstall'
1418                 cmdl = (b.apt_get() + ' --reinstall install '+
1419                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1420                 cmdl = ['sh','-c',cmdl]
1421                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1422                 if rc: badpkg("installation of basic binarries failed,"
1423                                 " exit code %d" % rc)
1424
1425         b._debug('publish install...')
1426         for pkg in b.install:
1427                 what = 'apt-get-install-%s' % pkg
1428                 testbed.blame(pkg)
1429                 cmdl = b.apt_get() + ' install ' + pkg + ' >&2'
1430                 cmdl = ['sh','-c',cmdl]
1431                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1432                 if rc: badpkg("installation of %s failed, exit code %d"
1433                                 % (pkg, rc))
1434
1435         b._debug('publish done')
1436
1437 #---------- processing of sources (building)
1438
1439 def source_rules_command(act,script,what,which,work,cwd,
1440                                 results_lines=0,xargs=[]):
1441         script = [      "exec 3>&1 >&2",
1442                         "set -x"        ] + script
1443         script = '\n'.join(script)
1444         so = TemporaryFile('%s-%s-results' % (what,which))
1445         rc = testbed.execute('%s-%s' % (what,which),
1446                         ['sh','-ec',script]+xargs, script=script,
1447                         so=so.write(True), cwd=cwd, kind='build')
1448         results = open(so.read()).read().rstrip('\n')
1449         if len(results): results = results.split("\n")
1450         else: results = []
1451         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1452         if results_lines is not None and len(results) != results_lines:
1453                 badpkg("got %d lines of results from %s where %d expected"
1454                         % (len(results), which, results_lines))
1455         if results_lines==1: return results[0]
1456         return results
1457
1458 def build_source(act, control_override):
1459         act.blame = 'arg:'+act.af.spec
1460         testbed.blame(act.blame)
1461         testbed.prepare1([])
1462         testbed.needs_reset()
1463
1464         what = act.what
1465         basename = act.af.spec
1466         debiancontrol = None
1467         act.binaries = []
1468
1469         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1470
1471         if act.kind == 'dsc':
1472                 dsc = act.af
1473                 dsc_file = open(dsc.read())
1474                 in_files = False
1475                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1476                 for l in dsc_file:
1477                         l = l.rstrip('\n')
1478                         if l.startswith('Files:'): in_files = True; continue
1479                         elif l.startswith('#'): pass
1480                         elif not l.startswith(' '):
1481                                 in_files = False
1482                                 if l.startswith('Source:'):
1483                                         act.blame = 'dsc:'+l[7:].strip()
1484                                         testbed.blame(act.blame)
1485                         if not in_files: continue
1486
1487                         m = fre.match(l)
1488                         if not m: badpkg(".dsc contains unparseable line"
1489                                         " in Files: `%s'" % l)
1490                         leaf = m.groups(0)[0]
1491                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1492                                         sibling=True)
1493                         subfile.read(True)
1494                 dsc.read(True)
1495
1496         if act.kind == 'ubtree':
1497                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1498                         act.af, 'debian/control')
1499                 dsc = TemporaryFile(what+'-fakedsc')
1500                 dsc_w = open(dsc.write(), 'w')
1501                 for l in open(debiancontrol.read()):
1502                         l = l.rstrip('\n')
1503                         if not len(l): break
1504                         print >>dsc_w, l
1505                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1506                 dsc_w.close()
1507
1508         if act.kind == 'dsc':
1509                 testbed.prepare2([])
1510                 script = binaries.apt_pkg_gdebi_script('', [[
1511                                 'from GDebi.DebPackage import DebPackage',
1512                                 'd = DebPackage(cache)',
1513                                 'res = d.satisfyDependsStr("dpkg-dev")',
1514                         ]])
1515                 cmdl = ['python','-c',script]
1516                 whatp = what+'-dpkgsource'
1517                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1518                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1519
1520         work = TemporaryDir(what+'-build')
1521         act.work = work
1522
1523         tmpdir = work.write(True)+'/tmpdir'
1524         tmpdir_script = [
1525                         'TMPDIR="$1"',
1526                         'rm -rf -- "$TMPDIR"',
1527                         'export TMPDIR',
1528                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1529                 ]
1530
1531         if act.kind == 'ubtree':
1532                 spec = '%s/real-tree' % work.write(True)
1533                 create_command = '''
1534                         rm -rf "$spec"
1535                         mkdir "$spec"
1536                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1537                         '''
1538                 initcwd = act.af.read(True)
1539
1540         if act.kind == 'dsc':
1541                 spec = dsc.read(True)
1542                 create_command = '''
1543                         dpkg-source -x $spec
1544                         '''
1545                 initcwd = work.write(True)
1546
1547         script = [
1548                 'spec="$2"',
1549                 'origpwd=`pwd`',
1550                 'cd '+work.write(True)
1551         ]
1552
1553         if opts.user:
1554                 script += ([ 'chown '+opts.user+' .' ] +
1555                         tmpdir_script +
1556                         [ 'spec="$spec" origpwd="$origpwd" '
1557                                 +opts.user_wrap(create_command) ])
1558         else:
1559                 script += (tmpdir_script +
1560                         [ create_command ])
1561
1562         script += [
1563                         'cd */.',
1564                         'pwd >&3',
1565                         'set +e; test -f debian/tests/control; echo $? >&3'
1566                 ]
1567         (result_pwd, control_test_rc) = source_rules_command(
1568                         act,script,what,'extract',work,
1569                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1570
1571         filter = act.ah['dsc_filter']
1572
1573         if control_test_rc == '1': act.missing_tests_control = True
1574
1575         # For optional builds:
1576         #
1577         # We might need to build the package because:
1578         #   - we want its binaries (filter isn't _ and at least one of the
1579         #       deb_... isn't ignore)
1580         #   - the test control file says so
1581         #       (assuming we have any tests)
1582
1583         class NeedBuildException: pass
1584         def build_needed(m):
1585                 debug_b('build needed for %s' % m)
1586                 raise NeedBuildException()
1587
1588         try:
1589                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1590                                       act.ah['deb_fortests'] != 'ignore'):
1591                         build_needed('binaries')
1592
1593                 result_pwd_af = InputDir(what+'-treeforcontrol',
1594                         result_pwd, True)
1595                 stanzas = read_control(act, result_pwd_af, control_override)
1596                 for stanza in stanzas:
1597                         for t in stanza[' tests']:
1598                                 if 'no-build-needed' not in t.feature_names:
1599                                         build_needed('test %s' % t.tname)
1600                                 for d in t.depends:
1601                                         if '@' in d:
1602                                                 build_needed('test %s '
1603                                                  'dependency %s' % (t.tname,d))
1604
1605                 debug_b('build not needed')
1606                 built = False
1607
1608         except NeedBuildException:
1609
1610                 if act.kind != 'dsc':
1611                         testbed.prepare2([])
1612
1613                 script = binaries.apt_pkg_gdebi_script(
1614                         dsc.read(True), [[
1615                         'from GDebi.DscSrcPackage import DscSrcPackage',
1616                         'd = DscSrcPackage(cache, arg)',
1617                         'res = d.checkDeb()',
1618                          ],[
1619                         'from GDebi.DebPackage import DebPackage',
1620                         'd = DebPackage(cache)',
1621                         'res = d.satisfyDependsStr("'+
1622                                         ','.join(build_essential)+
1623                                 '")',
1624                         ]])
1625
1626                 cmdl = ['python','-c',script]
1627                 whatp = what+'-builddeps'
1628                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1629                 if rc: badpkg('build-depends install failed,'
1630                               ' exit code %d' % rc)
1631
1632                 script = tmpdir_script + [
1633                         'cd "$2"',
1634                         'dpkg-checkbuilddeps',
1635                         opts.user_wrap('debian/rules build'),
1636                         ]
1637                 source_rules_command(act,script,what,'build',work,
1638                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1639
1640                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1641                         badpkg("results dir `%s' is not in expected parent"
1642                                " dir `%s'" % (result_pwd, work.read(True)))
1643
1644                 built = True
1645
1646         act.tests_tree = InputDir(what+'-tests-tree',
1647                                 work.read(True)+os.path.basename(result_pwd),
1648                                 True)
1649         if act.ah['dsc_tests']:
1650                 testbed.register_ephemeral(act.work)
1651                 testbed.register_ephemeral(act.tests_tree)
1652
1653         if not built:
1654                 act.blamed = []
1655                 return
1656
1657         act.blamed = copy.copy(testbed.blamed)
1658
1659         debug_b('filter=%s' % filter)
1660         if filter != '_':
1661                 script = tmpdir_script + [
1662                         'cd '+work.write(True)+'/*/.',
1663                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1664                         'cd ..',
1665                         'echo *.deb >&3',
1666                         ]
1667                 result_debs = source_rules_command(act,script,what,
1668                                 'binary',work,work.write(True),
1669                                 results_lines=1, xargs=['x',tmpdir])
1670                 if result_debs == '*': debs = []
1671                 else: debs = result_debs.split(' ')
1672                 debug_b('debs='+`debs`)
1673                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1674                 for deb in debs:
1675                         m = re.match(deb)
1676                         if not m: badpkg("badly-named binary `%s'" % deb)
1677                         pkg = m.groups()[0]
1678                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1679                         for pat in filter.split(','):
1680                                 debug_b('  pat=%s' % pat)
1681                                 if not fnmatch.fnmatchcase(pkg,pat):
1682                                         debug_b('   no match')
1683                                         continue
1684                                 deb_what = pkg+'_'+what+'.deb'
1685                                 bin = RelativeInputFile(deb_what,work,deb,True)
1686                                 debug_b('  deb_what=%s, bin=%s' %
1687                                         (deb_what, str(bin)))
1688                                 binaries.register(act,pkg,bin,
1689                                         'forbuilds',testbed.blamed)
1690                                 act.binaries.append((pkg,bin))
1691                                 break
1692                 debug_b('all done.')
1693
1694 #---------- main processing loop and main program
1695
1696 def process_actions():
1697         global binaries
1698
1699         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1700         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1701         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1702
1703         debug_a1('starting')
1704         testbed.open()
1705         binaries = Binaries()
1706
1707         for act in opts.actions:
1708                 if act.af is not None and not act.af.spec_tbp:
1709                         testbed.register_ephemeral(act.af)
1710
1711         binaries.reset()
1712         control_override = None
1713
1714         debug_a1('builds ...')
1715         for act in opts.actions:
1716                 debug_a2('%s %s' %
1717                         (act.kind, act.what))
1718
1719                 if act.kind == 'control':
1720                         control_override = act.af
1721                 if act.kind == 'deb':
1722                         testbed.blame('arg:'+act.af.spec)
1723                         determine_package(act)
1724                         testbed.blame('deb:'+act.pkg)
1725                         binaries.register(act,act.pkg,act.af,
1726                                 'forbuilds',testbed.blamed)
1727                 if act.kind == 'dsc' or act.kind == 'ubtree':
1728                         build_source(act, control_override)
1729                 if act.kind == 'tree':
1730                         act.binaries = []
1731                 if act.kind.endswith('tree') or act.kind == 'dsc':
1732                         control_override = None
1733                 if act.kind == 'instantiate':
1734                         pass
1735
1736         debug_a1('builds done.')
1737
1738         binaries.reset()
1739         control_override = None
1740
1741         debug_a1('tests ...')
1742         for act in opts.actions:
1743                 debug_a2('test %s %s' % (act.kind, act.what))
1744
1745                 testbed.needs_reset()
1746                 if act.kind == 'control':
1747                         control_override = act.af
1748                 if act.kind == 'deb':
1749                         binaries.register(act,act.pkg,act.af,'fortests',
1750                                 ['deb:'+act.pkg])
1751                 if act.kind == 'dsc' or act.kind == 'ubtree':
1752                         for (pkg,bin) in act.binaries:
1753                                 binaries.register(act,pkg,bin,'fortests',
1754                                         act.blamed)
1755                 if act.kind == 'dsc':
1756                         if act.ah['dsc_tests']:
1757                                 debug_a3('read control ...')
1758                                 stanzas = read_control(act, act.tests_tree,
1759                                                 control_override)
1760                                 testbed.blamed += act.blamed
1761                                 debug_a3('run_tests ...')
1762                                 run_tests(stanzas, act.tests_tree)
1763                         control_override = None
1764                 if act.kind == 'tree' or act.kind == 'ubtree':
1765                         testbed.blame('arg:'+act.af.spec)
1766                         stanzas = read_control(act, act.af, control_override)
1767                         debug_a3('run_tests ...')
1768                         run_tests(stanzas, act.af)
1769                         control_override = None
1770                 if act.kind == 'instantiate':
1771                         testbed.prepare([])
1772         debug_a1('tests done.')
1773
1774 def main():
1775         global testbed
1776         global tmpdir
1777         try:
1778                 parse_args()
1779         except SystemExit, se:
1780                 os._exit(20)
1781         try:
1782                 setup_trace()
1783                 testbed = Testbed()
1784                 testbed.start()
1785                 finalise_options()
1786                 process_actions()
1787         except:
1788                 ec = print_exception(sys.exc_info(), '')
1789                 cleanup()
1790                 os._exit(ec)
1791         cleanup()
1792         os._exit(errorcode)
1793
1794 main()