runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 binaries = None         # Binaries (.debs we have registered)
  49
  50 #---------- errors we define
  51
  52 class Quit:
  53         def __init__(q,ec,m): q.ec = ec; q.m = m
  54
  55 def bomb(m, se=''):
  56         print >>sys.stderr, se
  57         raise Quit(20, "unexpected error: %s" % m)
  58
  59 def badpkg(m, se=''):
  60         print >>sys.stderr, se
  61         print 'blame: ', ' '.join(testbed.blamed)
  62         raise Quit(12, "erroneous package: %s" % m)
  63
  64 def report(tname, result): print '%-20s %s' % (tname, result)
  65
  66 class Unsupported:
  67  def __init__(u, lno, m):
  68         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
  69         else: u.m = m
  70  def report(u, tname):
  71         global errorcode
  72         errorcode != 2
  73         report(tname, 'SKIP %s' % u.m)
  74
  75 def debug(m):
  76         global opts
  77         if not opts.debug: return
  78         for l in m.rstrip('\n').split('\n'):
  79                 print >>sys.stderr, 'atd-run: debug:', l
  80
  81 def mkdir_okexist(pathname, mode=02755):
  82         try:
  83                 os.mkdir(pathname, mode)
  84         except OSError, oe:
  85                 if oe.errno != errno.EEXIST: raise
  86
  87 def rmtree(what, pathname):
  88         debug('/ %s rmtree %s' % (what, pathname))
  89         shutil.rmtree(pathname)
  90
  91 def debug_subprocess(what, cmdl=None, script=None):
  92         o = '$ '+what+':'
  93         if cmdl is not None:
  94                 ol = []
  95                 for x in cmdl:
  96                         if x is script: x = '<SCRIPT>'
  97                         ol.append(x.    replace('\\','\\\\').
  98                                         replace(' ','\\ ')      )
  99                 o += ' '+ ' '.join(ol)
 100         if script is not None:
 101                 o += '\n'
 102                 for l in script.rstrip('\n').split('\n'):
 103                         o += '$     '+l+'\n'
 104         debug(o)
 105
 106 def flatten(l):
 107         return reduce((lambda a,b: a + b), l, [])
 108
 109 #---------- fancy automatic file-copying class
 110
 111 class AutoFile:
 112         # p.what
 113         # p.path[tb]    None or path    not None => path known
 114         # p.file[tb]    None or path    not None => file exists
 115         # p.spec        string or None
 116         # p.spec_tbp    True or False, or not set if spec is None
 117         # p.dir         '' or '/'
 118
 119  def __init__(p, what):
 120         p.what = what
 121         p.path = [None,None]
 122         p.file = [None,None]
 123         p.spec = None
 124         p.dir = ''
 125
 126  def __str__(p):
 127         def ptbp(tbp):
 128                 if p.path[tbp] is None: return '-'+p.dir
 129                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 130                 else: return p.path[tbp]+p.dir+'!'
 131         out = p.what
 132         if p.spec is not None:
 133                 if p.spec_tbp: out += '#'
 134                 else: out += '='
 135                 out += p.spec
 136         out += ':'
 137         out += ptbp(False)
 138         out += '|'
 139         out += ptbp(True)
 140         return out
 141
 142  def _wrong(p, how):
 143         xtra = ''
 144         if p.spec is not None: xtra = ' spec[%s]=%s' % (p.spec, p.spec_tb)
 145         raise ("internal error: %s (%s)" % (how, str(p)))
 146
 147  def _ensure_path(p, tbp):
 148         if p.path[tbp] is None:
 149                 if '/' in p.what:
 150                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 151                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 152                 if not tbp:
 153                         p.path[tbp] = tmpdir+'/'+p.what
 154                 else:
 155                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 156
 157  def write(p, tbp=False):
 158         p._debug('write %s...' % 'HT'[tbp])
 159         p._ensure_path(tbp)
 160
 161         if p.dir and not p.file[tbp]:
 162                 if not tbp:
 163                         p._debug('mkdir H')
 164                         mkdir_okexist(p.path[tbp])
 165                 else:
 166                         cmdl = ['sh','-ec',
 167                                 'test -d "$1" || mkdir -p "$1"',
 168                                 'x', p.path[tbp]]
 169                         tf_what = urllib.quote(p.what).replace('/','%2F')
 170                         (rc,se) = testbed.execute('mkdir-'+tf_what, cmdl)
 171                         if rc: bomb('failed to create directory %s' %
 172                                 p.path[tbp], se)
 173
 174         p.file[tbp] = p.path[tbp]
 175         return p.path[tbp]
 176
 177  def read(p, tbp=False):
 178         p._debug('read %s...' % 'HT'[tbp])
 179         p._ensure_path(tbp)
 180
 181         if p.file[tbp] is None:
 182                 if p.file[not tbp] is None:
 183                         p._wrong("requesting read but nonexistent")
 184                 cud = ['copyup','copydown'][tbp]
 185                 src = p.file[not tbp] + p.dir
 186                 dst = p.path[tbp] + p.dir
 187                 testbed.command(cud, (src, dst))
 188                 p.file[tbp] = p.path[tbp]
 189
 190         return p.file[tbp] + p.dir
 191
 192  def invalidate(p, tbp=False):
 193         p.file[tbp] = None
 194         p._debug('invalidated %s' % 'HT'[tbp])
 195
 196  def _debug(p, m):
 197         debug('/ %s#%x: %s' % (p.what, id(p), m))
 198
 199  def _constructed(p):
 200         p._debug('constructed: '+str(p))
 201         p._check()
 202
 203  def _check(p):
 204         for tbp in [False,True]:
 205          for pf in [p.path, p.file]:
 206                 if pf[tbp] is None: continue
 207                 if not pf[tbp]: bomb('empty path specified for '+p.what)
 208                 if p.dir and pf[tbp].endswith('/'):
 209                         pf[tbp] = pf[tbp].rstrip('/')
 210                         if not pf[tbp]: pf[tbp] = '/'
 211                 if not p.dir and pf[tbp].endswith('/'):
 212                         bomb("directory `%s' specified for "
 213                              "non-directory %s" % (pf[tbp], p.what))
 214
 215  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 216         AutoFile.__init__(p,what)
 217         sh_on = ''; sh_sibl = ''
 218         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 219         if sibling: sh_sibl=' (sibling)'
 220         parent._debug('using as base: %s: %s%s%s' %
 221                         (str(parent), leaf, sh_on, sh_sibl))
 222         if not sibling and not parent.dir:
 223                 parent._wrong('asked for non-sibling relative path of non-dir')
 224         if sibling: trim = os.path.dirname
 225         else: trim = lambda x: x
 226         for tbp in [False,True]:
 227                 if parent.path[tbp] is None: continue
 228                 trimmed = trim(parent.path[tbp])
 229                 if trimmed: trimmed += '/'
 230                 p.path[tbp] = trimmed + leaf
 231                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 232                         p.file[tbp] = p.path[tbp]
 233
 234 class InputFile(AutoFile):
 235  def _init(p, what, spec, spec_tbp=False):
 236         AutoFile.__init__(p, what)
 237         p.spec = spec
 238         p.spec_tbp = spec_tbp
 239         p.path[spec_tbp] = spec
 240         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 241  def __init__(p, what, spec, spec_tbp=False):
 242         p._init(what,spec,spec_tbp)
 243         p._constructed()
 244
 245 class InputDir(InputFile):
 246  def __init__(p, what, spec, spec_tbp=False):
 247         InputFile._init(p,what,spec,spec_tbp)
 248         p.dir = '/'
 249         p._constructed()
 250
 251 class OutputFile(AutoFile):
 252  def _init(p, what, spec, spec_tbp=False):
 253         AutoFile.__init__(p, what)
 254         p.spec = spec
 255         p.spec_tbp = spec_tbp
 256         p.path[spec_tbp] = spec
 257  def __init__(p, what, spec, spec_tbp=False):
 258         p._init(what,spec,spec_tbp)
 259         p._constructed()
 260
 261 class OutputDir(OutputFile):
 262  def __init__(p, what, spec, spec_tbp=False):
 263         OutputFile._init(p,what,spec,spec_tbp)
 264         p.dir = '/'
 265         p._constructed()
 266
 267 class RelativeInputFile(AutoFile):
 268  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 269         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 270         p._constructed()
 271
 272 class RelativeOutputFile(AutoFile):
 273  def __init__(p, what, parent, leaf, sibling=False):
 274         p._relative_init(what, parent, leaf, None, False, sibling)
 275         p._constructed()
 276
 277 class TemporaryFile(AutoFile):
 278  def __init__(p, what):
 279         AutoFile.__init__(p, what)
 280         p._constructed()
 281
 282 class TemporaryDir(AutoFile):
 283  def __init__(p, what):
 284         AutoFile.__init__(p, what)
 285         p.dir = '/'
 286         p._constructed()
 287
 288 #---------- parsing and representation of the arguments
 289
 290 class Action:
 291  def __init__(a, kind, af, arghandling, what):
 292         # extra attributes get added during processing
 293         a.kind = kind
 294         a.af = af
 295         a.ah = arghandling
 296         a.what = what
 297
 298 def parse_args():
 299         global opts
 300         usage = "%prog <options> --- <virt-server>..."
 301         parser = OptionParser(usage=usage)
 302         pa = parser.add_option
 303         pe = parser.add_option
 304
 305         arghandling = {
 306                 'dsc_tests': True,
 307                 'dsc_filter': '*',
 308                 'deb_forbuilds': 'auto',
 309                 'deb_fortests': 'auto',
 310                 'tb': False,
 311                 'override_control': None
 312         }
 313         initial_arghandling = arghandling.copy()
 314         n_non_actions = 0
 315
 316         #----------
 317         # actions (ie, test sets to run, sources to build, binaries to use):
 318
 319         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 320                 print >>sys.stderr, "cb_action", is_act
 321                 parser.largs.append((value,kindpath))
 322                 n_non_actions += not(is_act)
 323
 324         def pa_action(long, metavar, kindpath, help, is_act=True):
 325                 pa('','--'+long, action='callback', callback=cb_action,
 326                         nargs=1, type='string',
 327                         callback_args=(long,kindpath,is_act), help=help)
 328
 329         pa_action('build-tree',         'TREE', '@/',
 330                 help='run tests from build tree TREE')
 331
 332         pa_action('source',             'DSC', '@.dsc',
 333                 help='build DSC and use its tests and/or'
 334                     ' generated binary packages')
 335
 336         pa_action('binary',             'DEB', '@.deb',
 337                help='use binary package DEB according'
 338                     ' to most recent --binaries-* settings')
 339
 340         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 341                help='run tests from control file CONTROL instead,'
 342                     ' (applies to next test suite only)')
 343
 344         #----------
 345         # argument handling settings (what ways to use action
 346         #  arguments, and pathname processing):
 347
 348         def cb_setah(option, opt_str, value, parser, toset,setval):
 349                 if type(setval) == list:
 350                         if not value in setval:
 351                                 parser.error('value for %s option (%s) is not '
 352                                  'one of the permitted values (%s)' %
 353                                  (value, opt_str, setval.join(' ')))
 354                 elif setval is not None:
 355                         value = setval
 356                 for v in toset:
 357                         arghandling[v] = value
 358                 parser.largs.append(arghandling.copy())
 359
 360         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 361                 type = metavar
 362                 if type is not None: type = 'string'
 363                 pa('',long, action='callback', callback=cb_setah,
 364                    callback_args=(affected,effect), **kwargs)
 365
 366         #---- paths: host or testbed:
 367         #
 368         pa_setah('--paths-testbed', ['tb'],True,
 369                 help='subsequent path specifications refer to the testbed')
 370         pa_setah('--paths-host', ['tb'],False,
 371                 help='subsequent path specifications refer to the host')
 372
 373         #---- source processing settings:
 374
 375         pa_setah('--sources-tests', ['dsc_tests'],True,
 376                 help='run tests from builds of subsequent sources')
 377         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 378                 help='do not run tests from builds of subsequent sources')
 379
 380         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 381                 type='string', metavar='PATTERN-LIST',
 382                 help='from subsequent sources, use binaries matching'
 383                      ' PATTERN-LIST (comma-separated glob patterns)'
 384                      ' according to most recent --binaries-* settings')
 385         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 386                 help='from subsequent sources, do not use any binaries')
 387
 388         #---- binary package processing settings:
 389
 390         def pa_setahbins(long,toset,how):
 391          pa_setah(long, toset,['ignore','auto','install'],
 392                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 393                 help=how+' ignore binaries, install them as needed'
 394                         ' for dependencies, or unconditionally install'
 395                         ' them, respectively')
 396         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 397         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 398         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 399
 400         #----------
 401         # general options:
 402
 403         def cb_vserv(op,optstr,value,parser):
 404                 parser.values.vserver = list(parser.rargs)
 405                 del parser.rargs[:]
 406
 407         def cb_path(op,optstr,value,parser, constructor,long,dir):
 408                 name = long.replace('-','_')
 409                 af = constructor(arghandling['tb'], value, long, dir)
 410                 setattr(parser.values, name, af)
 411
 412         def pa_path(long, constructor, help, dir=False):
 413                 pa('','--'+long, action='callback', callback=cb_path,
 414                         callback_args=(constructor,long,dir),
 415                         nargs=1, type='string',
 416                         help=help, metavar='PATH')
 417
 418         pa_path('output-dir', OutputDir, dir=True,
 419                 help='write stderr/out files in PATH')
 420
 421         pa('','--tmp-dir',              type='string', dest='tmpdir',
 422                 help='write temporary files to TMPDIR, emptying it'
 423                      ' beforehand and leaving it behind at the end')
 424
 425         pa('','--user',                 type='string', dest='user',
 426                 help='run tests as USER (needs root on testbed)')
 427         pa('','--gain-root',            type='string', dest='gainroot',
 428                 help='prefix debian/rules binary with GAINROOT')
 429         pa('-d', '--debug', action='store_true', dest='debug');
 430         pa('','--gnupg-home',           type='string', dest='gnupghome',
 431                 default='~/.autopkgtest/gpg',
 432                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 433                         " signing private apt archive);"
 434                         " `fresh' means generate new key each time.")
 435
 436         #----------
 437         # actual meat:
 438
 439         class SpecialOption(optparse.Option): pass
 440         vs_op = SpecialOption('','--VSERVER-DUMMY')
 441         vs_op.action = 'callback'
 442         vs_op.type = None
 443         vs_op.default = None
 444         vs_op.nargs = 0
 445         vs_op.callback = cb_vserv
 446         vs_op.callback_args = ( )
 447         vs_op.callback_kwargs = { }
 448         vs_op.help = 'introduces virtualisation server and args'
 449         vs_op._short_opts = []
 450         vs_op._long_opts = ['---']
 451
 452         pa(vs_op)
 453
 454         (opts,args) = parser.parse_args()
 455         if not hasattr(opts,'vserver'):
 456                 parser.error('you must specifiy --- <virt-server>...')
 457         if n_non_actions >= len(parser.largs):
 458                 parser.error('nothing to do specified')
 459
 460         arghandling = initial_arghandling
 461         opts.actions = []
 462         ix = 0
 463         for act in args:
 464                 if type(act) == dict:
 465                         arghandling = act
 466                         continue
 467                 elif type(act) == tuple:
 468                         pass
 469                 elif type(act) == str:
 470                         act = (act,act)
 471                 else:
 472                         raise ("unknown action in list `%s' having"
 473                               " type `%s'" % (act, type(act)))
 474                 (pathstr, kindpath) = act
 475
 476                 constructor = InputFile
 477                 if type(kindpath) is tuple:             kind = kindpath[0]
 478                 elif kindpath.endswith('.deb'):         kind = 'deb'
 479                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 480                 elif kindpath.endswith('/'):
 481                         kind = 'tree'
 482                         constructor = InputDir
 483                 else: parser.error("do not know how to handle filename \`%s';"
 484                         " specify --source --binary or --build-tree")
 485
 486                 what = '%s%s' % (kind,ix); ix += 1
 487
 488                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 489                 else: fwhatx = '-'+kind
 490
 491                 af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 492                 opts.actions.append(Action(kind, af, arghandling, what))
 493
 494 def finalise_options():
 495         global opts, tb, tmpdir
 496
 497         if opts.tmpdir is not None:
 498                 rmtree('tmpdir(specified)',opts.tmpdir)
 499                 mkdir_okexist(opts.tmpdir, 0700)
 500                 tmpdir = opts.tmpdir
 501         else:
 502                 assert(tmpdir is None)
 503                 tmpdir = tempfile.mkdtemp()
 504
 505         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 506                 opts.user = ''
 507
 508         if opts.user is None:
 509                 su = 'suggested-normal-user='
 510                 ul = [
 511                         e[len(su):]
 512                         for e in testbed.caps
 513                         if e.startswith(su)
 514                         ]
 515                 if ul:
 516                         opts.user = ul[0]
 517                 else:
 518                         opts.user = ''
 519
 520         if opts.user:
 521                 if 'root-on-testbed' not in testbed.caps:
 522                         print >>sys.stderr, ("warning: virtualisation"
 523                                 " system does not offer root on testbed,"
 524                                 " but --user option specified: failure likely")
 525                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 526         else:
 527                 opts.user_wrap = lambda x: x
 528
 529         if opts.gainroot is None:
 530                 opts.gainroot = ''
 531                 if (opts.user or
 532                     'root-on-testbed' not in testbed.caps):
 533                         opts.gainroot = 'fakeroot'
 534
 535         if opts.gnupghome.startswith('~/'):
 536                 try: home = os.environ['HOME']
 537                 except KeyError:
 538                         parser.error("HOME environment variable"
 539                                 " not set, needed for --gnupghome=`%s"
 540                                 % opts.gnupghome)
 541                 opts.gnupghome = home + opts.gnupghome[1:]
 542         elif opts.gnupghome == 'fresh':
 543                 opts.gnupghome = None
 544
 545 #---------- testbed management - the Testbed class
 546
 547 class Testbed:
 548  def __init__(tb):
 549         tb.sp = None
 550         tb.lastsend = None
 551         tb.scratch = None
 552         tb.modified = False
 553         tb.blamed = []
 554         tb._debug('init')
 555  def _debug(tb, m):
 556         debug('** '+m)
 557  def start(tb):
 558         tb._debug('start')
 559         p = subprocess.PIPE
 560         debug_subprocess('vserver', opts.vserver)
 561         tb.sp = subprocess.Popen(opts.vserver,
 562                 stdin=p, stdout=p, stderr=None)
 563         tb.expect('ok')
 564         tb.caps = tb.commandr('capabilities')
 565         tb._need_reset_apt = False
 566  def stop(tb):
 567         tb._debug('stop')
 568         tb.close()
 569         if tb.sp is None: return
 570         ec = tb.sp.returncode
 571         if ec is None:
 572                 tb.sp.stdout.close()
 573                 tb.send('quit')
 574                 tb.sp.stdin.close()
 575                 ec = tb.sp.wait()
 576         if ec:
 577                 tb.bomb('testbed gave exit status %d after quit' % ec)
 578  def open(tb):
 579         tb._debug('open, scratch=%s' % tb.scratch)
 580         if tb.scratch is not None: return
 581         pl = tb.commandr('open')
 582         tb.scratch = InputDir('tb-scratch', pl[0], True)
 583         tb.deps_processed = []
 584  def mungeing_apt(tb):
 585         if not 'revert' in tb.caps:
 586                 tb._need_reset_apt = True
 587  def reset_apt(tb):
 588         if not tb._need_reset_apt: return
 589         what = 'aptget-update-reset'
 590         cmdl = ['apt-get','-qy','update']
 591         (rc,se) = tb.execute(what, cmdl)
 592         if rc:
 593                 print >>sys.stderr, se, ("\n" "warning: failed to restore"
 594                                 " testbed apt cache, exit code %d" % rc)
 595         tb._need_reset_apt = False
 596  def close(tb):
 597         tb._debug('close, scratch=%s' % tb.scratch)
 598         if tb.scratch is None: return
 599         tb.scratch = None
 600         if tb.sp is None: return
 601         tb.command('close')
 602  def prepare(tb, deps_new):
 603         tb._debug('prepare, modified=%s, deps_processed=%s, deps_new=%s' %
 604                 (tb.modified, tb.deps_processed, deps_new))
 605         if 'revert' in tb.caps and (tb.modified or
 606             [d for d in tb.deps_processed if d not in deps_new]):
 607                 tb._debug('reset **')
 608                 tb.command('reset')
 609                 tb.blamed = []
 610         binaries.publish()
 611         tb.modified = False
 612         tb._install_deps(deps_new)
 613  def _install_deps(tb, deps_new):
 614         tb._debug(' installing dependencies '+`deps_new`)
 615         tb.deps_processed = deps_new
 616         if not deps_new: return
 617         dstr = ', '.join(deps_new)
 618         script = binaries.apt_pkg_gdebi_script(
 619                 dstr, [[
 620                 'from GDebi.DebPackage import DebPackage',
 621                 'd = DebPackage(cache)',
 622                 'res = d.satisfyDependsStr(arg)',
 623                 ]])
 624         cmdl = ['python','-c',script]
 625         what = 'install-deps'
 626         debug_subprocess(what, cmdl, script=script)
 627         (rc,se) = testbed.execute(what, cmdl)
 628         if rc: badpkg('dependency install failed, exit code %d' % rc, se)
 629  def needs_reset(tb):
 630         tb._debug('needs_reset, previously=%s' % tb.modified)
 631         tb.modified = True
 632  def blame(tb, m):
 633         tb._debug('blame += %s' % m)
 634         tb.blamed.append(m)
 635  def bomb(tb, m):
 636         tb._debug('bomb %s' % m)
 637         if tb.sp is not None:
 638                 tb.sp.stdout.close()
 639                 tb.sp.stdin.close()
 640                 ec = tb.sp.wait()
 641                 if ec: print >>sys.stderr, ('adt-run: testbed failing,'
 642                         ' exit status %d' % ec)
 643         tb.sp = None
 644         raise Quit(16, 'testbed failed: %s' % m)
 645  def send(tb, string):
 646         tb.sp.stdin
 647         try:
 648                 debug('>> '+string)
 649                 print >>tb.sp.stdin, string
 650                 tb.sp.stdin.flush()
 651                 tb.lastsend = string
 652         except:
 653                 (type, value, dummy) = sys.exc_info()
 654                 tb.bomb('cannot send to testbed: %s' % traceback.
 655                         format_exception_only(type, value))
 656  def expect(tb, keyword, nresults=None):
 657         l = tb.sp.stdout.readline()
 658         if not l: tb.bomb('unexpected eof from the testbed')
 659         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 660         l = l.rstrip('\n')
 661         debug('<< '+l)
 662         ll = l.split()
 663         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 664         if ll[0] != keyword:
 665                 if tb.lastsend is None:
 666                         tb.bomb("got banner `%s', expected `%s...'" %
 667                                 (l, keyword))
 668                 else:
 669                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 670                                 (tb.lastsend, l, keyword))
 671         ll = ll[1:]
 672         if nresults is not None and len(ll) != nresults:
 673                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 674                         " expected %d result parameters" %
 675                         (string, l, len(ll), nresults))
 676         return ll
 677  def commandr(tb, cmd, args=(), nresults=None):
 678         # pass args=[None,...] or =(None,...) to avoid more url quoting
 679         if type(cmd) is str: cmd = [cmd]
 680         if len(args) and args[0] is None: args = args[1:]
 681         else: args = map(urllib.quote, args)
 682         al = cmd + args
 683         tb.send(string.join(al))
 684         ll = tb.expect('ok', nresults)
 685         rl = map(urllib.unquote, ll)
 686         return rl
 687  def command(tb, cmd, args=()):
 688         tb.commandr(cmd, args, 0)
 689  def commandr1(tb, cmd, args=()):
 690         rl = tb.commandr(cmd, args, 1)
 691         return rl[0]
 692  def execute(tb, what, cmdargs,
 693                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 694                 dump_fd=None):
 695         if cwd is None: cwd = tb.scratch.write(True)
 696         se_use = se
 697         if se_use is None:
 698                 se_af = TemporaryFile('xerr-'+what)
 699                 se_use = se_af.write(True)
 700         cmdl = [None,
 701                 ','.join(map(urllib.quote, cmdargs)),
 702                 si, so, se_use, cwd]
 703         if dump_fd is not None: cmdl += ['debug=%d-%d' % (dump_fd,2)]
 704         rc = tb.commandr1('execute', cmdl)
 705         try: rc = int(rc)
 706         except ValueError: bomb("execute for %s gave invalid response `%s'"
 707                                         % (what,rc))
 708         if se is not None: return rc
 709         return (rc, file(se_af.read()).read())
 710
 711 #---------- representation of test control files: Field*, Test, etc.
 712
 713 class FieldBase:
 714  def __init__(f, fname, stz, base, tnames, vl):
 715         assert(vl)
 716         f.stz = stz
 717         f.base = base
 718         f.tnames = tnames
 719         f.vl = vl
 720  def words(f):
 721         def distribute(vle):
 722                 (lno, v) = vle
 723                 r = v.split()
 724                 r = map((lambda w: (lno, w)), r)
 725                 return r
 726         return flatten(map(distribute, f.vl))
 727  def atmostone(f):
 728         if len(vl) == 1:
 729                 (f.lno, f.v) = vl[0]
 730         else:
 731                 raise Unsupported(f.vl[1][0],
 732                         'only one %s field allowed' % fn)
 733         return f.v
 734
 735 class FieldIgnore(FieldBase):
 736  def parse(f): pass
 737
 738 class Restriction:
 739  def __init__(r,rname,base): pass
 740
 741 class Restriction_rw_build_tree(Restriction): pass
 742 class Restriction_rw_tests_tree(Restriction): pass
 743 class Restriction_breaks_testbed(Restriction):
 744  def __init__(r, rname, base):
 745         if 'revert' not in testbed.caps:
 746                 raise Unsupported(f.lno,
 747                         'Test breaks testbed but testbed cannot reset')
 748
 749 class Field_Restrictions(FieldBase):
 750  def parse(f):
 751         for wle in f.words():
 752                 (lno, rname) = wle
 753                 rname = rname.replace('-','_')
 754                 try: rclass = globals()['Restriction_'+rname]
 755                 except KeyError: raise Unsupported(lno,
 756                         'unknown restriction %s' % rname)
 757                 r = rclass(rname, f.base)
 758                 f.base['restrictions'].append(r)
 759
 760 class Field_Tests(FieldIgnore): pass
 761
 762 class Field_Depends(FieldBase):
 763  def parse(f):
 764         dl = map(lambda x: x.strip(),
 765                 flatten(map(lambda x: x.split(','), f.vl)))
 766         re = regexp.compile('[^-.+:~]')
 767         for d in dl:
 768                 if re.search(d):
 769                         badpkg("Test Depends field contains dependency"
 770                                " `%s' with invalid characters" % d)
 771         f.base['depends'] = dl
 772
 773 class Field_Tests_directory(FieldBase):
 774  def parse(f):
 775         td = atmostone(f)
 776         if td.startswith('/'): raise Unspported(f.lno,
 777                 'Tests-Directory may not be absolute')
 778         f.base['testsdir'] = td
 779
 780 def run_tests(stanzas, tree):
 781         global errorcode, testbed
 782         for stanza in stanzas:
 783                 tests = stanza[' tests']
 784                 if not tests:
 785                         report('*', 'SKIP no tests in this package')
 786                         errorcode |= 8
 787                 for t in tests:
 788                         t.prepare()
 789                         t.run(tree)
 790                         if 'breaks-testbed' in t.restrictions:
 791                                 testbed.needs_reset()
 792                 testbed.needs_reset()
 793
 794 class Test:
 795  def __init__(t, tname, base, act):
 796         if '/' in tname: raise Unsupported(base[' lno'],
 797                 'test name may not contain / character')
 798         for k in base: setattr(t,k,base[k])
 799         t.tname = tname
 800         t.act = act
 801         t.what = act.what+'t-'+tname
 802         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
 803         else: t.path = tname
 804         t._debug('constructed; path=%s' % t.path)
 805         t._debug(' .depends=%s' % t.depends)
 806  def _debug(t, m):
 807         debug('& %s: %s' % (t.what, m))
 808  def report(t, m):
 809         report(t.what, m)
 810  def reportfail(t, m):
 811         global errorcode
 812         errorcode |= 4
 813         report(t.what, 'FAIL ' + m)
 814  def prepare(t):
 815         t._debug('preparing')
 816         dn = []
 817         for d in t.depends:
 818                 t._debug(' processing dependency '+d)
 819                 if not '*' in d:
 820                         t._debug('  literal dependency '+d)
 821                         dn.append(d)
 822                 else:
 823                         for (pkg,bin) in t.act.binaries:
 824                                 d = d.replace('*',pkg)
 825                                 t._debug('  synthesised dependency '+d)
 826                                 dn.append(d)
 827         testbed.prepare(dn)
 828  def run(t, tree):
 829         t._debug('running')
 830         def stdouterr(oe):
 831                 idstr = t.what + '-' + oe
 832                 if opts.output_dir is not None and opts.output_dir.tb:
 833                         use_dir = opts.output_dir
 834                 else:
 835                         use_dir = testbed.scratch
 836                 return RelativeOutputFile(idstr, use_dir, idstr)
 837
 838         t.act.work.write(True)
 839
 840         af = RelativeInputFile(t.what, tree, t.path)
 841         so = stdouterr('stdout')
 842         se = stdouterr('stderr')
 843         rc = testbed.execute('test-'+t.what,
 844                 [opts.user_wrap(af.read(True))],
 845                 so=so.write(True), se=se.write(True), cwd=tree.read(True))
 846
 847         stab = os.stat(se.read())
 848         if stab.st_size != 0:
 849                 l = file(se.read()).readline()
 850                 l = l.rstrip('\n \t\r')
 851                 if len(l) > 40: l = l[:40] + '...'
 852                 t.reportfail('status: %d, stderr: %s' % (rc, l))
 853         elif rc != 0:
 854                 t.reportfail('non-zero exit status %d' % rc)
 855         else:
 856                 t.report('PASS')
 857
 858 def read_control(act, tree, control_override):
 859         stanzas = [ ]
 860
 861         if control_override is not None:
 862                 control_af = control_override
 863                 testbed.blame('arg:'+control_override.spec)
 864         else:
 865                 control_af = RelativeInputFile(act.what+'-testcontrol',
 866                         tree, 'debian/tests/control')
 867         try:
 868                 control = file(control_af.read(), 'r')
 869         except OSError, oe:
 870                 if oe[0] != errno.ENOENT: raise
 871                 return []
 872
 873         lno = 0
 874         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
 875         stz = None      # stz[field_name][index] = (lno, value)
 876                         # special field names:
 877                         # stz[' lno'] = number
 878                         # stz[' tests'] = list of Test objects
 879         def end_stanza(stz):
 880                 if stz is None: return
 881                 stz[' errs'] = 0
 882                 stanzas.append(stz)
 883                 stz = None
 884                 hcurrent = None
 885
 886         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
 887         while 1:
 888                 l = control.readline()
 889                 if not l: break
 890                 lno += 1
 891                 if not l.endswith('\n'): badctrl('unterminated line')
 892                 if regexp.compile('\s*\#').match(l): continue
 893                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
 894                 initmat = initre.match(l)
 895                 if initmat:
 896                         (fname, l) = initmat.groups()
 897                         fname = string.capwords(fname)
 898                         if stz is None:
 899                                 stz = { ' lno': lno, ' tests': [] }
 900                         if not stz.has_key(fname): stz[fname] = [ ]
 901                         hcurrent = stz[fname]
 902                 elif regexp.compile('\s').match(l):
 903                         if not hcurrent: badctrl('unexpected continuation')
 904                 else:
 905                         badctrl('syntax error')
 906                 hcurrent.append((lno, l))
 907         end_stanza(stz)
 908
 909         def testbadctrl(stz, lno, m):
 910                 report_badctrl(lno, m)
 911                 stz[' errs'] += 1
 912
 913         for stz in stanzas:
 914                 try:
 915                         try: tnames = stz['Tests']
 916                         except KeyError:
 917                                 tnames = ['*']
 918                                 raise Unsupported(stz[' lno'],
 919                                         'no Tests field')
 920                         tnames = map((lambda lt: lt[1]), tnames)
 921                         tnames = string.join(tnames).split()
 922                         base = {
 923                                 'restrictions': [],
 924                                 'testsdir': 'debian/tests',
 925                                 'depends' : '*'
 926                         }
 927                         for fname in stz.keys():
 928                                 if fname.startswith(' '): continue
 929                                 vl = stz[fname]
 930                                 try: fclass = globals()['Field_'+
 931                                         fname.replace('-','_')]
 932                                 except KeyError: raise Unsupported(vl[0][0],
 933                                         'unknown metadata field %s' % fname)
 934                                 f = fclass(stz, fname, base, tnames, vl)
 935                                 f.parse()
 936                         for tname in tnames:
 937                                 t = Test(tname, base, act)
 938                                 stz[' tests'].append(t)
 939                 except Unsupported, u:
 940                         for tname in tnames: u.report(tname)
 941                         continue
 942
 943         return stanzas
 944
 945 def print_exception(ei, msgprefix=''):
 946         if msgprefix: print >>sys.stderr, msgprefix
 947         (et, q, tb) = ei
 948         if et is Quit:
 949                 print >>sys.stderr, 'adt-run:', q.m
 950                 return q.ec
 951         else:
 952                 print >>sys.stderr, "adt-run: unexpected, exceptional, error:"
 953                 traceback.print_exc()
 954                 return 20
 955
 956 def cleanup():
 957         try:
 958                 rm_ec = 0
 959                 if opts.tmpdir is None and tmpdir is not None:
 960                         rmtree('tmpdir', tmpdir)
 961                 if testbed is not None:
 962                         testbed.reset_apt()
 963                         testbed.stop()
 964                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
 965         except:
 966                 print_exception(sys.exc_info(),
 967                         '\nadt-run: error cleaning up:\n')
 968                 os._exit(20)
 969
 970 #---------- registration, installation etc. of .deb's: Binaries
 971
 972 def determine_package(act):
 973         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
 974         running = Popen(cmd, stdout=PIPE)
 975         output = running.communicate()[0]
 976         rc = running.wait()
 977         if rc: badpkg('failed to parse binary package, code %d' % rc)
 978         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
 979         act.pkg = None
 980         for l in '\n'.split(output):
 981                 m = re.match(output)
 982                 if not m: continue
 983                 if act.pkg: badpkg('two Package: lines in control file')
 984                 act.pkg = m.groups
 985         if not act.pkg: badpkg('no good Package: line in control file')
 986
 987 class Binaries:
 988  def __init__(b):
 989         b.dir = TemporaryDir('binaries')
 990         b.dir.write()
 991         ok = False
 992
 993         if opts.gnupghome is None:
 994                 opts.gnupghome = tmpdir+'/gnupg'
 995
 996         b._debug('initialising')
 997         try:
 998                 for x in ['pubring','secring']:
 999                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1000                 ok = True
1001         except OSError, oe:
1002                 if oe.errno != errno.ENOENT: raise
1003
1004         if ok: b._debug('no key generation needed')
1005         else: b.genkey()
1006
1007  def _debug(b, s):
1008         debug('* '+s)
1009
1010  def genkey(b):
1011         b._debug('preparing for key generation')
1012
1013         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1014         mkdir_okexist(opts.gnupghome, 0700)
1015
1016         script = '''
1017   cd "$1"
1018   exec >key-gen-log 2>&1
1019   cat <<"END" >key-gen-params
1020 Key-Type: DSA
1021 Key-Length: 1024
1022 Key-Usage: sign
1023 Name-Real: autopkgtest per-run key
1024 Name-Comment: do not trust this key
1025 Name-Email: autopkgtest@example.com
1026 END
1027   set -x
1028   gpg --homedir="$1" --batch --gen-key key-gen-params
1029                 '''
1030         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1031         debug_subprocess('genkey', cmdl, script=script)
1032         rc = subprocess.call(cmdl)
1033         if rc:
1034                 try:
1035                         f = open(opts.gnupghome+'/key-gen-log')
1036                         tp = file.read()
1037                 except OSError, e: tp = e
1038                 print >>sys.stderr, tp
1039                 bomb('key generation failed, code %d' % rc)
1040
1041  def apt_configs(b):
1042         return {
1043                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1044         }
1045
1046  def apt_pkg_gdebi_script(b, arg, middle):
1047         script = [
1048                 'import apt_pkg',
1049                 'import urllib',
1050                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1051                 ]
1052         for (k,v) in b.apt_configs().iteritems():
1053                 v = urllib.quote(v)
1054                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1055                                 % (k, v))
1056         script += [
1057                 'from GDebi.Cache import Cache',
1058                 'cache = Cache()',
1059                 ]
1060         for m in middle:
1061                 script += m + [
1062                 'print res',
1063                 'print d.missingDeps',
1064                 'print d.requiredChanges',
1065                 'assert(res)',
1066                 'cache.commit()',
1067                 ''
1068                 ]
1069         return '\n'.join(script)
1070  def apt_get(b):
1071         ag = ['apt-get','-qy']
1072         for kv in b.apt_configs().iteritems():
1073                 ag += ['-o', '%s=%s' % kv]
1074         return ag
1075
1076  def reset(b):
1077         b._debug('reset')
1078         rmtree('binaries', b.dir.read())
1079         b.dir.invalidate()
1080         b.dir.write()
1081         b.install = []
1082         b.blamed = []
1083         b.registered = set()
1084
1085  def register(b, act, pkg, af, forwhat, blamed):
1086         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1087                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1088
1089         if act.ah['deb_'+forwhat] == 'ignore': return
1090
1091         b.blamed += testbed.blamed
1092
1093         leafname = pkg+'.deb'
1094         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1095
1096         try: os.remove(dest.write())
1097         except OSError, oe:
1098                 if oe.errno != errno.ENOENT: raise e
1099
1100         try: os.link(af.read(), dest.write())
1101         except OSError, oe:
1102                 if oe.errno != errno.EXDEV: raise e
1103                 shutil.copy(af.read(), dest)
1104
1105         if act.ah['deb_'+forwhat] == 'install':
1106                 b.install.append(pkg)
1107
1108         b.registered.add(pkg)
1109
1110  def publish(b):
1111         b._debug('publish')
1112
1113         script = '''
1114   cd "$1"
1115   apt-ftparchive packages . >Packages
1116   gzip <Packages >Packages.gz
1117   apt-ftparchive release . >Release
1118   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1119   gpg --homedir="$2" --batch --export >archive-key.pgp
1120                 '''
1121         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1122         debug_subprocess('ftparchive', cmdl, script)
1123         rc = subprocess.call(cmdl)
1124         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1125
1126         b.dir.invalidate(True)
1127         apt_source = b.dir.read(True)
1128
1129         so = TemporaryFile('vlds')
1130         script = '''
1131   apt-key add archive-key.pgp >&2
1132   echo "deb file://'''+apt_source+''' /" >sources.list
1133   cat /etc/apt/sources.list >>sources.list
1134   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1135     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1136   fi
1137   '''+ ' '.join(b.apt_get()) +''' update >&2
1138   cat /var/lib/dpkg/status
1139                 '''
1140         testbed.mungeing_apt()
1141         debug_subprocess('apt-key', script=script)
1142         (rc,se) = testbed.execute('apt-key',
1143                                 ['sh','-ec',script],
1144                                 so=so.write(True), cwd=b.dir.write(True))
1145         if rc: bomb('apt setup failed with exit code %d' % rc, se)
1146
1147         testbed.blamed += b.blamed
1148
1149         b._debug('publish reinstall checking...')
1150         pkgs_reinstall = set()
1151         pkg = None
1152         for l in file(so.read()):
1153                 if l.startswith('Package: '):
1154                         pkg = l[9:].rstrip()
1155                 elif l.startswith('Status: install '):
1156                         if pkg in b.registered:
1157                                 pkgs_reinstall.add(pkg)
1158                                 b._debug(' publish reinstall needs '+pkg)
1159
1160         if pkgs_reinstall:
1161                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1162                 what = 'apt-get-reinstall'
1163                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1164                         [pkg for pkg in pkgs_reinstall])
1165                 debug_subprocess(what, cmdl)
1166                 (rc,se) = testbed.execute(what, cmdl)
1167                 if rc: badpkg("installation of basic binarries failed,"
1168                                 " exit code %d" % rc, se)
1169
1170         b._debug('publish install...')
1171         for pkg in b.install:
1172                 what = 'apt-get-install-%s' % pkg
1173                 testbed.blame(pkg)
1174                 cmdl = b.apt_get() + ['install',pkg]
1175                 debug_subprocess(what, cmdl)
1176                 (rc,se) = testbed.execute(what, cmdl)
1177                 if rc: badpkg("installation of %s failed, exit code %d"
1178                                 % (pkg, rc), se)
1179
1180         b._debug('publish done')
1181
1182 #---------- processing of sources (building)
1183
1184 def source_rules_command(act,script,what,which,work,results_lines=0):
1185         if opts.debug:
1186                 trace = "%s-%s-trace" % (what,which)
1187                 script = [      "mkfifo -m600 "+trace,
1188                                 "tee <"+trace+" /dev/stderr >&4 &",
1189                                 "exec >"+trace+" 2>&1"  ] + script
1190
1191         script = [      "exec 3>&1 >&2",
1192                         "set -x"        ] + script
1193         script = '\n'.join(script)
1194         so = TemporaryFile('%s-%s-results' % (what,which))
1195         se = TemporaryFile('%s-%s-log' % (what,which))
1196         debug_subprocess('source-rules-command/%s/%s' % (act.what, which),
1197                         script=script)
1198         rc = testbed.execute('%s-%s' % (what,which),
1199                         ['sh','-xec',script],
1200                         so=so.write(True), se=se.write(True),
1201                         cwd= work.write(True), dump_fd=4)
1202         results = file(so.read()).read().rstrip('\n').split("\n")
1203         se = file(se.read()).read()
1204         if rc: badpkg("%s failed with exit code %d" % (which,rc), se)
1205         if results_lines is not None and len(results) != results_lines:
1206                 badpkg("got %d lines of results from %s where %d expected"
1207                         % (len(results), which, results_lines), se)
1208         if results_lines==1: return results[0]
1209         return results
1210
1211 def build_source(act):
1212         act.blame = 'arg:'+act.af.spec
1213         testbed.blame(act.blame)
1214         testbed.needs_reset()
1215
1216         what = act.what
1217         dsc = act.af
1218         basename = dsc.spec
1219
1220         dsc_file = open(dsc.read())
1221         in_files = False
1222         fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1223         for l in dsc_file:
1224                 l = l.rstrip('\n')
1225                 if l.startswith('Files:'): in_files = True; continue
1226                 elif l.startswith('#'): pass
1227                 elif not l.startswith(' '):
1228                         in_files = False
1229                         if l.startswith('Source:'):
1230                                 act.blame = 'dsc:'+l[7:].strip()
1231                                 testbed.blame(act.blame)
1232                 if not in_files: continue
1233
1234                 m = fre.match(l)
1235                 if not m: badpkg(".dsc contains unparseable line"
1236                                 " in Files: `%s'" % l)
1237                 leaf = m.groups(0)[0]
1238                 subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1239                                 sibling=True)
1240                 subfile.read(True)
1241         dsc.read(True)
1242
1243         script = binaries.apt_pkg_gdebi_script(
1244                 dsc.read(True), [[
1245                 'from GDebi.DscSrcPackage import DscSrcPackage',
1246                 'd = DscSrcPackage(cache, arg)',
1247                 'res = d.checkDeb()',
1248                  ],[
1249                 'from GDebi.DebPackage import DebPackage',
1250                 'd = DebPackage(cache)',
1251                 'res = d.satisfyDependsStr("build-essential")',
1252                 ]])
1253         cmdl = ['python','-c',script]
1254         whatp = what+'-builddeps'
1255         debug_subprocess(whatp, cmdl, script=script)
1256         (rc,se) = testbed.execute(what, cmdl)
1257         if rc: badpkg('build-depends install failed, exit code %d' % rc, se)
1258
1259         work = TemporaryDir(what+'-build')
1260
1261         script = [
1262                         'cd '+work.write(True),
1263                 ]
1264         if opts.user: script += [
1265                         'chown '+opts.user+' .',
1266                         'dsc=dsc.read(True) '+
1267                                 opts.user_wrap('dpkg-source -x $dsc')
1268                 ]
1269         else: script += [
1270                         'dpkg-source -x '+dsc.read(True),
1271                 ]
1272         script += [
1273                         'cd */.',
1274                         'dpkg-checkbuilddeps',
1275                         'pwd >&3',
1276                         opts.user_wrap('debian/rules build'),
1277                 ]
1278         result_pwd = source_rules_command(act,script,what,'build',work,1)
1279
1280         if os.path.dirname(result_pwd)+'/' != work.read(True):
1281                 badpkg("results dir `%s' is not in expected parent dir `%s'"
1282                         % (result_pwd, work.read(True)))
1283
1284         act.work = work
1285         act.tests_tree = InputDir(what+'-tests-tree',
1286                                 work.read(True)+os.path.basename(result_pwd),
1287                                 True)
1288         if act.ah['dsc_tests']:
1289                 act.tests_tree.read()
1290                 act.tests_tree.invalidate(True)
1291                 act.work.invalidate(True)
1292
1293         act.blamed = copy.copy(testbed.blamed)
1294
1295         def debug_b(m): debug('* <dsc:%s> %s' % (act.what, m))
1296         act.binaries = []
1297         filter = act.ah['dsc_filter']
1298         debug_b('filter=%s' % filter)
1299         if filter != '_':
1300                 script = [
1301                         'cd '+work.write(True)+'/*/.',
1302                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1303                         'cd ..',
1304                         'echo *.deb >&3',
1305                         ]
1306                 result_debs = source_rules_command(act,script,what,
1307                                 'binary',work,1)
1308                 if result_debs == '*': debs = []
1309                 else: debs = result_debs.split(' ')
1310                 debug_b('debs='+`debs`)
1311                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1312                 for deb in debs:
1313                         m = re.match(deb)
1314                         if not m: badpkg("badly-named binary `%s'" % deb)
1315                         pkg = m.groups()[0]
1316                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1317                         for pat in filter.split(','):
1318                                 debug_b('  pat=%s' % pat)
1319                                 if not fnmatch.fnmatchcase(pkg,pat):
1320                                         debug_b('   no match')
1321                                         continue
1322                                 deb_what = pkg+'_'+what+'.deb'
1323                                 bin = RelativeInputFile(deb_what,work,deb,True)
1324                                 debug_b('  deb_what=%s, bin=%s' %
1325                                         (deb_what, str(bin)))
1326                                 binaries.register(act,pkg,bin,
1327                                         'forbuilds',testbed.blamed)
1328                                 act.binaries.append((pkg,bin))
1329                                 break
1330                 debug_b('all done.')
1331
1332 #---------- main processing loop and main program
1333
1334 def process_actions():
1335         global binaries
1336
1337         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1338         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1339         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1340
1341         debug_a1('starting')
1342         testbed.open()
1343         binaries = Binaries()
1344
1345         binaries.reset()
1346
1347         debug_a1('builds ...')
1348         for act in opts.actions:
1349                 debug_a2('%s %s' %
1350                         (act.kind, act.what))
1351
1352                 testbed.prepare([])
1353                 if act.kind == 'deb':
1354                         blame('arg:'+act.af.spec)
1355                         determine_package(act)
1356                         blame('deb:'+act.pkg)
1357                         binaries.register(act,act.pkg,act.af,
1358                                 'forbuilds',testbed.blamed)
1359                 if act.kind == 'dsc':
1360                         build_source(act)
1361                 if act.kind == 'tree':
1362                         act.binaries = []
1363
1364         debug_a1('builds done.')
1365
1366         binaries.reset()
1367         control_override = None
1368
1369         debug_a1('tests ...')
1370         for act in opts.actions:
1371                 debug_a2('test %s %s' % (act.kind, act.what))
1372
1373                 testbed.needs_reset()
1374                 if act.kind == 'control':
1375                         control_override = act.af
1376                 if act.kind == 'deb':
1377                         binaries.register(act,act.pkg,act.af,'fortests',
1378                                 ['deb:'+act.pkg])
1379                 if act.kind == 'dsc':
1380                         for (pkg,bin) in act.binaries:
1381                                 binaries.register(act,pkg,bin,'fortests',
1382                                         act.blamed)
1383                         if act.ah['dsc_tests']:
1384                                 debug_a3('read control ...')
1385                                 stanzas = read_control(act, act.tests_tree,
1386                                                 control_override)
1387                                 testbed.blamed += act.blamed
1388                                 debug_a3('run_tests ...')
1389                                 run_tests(stanzas, act.tests_tree)
1390                         control_override = None
1391                 if act.kind == 'tree':
1392                         testbed.blame('arg:'+act.af.spec)
1393                         stanzas = read_control(act, act.af, control_override)
1394                         debug_a3('run_tests ...')
1395                         run_tests(stanzas, act.af)
1396                         control_override = None
1397         debug_a1('tests done.')
1398
1399 def main():
1400         global testbed
1401         global tmpdir
1402         try:
1403                 parse_args()
1404         except SystemExit, se:
1405                 os._exit(20)
1406         try:
1407                 testbed = Testbed()
1408                 testbed.start()
1409                 finalise_options()
1410                 process_actions()
1411         except:
1412                 ec = print_exception(sys.exc_info(), '')
1413                 cleanup()
1414                 os._exit(ec)
1415         cleanup()
1416         os._exit(errorcode)
1417
1418 main()