runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 binaries = None         # Binaries (.debs we have registered)
  49
  50 #---------- errors we define
  51
  52 class Quit:
  53         def __init__(q,ec,m): q.ec = ec; q.m = m
  54
  55 def bomb(m, se=''):
  56         print >>sys.stderr, se
  57         raise Quit(20, "unexpected error: %s" % m)
  58
  59 def badpkg(m, se=''):
  60         print >>sys.stderr, se
  61         print 'blame: ', ' '.join(testbed.blamed)
  62         raise Quit(12, "erroneous package: %s" % m)
  63
  64 def report(tname, result): print '%-20s %s' % (tname, result)
  65
  66 class Unsupported:
  67  def __init__(u, lno, m):
  68         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
  69         else: u.m = m
  70  def report(u, tname):
  71         global errorcode
  72         errorcode != 2
  73         report(tname, 'SKIP %s' % u.m)
  74
  75 def debug(m):
  76         global opts
  77         if not opts.debug: return
  78         for l in m.rstrip('\n').split('\n'):
  79                 print >>sys.stderr, 'atd-run: debug:', l
  80
  81 def mkdir_okexist(pathname, mode=02755):
  82         try:
  83                 os.mkdir(pathname, mode)
  84         except OSError, oe:
  85                 if oe.errno != errno.EEXIST: raise
  86
  87 def rmtree(what, pathname):
  88         debug('/ %s rmtree %s' % (what, pathname))
  89         shutil.rmtree(pathname)
  90
  91 def debug_subprocess(what, cmdl=None, script=None):
  92         o = '$ '+what+':'
  93         if cmdl is not None:
  94                 ol = []
  95                 for x in cmdl:
  96                         if x is script: x = '<SCRIPT>'
  97                         ol.append(x.    replace('\\','\\\\').
  98                                         replace(' ','\\ ')      )
  99                 o += ' '+ ' '.join(ol)
 100         if script is not None:
 101                 o += '\n'
 102                 for l in script.rstrip('\n').split('\n'):
 103                         o += '$     '+l+'\n'
 104         debug(o)
 105
 106 def flatten(l):
 107         return reduce((lambda a,b: a + b), l, [])
 108
 109 #---------- fancy automatic file-copying class
 110
 111 class AutoFile:
 112         # p.what
 113         # p.path[tb]    None or path    not None => path known
 114         # p.file[tb]    None or path    not None => file exists
 115         # p.spec        string or None
 116         # p.spec_tbp    True or False, or not set if spec is None
 117         # p.dir         '' or '/'
 118
 119  def __init__(p, what):
 120         p.what = what
 121         p.path = [None,None]
 122         p.file = [None,None]
 123         p.spec = None
 124         p.dir = ''
 125
 126  def __str__(p):
 127         def ptbp(tbp):
 128                 if p.path[tbp] is None: return '-'+p.dir
 129                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 130                 else: return p.path[tbp]+p.dir+'!'
 131         out = p.what
 132         if p.spec is not None:
 133                 if p.spec_tbp: out += '#'
 134                 else: out += '='
 135                 out += p.spec
 136         out += ':'
 137         out += ptbp(False)
 138         out += '|'
 139         out += ptbp(True)
 140         return out
 141
 142  def _wrong(p, how):
 143         xtra = ''
 144         if p.spec is not None: xtra = ' spec[%s]=%s' % (p.spec, p.spec_tb)
 145         raise ("internal error: %s (%s)" % (how, str(p)))
 146
 147  def _ensure_path(p, tbp):
 148         if p.path[tbp] is None:
 149                 if '/' in p.what:
 150                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 151                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 152                 if not tbp:
 153                         p.path[tbp] = tmpdir+'/'+p.what
 154                 else:
 155                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 156
 157  def write(p, tbp=False):
 158         p._debug('write %s...' % 'HT'[tbp])
 159         p._ensure_path(tbp)
 160
 161         if p.dir and not p.file[tbp]:
 162                 if not tbp:
 163                         p._debug('mkdir H')
 164                         mkdir_okexist(p.path[tbp])
 165                 else:
 166                         cmdl = ['sh','-ec',
 167                                 'test -d "$1" || mkdir -p "$1"',
 168                                 'x', p.path[tbp]]
 169                         tf_what = urllib.quote(p.what).replace('/','%2F')
 170                         (rc,se) = testbed.execute('mkdir-'+tf_what, cmdl)
 171                         if rc: bomb('failed to create directory %s' %
 172                                 p.path[tbp], se)
 173
 174         p.file[tbp] = p.path[tbp]
 175         return p.path[tbp]
 176
 177  def read(p, tbp=False):
 178         p._debug('read %s...' % 'HT'[tbp])
 179         p._ensure_path(tbp)
 180
 181         if p.file[tbp] is None:
 182                 if p.file[not tbp] is None:
 183                         p._wrong("requesting read but nonexistent")
 184                 cud = ['copyup','copydown'][tbp]
 185                 src = p.file[not tbp] + p.dir
 186                 dst = p.path[tbp] + p.dir
 187                 testbed.command(cud, (src, dst))
 188                 p.file[tbp] = p.path[tbp]
 189
 190         return p.file[tbp] + p.dir
 191
 192  def invalidate(p, tbp=False):
 193         p.file[tbp] = None
 194         p._debug('invalidated %s' % 'HT'[tbp])
 195
 196  def _debug(p, m):
 197         debug('/ %s#%x: %s' % (p.what, id(p), m))
 198
 199  def _constructed(p):
 200         p._debug('constructed: '+str(p))
 201         p._check()
 202
 203  def _check(p):
 204         for tbp in [False,True]:
 205          for pf in [p.path, p.file]:
 206                 if pf[tbp] is None: continue
 207                 if not pf[tbp]: bomb('empty path specified for '+p.what)
 208                 if p.dir and pf[tbp].endswith('/'):
 209                         pf[tbp] = pf[tbp].rstrip('/')
 210                         if not pf[tbp]: pf[tbp] = '/'
 211                 if not p.dir and pf[tbp].endswith('/'):
 212                         bomb("directory `%s' specified for "
 213                              "non-directory %s" % (pf[tbp], p.what))
 214
 215  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 216         AutoFile.__init__(p,what)
 217         sh_on = ''; sh_sibl = ''
 218         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 219         if sibling: sh_sibl=' (sibling)'
 220         parent._debug('using as base: %s: %s%s%s' %
 221                         (str(parent), leaf, sh_on, sh_sibl))
 222         if not sibling and not parent.dir:
 223                 parent._wrong('asked for non-sibling relative path of non-dir')
 224         if sibling: trim = os.path.dirname
 225         else: trim = lambda x: x
 226         for tbp in [False,True]:
 227                 if parent.path[tbp] is None: continue
 228                 trimmed = trim(parent.path[tbp])
 229                 if trimmed: trimmed += '/'
 230                 p.path[tbp] = trimmed + leaf
 231                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 232                         p.file[tbp] = p.path[tbp]
 233
 234 class InputFile(AutoFile):
 235  def _init(p, what, spec, spec_tbp=False):
 236         AutoFile.__init__(p, what)
 237         p.spec = spec
 238         p.spec_tbp = spec_tbp
 239         p.path[spec_tbp] = spec
 240         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 241  def __init__(p, what, spec, spec_tbp=False):
 242         p._init(what,spec,spec_tbp)
 243         p._constructed()
 244
 245 class InputDir(InputFile):
 246  def __init__(p, what, spec, spec_tbp=False):
 247         InputFile._init(p,what,spec,spec_tbp)
 248         p.dir = '/'
 249         p._constructed()
 250
 251 class OutputFile(AutoFile):
 252  def _init(p, what, spec, spec_tbp=False):
 253         AutoFile.__init__(p, what)
 254         p.spec = spec
 255         p.spec_tbp = spec_tbp
 256         p.path[spec_tbp] = spec
 257  def __init__(p, what, spec, spec_tbp=False):
 258         p._init(what,spec,spec_tbp)
 259         p._constructed()
 260
 261 class OutputDir(OutputFile):
 262  def __init__(p, what, spec, spec_tbp=False):
 263         OutputFile._init(p,what,spec,spec_tbp)
 264         p.dir = '/'
 265         p._constructed()
 266
 267 class RelativeInputFile(AutoFile):
 268  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 269         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 270         p._constructed()
 271
 272 class RelativeOutputFile(AutoFile):
 273  def __init__(p, what, parent, leaf, sibling=False):
 274         p._relative_init(what, parent, leaf, None, False, sibling)
 275         p._constructed()
 276
 277 class TemporaryFile(AutoFile):
 278  def __init__(p, what):
 279         AutoFile.__init__(p, what)
 280         p._constructed()
 281
 282 class TemporaryDir(AutoFile):
 283  def __init__(p, what):
 284         AutoFile.__init__(p, what)
 285         p.dir = '/'
 286         p._constructed()
 287
 288 #---------- parsing and representation of the arguments
 289
 290 class Action:
 291  def __init__(a, kind, af, arghandling, what):
 292         # extra attributes get added during processing
 293         a.kind = kind
 294         a.af = af
 295         a.ah = arghandling
 296         a.what = what
 297
 298 def parse_args():
 299         global opts
 300         usage = "%prog <options> --- <virt-server>..."
 301         parser = OptionParser(usage=usage)
 302         pa = parser.add_option
 303         pe = parser.add_option
 304
 305         arghandling = {
 306                 'dsc_tests': True,
 307                 'dsc_filter': '*',
 308                 'deb_forbuilds': 'auto',
 309                 'deb_fortests': 'auto',
 310                 'tb': False,
 311                 'override_control': None
 312         }
 313         initial_arghandling = arghandling.copy()
 314         n_non_actions = 0
 315
 316         #----------
 317         # actions (ie, test sets to run, sources to build, binaries to use):
 318
 319         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 320                 print >>sys.stderr, "cb_action", is_act
 321                 parser.largs.append((value,kindpath))
 322                 n_non_actions += not(is_act)
 323
 324         def pa_action(long, metavar, kindpath, help, is_act=True):
 325                 pa('','--'+long, action='callback', callback=cb_action,
 326                         nargs=1, type='string',
 327                         callback_args=(long,kindpath,is_act), help=help)
 328
 329         pa_action('build-tree',         'TREE', '@/',
 330                 help='run tests from build tree TREE')
 331
 332         pa_action('source',             'DSC', '@.dsc',
 333                 help='build DSC and use its tests and/or'
 334                     ' generated binary packages')
 335
 336         pa_action('binary',             'DEB', '@.deb',
 337                help='use binary package DEB according'
 338                     ' to most recent --binaries-* settings')
 339
 340         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 341                help='run tests from control file CONTROL instead,'
 342                     ' (applies to next test suite only)')
 343
 344         #----------
 345         # argument handling settings (what ways to use action
 346         #  arguments, and pathname processing):
 347
 348         def cb_setah(option, opt_str, value, parser, toset,setval):
 349                 if type(setval) == list:
 350                         if not value in setval:
 351                                 parser.error('value for %s option (%s) is not '
 352                                  'one of the permitted values (%s)' %
 353                                  (value, opt_str, setval.join(' ')))
 354                 elif setval is not None:
 355                         value = setval
 356                 for v in toset:
 357                         arghandling[v] = value
 358                 parser.largs.append(arghandling.copy())
 359
 360         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 361                 type = metavar
 362                 if type is not None: type = 'string'
 363                 pa('',long, action='callback', callback=cb_setah,
 364                    callback_args=(affected,effect), **kwargs)
 365
 366         #---- paths: host or testbed:
 367         #
 368         pa_setah('--paths-testbed', ['tb'],True,
 369                 help='subsequent path specifications refer to the testbed')
 370         pa_setah('--paths-host', ['tb'],False,
 371                 help='subsequent path specifications refer to the host')
 372
 373         #---- source processing settings:
 374
 375         pa_setah('--sources-tests', ['dsc_tests'],True,
 376                 help='run tests from builds of subsequent sources')
 377         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 378                 help='do not run tests from builds of subsequent sources')
 379
 380         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 381                 type='string', metavar='PATTERN-LIST',
 382                 help='from subsequent sources, use binaries matching'
 383                      ' PATTERN-LIST (comma-separated glob patterns)'
 384                      ' according to most recent --binaries-* settings')
 385         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 386                 help='from subsequent sources, do not use any binaries')
 387
 388         #---- binary package processing settings:
 389
 390         def pa_setahbins(long,toset,how):
 391          pa_setah(long, toset,['ignore','auto','install'],
 392                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 393                 help=how+' ignore binaries, install them as needed'
 394                         ' for dependencies, or unconditionally install'
 395                         ' them, respectively')
 396         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 397         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 398         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 399
 400         #----------
 401         # general options:
 402
 403         def cb_vserv(op,optstr,value,parser):
 404                 parser.values.vserver = list(parser.rargs)
 405                 del parser.rargs[:]
 406
 407         def cb_path(op,optstr,value,parser, constructor,long,dir):
 408                 name = long.replace('-','_')
 409                 af = constructor(arghandling['tb'], value, long, dir)
 410                 setattr(parser.values, name, af)
 411
 412         def pa_path(long, constructor, help, dir=False):
 413                 pa('','--'+long, action='callback', callback=cb_path,
 414                         callback_args=(constructor,long,dir),
 415                         nargs=1, type='string',
 416                         help=help, metavar='PATH')
 417
 418         pa_path('output-dir', OutputDir, dir=True,
 419                 help='write stderr/out files in PATH')
 420
 421         pa('','--tmp-dir',              type='string', dest='tmpdir',
 422                 help='write temporary files to TMPDIR, emptying it'
 423                      ' beforehand and leaving it behind at the end')
 424
 425         pa('','--user',                 type='string', dest='user',
 426                 help='run tests as USER (needs root on testbed)')
 427         pa('','--gain-root',            type='string', dest='gainroot',
 428                 help='prefix debian/rules binary with GAINROOT')
 429         pa('-d', '--debug', action='store_true', dest='debug');
 430         pa('','--gnupg-home',           type='string', dest='gnupghome',
 431                 default='~/.autopkgtest/gpg',
 432                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 433                         " signing private apt archive);"
 434                         " `fresh' means generate new key each time.")
 435
 436         #----------
 437         # actual meat:
 438
 439         class SpecialOption(optparse.Option): pass
 440         vs_op = SpecialOption('','--VSERVER-DUMMY')
 441         vs_op.action = 'callback'
 442         vs_op.type = None
 443         vs_op.default = None
 444         vs_op.nargs = 0
 445         vs_op.callback = cb_vserv
 446         vs_op.callback_args = ( )
 447         vs_op.callback_kwargs = { }
 448         vs_op.help = 'introduces virtualisation server and args'
 449         vs_op._short_opts = []
 450         vs_op._long_opts = ['---']
 451
 452         pa(vs_op)
 453
 454         (opts,args) = parser.parse_args()
 455         if not hasattr(opts,'vserver'):
 456                 parser.error('you must specifiy --- <virt-server>...')
 457         if n_non_actions >= len(parser.largs):
 458                 parser.error('nothing to do specified')
 459
 460         arghandling = initial_arghandling
 461         opts.actions = []
 462         ix = 0
 463         for act in args:
 464                 if type(act) == dict:
 465                         arghandling = act
 466                         continue
 467                 elif type(act) == tuple:
 468                         pass
 469                 elif type(act) == str:
 470                         act = (act,act)
 471                 else:
 472                         raise ("unknown action in list `%s' having"
 473                               " type `%s'" % (act, type(act)))
 474                 (pathstr, kindpath) = act
 475
 476                 constructor = InputFile
 477                 if type(kindpath) is tuple:             kind = kindpath[0]
 478                 elif kindpath.endswith('.deb'):         kind = 'deb'
 479                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 480                 elif kindpath.endswith('/'):
 481                         kind = 'tree'
 482                         constructor = InputDir
 483                 else: parser.error("do not know how to handle filename \`%s';"
 484                         " specify --source --binary or --build-tree")
 485
 486                 what = '%s%s' % (kind,ix); ix += 1
 487
 488                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 489                 else: fwhatx = '-'+kind
 490
 491                 af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 492                 opts.actions.append(Action(kind, af, arghandling, what))
 493
 494 def finalise_options():
 495         global opts, tb, tmpdir
 496
 497         if opts.tmpdir is not None:
 498                 rmtree('tmpdir(specified)',opts.tmpdir)
 499                 mkdir_okexist(opts.tmpdir, 0700)
 500                 tmpdir = opts.tmpdir
 501         else:
 502                 assert(tmpdir is None)
 503                 tmpdir = tempfile.mkdtemp()
 504
 505         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 506                 opts.user = ''
 507
 508         if opts.user is None:
 509                 su = 'suggested-normal-user='
 510                 ul = [
 511                         e[len(su):]
 512                         for e in testbed.caps
 513                         if e.startswith(su)
 514                         ]
 515                 if ul:
 516                         opts.user = ul[0]
 517                 else:
 518                         opts.user = ''
 519
 520         if opts.user:
 521                 if 'root-on-testbed' not in testbed.caps:
 522                         print >>sys.stderr, ("warning: virtualisation"
 523                                 " system does not offer root on testbed,"
 524                                 " but --user option specified: failure likely")
 525                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 526         else:
 527                 opts.user_wrap = lambda x: x
 528
 529         if opts.gainroot is None:
 530                 opts.gainroot = ''
 531                 if (opts.user or
 532                     'root-on-testbed' not in testbed.caps):
 533                         opts.gainroot = 'fakeroot'
 534
 535         if opts.gnupghome.startswith('~/'):
 536                 try: home = os.environ['HOME']
 537                 except KeyError:
 538                         parser.error("HOME environment variable"
 539                                 " not set, needed for --gnupghome=`%s"
 540                                 % opts.gnupghome)
 541                 opts.gnupghome = home + opts.gnupghome[1:]
 542         elif opts.gnupghome == 'fresh':
 543                 opts.gnupghome = None
 544
 545 #---------- testbed management - the Testbed class
 546
 547 class Testbed:
 548  def __init__(tb):
 549         tb.sp = None
 550         tb.lastsend = None
 551         tb.scratch = None
 552         tb.modified = False
 553         tb.blamed = []
 554         tb._ephemeral = []
 555         tb._debug('init')
 556  def _debug(tb, m):
 557         debug('** '+m)
 558  def start(tb):
 559         tb._debug('start')
 560         p = subprocess.PIPE
 561         debug_subprocess('vserver', opts.vserver)
 562         tb.sp = subprocess.Popen(opts.vserver,
 563                 stdin=p, stdout=p, stderr=None)
 564         tb.expect('ok')
 565         tb.caps = tb.commandr('capabilities')
 566         tb._need_reset_apt = False
 567  def stop(tb):
 568         tb._debug('stop')
 569         tb.close()
 570         if tb.sp is None: return
 571         ec = tb.sp.returncode
 572         if ec is None:
 573                 tb.sp.stdout.close()
 574                 tb.send('quit')
 575                 tb.sp.stdin.close()
 576                 ec = tb.sp.wait()
 577         if ec:
 578                 tb.bomb('testbed gave exit status %d after quit' % ec)
 579  def open(tb):
 580         tb._debug('open, scratch=%s' % tb.scratch)
 581         if tb.scratch is not None: return
 582         pl = tb.commandr('open')
 583         tb.scratch = InputDir('tb-scratch', pl[0], True)
 584         tb.deps_processed = []
 585  def mungeing_apt(tb):
 586         if not 'revert' in tb.caps:
 587                 tb._need_reset_apt = True
 588  def reset_apt(tb):
 589         if not tb._need_reset_apt: return
 590         what = 'aptget-update-reset'
 591         cmdl = ['apt-get','-qy','update']
 592         (rc,se) = tb.execute(what, cmdl)
 593         if rc:
 594                 print >>sys.stderr, se, ("\n" "warning: failed to restore"
 595                                 " testbed apt cache, exit code %d" % rc)
 596         tb._need_reset_apt = False
 597  def close(tb):
 598         tb._debug('close, scratch=%s' % tb.scratch)
 599         if tb.scratch is None: return
 600         tb.scratch = None
 601         if tb.sp is None: return
 602         tb.command('close')
 603  def prepare(tb, deps_new):
 604         tb._debug('prepare, modified=%s, deps_processed=%s, deps_new=%s' %
 605                 (tb.modified, tb.deps_processed, deps_new))
 606         if 'revert' in tb.caps and (tb.modified or
 607             [d for d in tb.deps_processed if d not in deps_new]):
 608                 tb._debug('reset **')
 609                 tb.command('reset')
 610                 tb.blamed = []
 611                 for af in tb._ephemeral: af.invalidate(True)
 612         binaries.publish()
 613         tb.modified = False
 614         tb._install_deps(deps_new)
 615  def register_ephemeral(tb, af):
 616         tb._ephemeral.append(af)
 617  def _install_deps(tb, deps_new):
 618         tb._debug(' installing dependencies '+`deps_new`)
 619         tb.deps_processed = deps_new
 620         if not deps_new: return
 621         dstr = ', '.join(deps_new)
 622         script = binaries.apt_pkg_gdebi_script(
 623                 dstr, [[
 624                 'from GDebi.DebPackage import DebPackage',
 625                 'd = DebPackage(cache)',
 626                 'res = d.satisfyDependsStr(arg)',
 627                 ]])
 628         cmdl = ['python','-c',script]
 629         what = 'install-deps'
 630         debug_subprocess(what, cmdl, script=script)
 631         (rc,se) = testbed.execute(what, cmdl)
 632         if rc: badpkg('dependency install failed, exit code %d' % rc, se)
 633  def needs_reset(tb):
 634         tb._debug('needs_reset, previously=%s' % tb.modified)
 635         tb.modified = True
 636  def blame(tb, m):
 637         tb._debug('blame += %s' % m)
 638         tb.blamed.append(m)
 639  def bomb(tb, m):
 640         tb._debug('bomb %s' % m)
 641         if tb.sp is not None:
 642                 tb.sp.stdout.close()
 643                 tb.sp.stdin.close()
 644                 ec = tb.sp.wait()
 645                 if ec: print >>sys.stderr, ('adt-run: testbed failing,'
 646                         ' exit status %d' % ec)
 647         tb.sp = None
 648         raise Quit(16, 'testbed failed: %s' % m)
 649  def send(tb, string):
 650         tb.sp.stdin
 651         try:
 652                 debug('>> '+string)
 653                 print >>tb.sp.stdin, string
 654                 tb.sp.stdin.flush()
 655                 tb.lastsend = string
 656         except:
 657                 (type, value, dummy) = sys.exc_info()
 658                 tb.bomb('cannot send to testbed: %s' % traceback.
 659                         format_exception_only(type, value))
 660  def expect(tb, keyword, nresults=None):
 661         l = tb.sp.stdout.readline()
 662         if not l: tb.bomb('unexpected eof from the testbed')
 663         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 664         l = l.rstrip('\n')
 665         debug('<< '+l)
 666         ll = l.split()
 667         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 668         if ll[0] != keyword:
 669                 if tb.lastsend is None:
 670                         tb.bomb("got banner `%s', expected `%s...'" %
 671                                 (l, keyword))
 672                 else:
 673                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 674                                 (tb.lastsend, l, keyword))
 675         ll = ll[1:]
 676         if nresults is not None and len(ll) != nresults:
 677                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 678                         " expected %d result parameters" %
 679                         (string, l, len(ll), nresults))
 680         return ll
 681  def commandr(tb, cmd, args=(), nresults=None):
 682         # pass args=[None,...] or =(None,...) to avoid more url quoting
 683         if type(cmd) is str: cmd = [cmd]
 684         if len(args) and args[0] is None: args = args[1:]
 685         else: args = map(urllib.quote, args)
 686         al = cmd + args
 687         tb.send(string.join(al))
 688         ll = tb.expect('ok', nresults)
 689         rl = map(urllib.unquote, ll)
 690         return rl
 691  def command(tb, cmd, args=()):
 692         tb.commandr(cmd, args, 0)
 693  def commandr1(tb, cmd, args=()):
 694         rl = tb.commandr(cmd, args, 1)
 695         return rl[0]
 696  def execute(tb, what, cmdargs,
 697                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 698                 dump_fd=None):
 699         if cwd is None: cwd = tb.scratch.write(True)
 700         se_use = se
 701         if se_use is None:
 702                 se_af = TemporaryFile('xerr-'+what)
 703                 se_use = se_af.write(True)
 704         cmdl = [None,
 705                 ','.join(map(urllib.quote, cmdargs)),
 706                 si, so, se_use, cwd]
 707         if dump_fd is not None: cmdl += ['debug=%d-%d' % (dump_fd,2)]
 708         rc = tb.commandr1('execute', cmdl)
 709         try: rc = int(rc)
 710         except ValueError: bomb("execute for %s gave invalid response `%s'"
 711                                         % (what,rc))
 712         if se is not None: return rc
 713         return (rc, file(se_af.read()).read())
 714
 715 #---------- representation of test control files: Field*, Test, etc.
 716
 717 class FieldBase:
 718  def __init__(f, fname, stz, base, tnames, vl):
 719         assert(vl)
 720         f.stz = stz
 721         f.base = base
 722         f.tnames = tnames
 723         f.vl = vl
 724  def words(f):
 725         def distribute(vle):
 726                 (lno, v) = vle
 727                 r = v.split()
 728                 r = map((lambda w: (lno, w)), r)
 729                 return r
 730         return flatten(map(distribute, f.vl))
 731  def atmostone(f):
 732         if len(vl) == 1:
 733                 (f.lno, f.v) = vl[0]
 734         else:
 735                 raise Unsupported(f.vl[1][0],
 736                         'only one %s field allowed' % fn)
 737         return f.v
 738
 739 class FieldIgnore(FieldBase):
 740  def parse(f): pass
 741
 742 class Restriction:
 743  def __init__(r,rname,base): pass
 744
 745 class Restriction_rw_build_tree(Restriction): pass
 746 class Restriction_rw_tests_tree(Restriction): pass
 747 class Restriction_breaks_testbed(Restriction):
 748  def __init__(r, rname, base):
 749         if 'revert' not in testbed.caps:
 750                 raise Unsupported(f.lno,
 751                         'Test breaks testbed but testbed cannot reset')
 752
 753 class Field_Restrictions(FieldBase):
 754  def parse(f):
 755         for wle in f.words():
 756                 (lno, rname) = wle
 757                 rname = rname.replace('-','_')
 758                 try: rclass = globals()['Restriction_'+rname]
 759                 except KeyError: raise Unsupported(lno,
 760                         'unknown restriction %s' % rname)
 761                 r = rclass(rname, f.base)
 762                 f.base['restrictions'].append(r)
 763
 764 class Field_Tests(FieldIgnore): pass
 765
 766 class Field_Depends(FieldBase):
 767  def parse(f):
 768         dl = map(lambda x: x.strip(),
 769                 flatten(map(lambda x: x.split(','), f.vl)))
 770         re = regexp.compile('[^-.+:~]')
 771         for d in dl:
 772                 if re.search(d):
 773                         badpkg("Test Depends field contains dependency"
 774                                " `%s' with invalid characters" % d)
 775         f.base['depends'] = dl
 776
 777 class Field_Tests_directory(FieldBase):
 778  def parse(f):
 779         td = atmostone(f)
 780         if td.startswith('/'): raise Unspported(f.lno,
 781                 'Tests-Directory may not be absolute')
 782         f.base['testsdir'] = td
 783
 784 def run_tests(stanzas, tree):
 785         global errorcode, testbed
 786         for stanza in stanzas:
 787                 tests = stanza[' tests']
 788                 if not tests:
 789                         report('*', 'SKIP no tests in this package')
 790                         errorcode |= 8
 791                 for t in tests:
 792                         t.prepare()
 793                         t.run(tree)
 794                         if 'breaks-testbed' in t.restrictions:
 795                                 testbed.needs_reset()
 796                 testbed.needs_reset()
 797
 798 class Test:
 799  def __init__(t, tname, base, act):
 800         if '/' in tname: raise Unsupported(base[' lno'],
 801                 'test name may not contain / character')
 802         for k in base: setattr(t,k,base[k])
 803         t.tname = tname
 804         t.act = act
 805         t.what = act.what+'t-'+tname
 806         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
 807         else: t.path = tname
 808         t._debug('constructed; path=%s' % t.path)
 809         t._debug(' .depends=%s' % t.depends)
 810  def _debug(t, m):
 811         debug('& %s: %s' % (t.what, m))
 812  def report(t, m):
 813         report(t.what, m)
 814  def reportfail(t, m):
 815         global errorcode
 816         errorcode |= 4
 817         report(t.what, 'FAIL ' + m)
 818  def prepare(t):
 819         t._debug('preparing')
 820         dn = []
 821         for d in t.depends:
 822                 t._debug(' processing dependency '+d)
 823                 if not '*' in d:
 824                         t._debug('  literal dependency '+d)
 825                         dn.append(d)
 826                 else:
 827                         for (pkg,bin) in t.act.binaries:
 828                                 d = d.replace('*',pkg)
 829                                 t._debug('  synthesised dependency '+d)
 830                                 dn.append(d)
 831         testbed.prepare(dn)
 832  def run(t, tree):
 833         t._debug('running')
 834         def stdouterr(oe):
 835                 idstr = t.what + '-' + oe
 836                 if opts.output_dir is not None and opts.output_dir.tb:
 837                         use_dir = opts.output_dir
 838                 else:
 839                         use_dir = testbed.scratch
 840                 return RelativeOutputFile(idstr, use_dir, idstr)
 841
 842         t.act.work.write(True)
 843
 844         af = RelativeInputFile(t.what, tree, t.path)
 845         so = stdouterr('stdout')
 846         se = stdouterr('stderr')
 847         rc = testbed.execute('test-'+t.what,
 848                 [opts.user_wrap(af.read(True))],
 849                 so=so.write(True), se=se.write(True), cwd=tree.read(True))
 850
 851         stab = os.stat(se.read())
 852         if stab.st_size != 0:
 853                 l = file(se.read()).readline()
 854                 l = l.rstrip('\n \t\r')
 855                 if len(l) > 40: l = l[:40] + '...'
 856                 t.reportfail('status: %d, stderr: %s' % (rc, l))
 857         elif rc != 0:
 858                 t.reportfail('non-zero exit status %d' % rc)
 859         else:
 860                 t.report('PASS')
 861
 862 def read_control(act, tree, control_override):
 863         stanzas = [ ]
 864
 865         if control_override is not None:
 866                 control_af = control_override
 867                 testbed.blame('arg:'+control_override.spec)
 868         else:
 869                 control_af = RelativeInputFile(act.what+'-testcontrol',
 870                         tree, 'debian/tests/control')
 871         try:
 872                 control = file(control_af.read(), 'r')
 873         except OSError, oe:
 874                 if oe[0] != errno.ENOENT: raise
 875                 return []
 876
 877         lno = 0
 878         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
 879         stz = None      # stz[field_name][index] = (lno, value)
 880                         # special field names:
 881                         # stz[' lno'] = number
 882                         # stz[' tests'] = list of Test objects
 883         def end_stanza(stz):
 884                 if stz is None: return
 885                 stz[' errs'] = 0
 886                 stanzas.append(stz)
 887                 stz = None
 888                 hcurrent = None
 889
 890         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
 891         while 1:
 892                 l = control.readline()
 893                 if not l: break
 894                 lno += 1
 895                 if not l.endswith('\n'): badctrl('unterminated line')
 896                 if regexp.compile('\s*\#').match(l): continue
 897                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
 898                 initmat = initre.match(l)
 899                 if initmat:
 900                         (fname, l) = initmat.groups()
 901                         fname = string.capwords(fname)
 902                         if stz is None:
 903                                 stz = { ' lno': lno, ' tests': [] }
 904                         if not stz.has_key(fname): stz[fname] = [ ]
 905                         hcurrent = stz[fname]
 906                 elif regexp.compile('\s').match(l):
 907                         if not hcurrent: badctrl('unexpected continuation')
 908                 else:
 909                         badctrl('syntax error')
 910                 hcurrent.append((lno, l))
 911         end_stanza(stz)
 912
 913         def testbadctrl(stz, lno, m):
 914                 report_badctrl(lno, m)
 915                 stz[' errs'] += 1
 916
 917         for stz in stanzas:
 918                 try:
 919                         try: tnames = stz['Tests']
 920                         except KeyError:
 921                                 tnames = ['*']
 922                                 raise Unsupported(stz[' lno'],
 923                                         'no Tests field')
 924                         tnames = map((lambda lt: lt[1]), tnames)
 925                         tnames = string.join(tnames).split()
 926                         base = {
 927                                 'restrictions': [],
 928                                 'testsdir': 'debian/tests',
 929                                 'depends' : '*'
 930                         }
 931                         for fname in stz.keys():
 932                                 if fname.startswith(' '): continue
 933                                 vl = stz[fname]
 934                                 try: fclass = globals()['Field_'+
 935                                         fname.replace('-','_')]
 936                                 except KeyError: raise Unsupported(vl[0][0],
 937                                         'unknown metadata field %s' % fname)
 938                                 f = fclass(stz, fname, base, tnames, vl)
 939                                 f.parse()
 940                         for tname in tnames:
 941                                 t = Test(tname, base, act)
 942                                 stz[' tests'].append(t)
 943                 except Unsupported, u:
 944                         for tname in tnames: u.report(tname)
 945                         continue
 946
 947         return stanzas
 948
 949 def print_exception(ei, msgprefix=''):
 950         if msgprefix: print >>sys.stderr, msgprefix
 951         (et, q, tb) = ei
 952         if et is Quit:
 953                 print >>sys.stderr, 'adt-run:', q.m
 954                 return q.ec
 955         else:
 956                 print >>sys.stderr, "adt-run: unexpected, exceptional, error:"
 957                 traceback.print_exc()
 958                 return 20
 959
 960 def cleanup():
 961         try:
 962                 rm_ec = 0
 963                 if opts.tmpdir is None and tmpdir is not None:
 964                         rmtree('tmpdir', tmpdir)
 965                 if testbed is not None:
 966                         testbed.reset_apt()
 967                         testbed.stop()
 968                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
 969         except:
 970                 print_exception(sys.exc_info(),
 971                         '\nadt-run: error cleaning up:\n')
 972                 os._exit(20)
 973
 974 #---------- registration, installation etc. of .deb's: Binaries
 975
 976 def determine_package(act):
 977         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
 978         running = Popen(cmd, stdout=PIPE)
 979         output = running.communicate()[0]
 980         rc = running.wait()
 981         if rc: badpkg('failed to parse binary package, code %d' % rc)
 982         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
 983         act.pkg = None
 984         for l in '\n'.split(output):
 985                 m = re.match(output)
 986                 if not m: continue
 987                 if act.pkg: badpkg('two Package: lines in control file')
 988                 act.pkg = m.groups
 989         if not act.pkg: badpkg('no good Package: line in control file')
 990
 991 class Binaries:
 992  def __init__(b):
 993         b.dir = TemporaryDir('binaries')
 994         b.dir.write()
 995         ok = False
 996
 997         if opts.gnupghome is None:
 998                 opts.gnupghome = tmpdir+'/gnupg'
 999
1000         b._debug('initialising')
1001         try:
1002                 for x in ['pubring','secring']:
1003                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1004                 ok = True
1005         except OSError, oe:
1006                 if oe.errno != errno.ENOENT: raise
1007
1008         if ok: b._debug('no key generation needed')
1009         else: b.genkey()
1010
1011  def _debug(b, s):
1012         debug('* '+s)
1013
1014  def genkey(b):
1015         b._debug('preparing for key generation')
1016
1017         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1018         mkdir_okexist(opts.gnupghome, 0700)
1019
1020         script = '''
1021   cd "$1"
1022   exec >key-gen-log 2>&1
1023   cat <<"END" >key-gen-params
1024 Key-Type: DSA
1025 Key-Length: 1024
1026 Key-Usage: sign
1027 Name-Real: autopkgtest per-run key
1028 Name-Comment: do not trust this key
1029 Name-Email: autopkgtest@example.com
1030 END
1031   set -x
1032   gpg --homedir="$1" --batch --gen-key key-gen-params
1033                 '''
1034         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1035         debug_subprocess('genkey', cmdl, script=script)
1036         rc = subprocess.call(cmdl)
1037         if rc:
1038                 try:
1039                         f = open(opts.gnupghome+'/key-gen-log')
1040                         tp = file.read()
1041                 except OSError, e: tp = e
1042                 print >>sys.stderr, tp
1043                 bomb('key generation failed, code %d' % rc)
1044
1045  def apt_configs(b):
1046         return {
1047                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1048         }
1049
1050  def apt_pkg_gdebi_script(b, arg, middle):
1051         script = [
1052                 'import apt_pkg',
1053                 'import urllib',
1054                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1055                 ]
1056         for (k,v) in b.apt_configs().iteritems():
1057                 v = urllib.quote(v)
1058                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1059                                 % (k, v))
1060         script += [
1061                 'from GDebi.Cache import Cache',
1062                 'cache = Cache()',
1063                 ]
1064         for m in middle:
1065                 script += m + [
1066                 'print res',
1067                 'print d.missingDeps',
1068                 'print d.requiredChanges',
1069                 'assert(res)',
1070                 'cache.commit()',
1071                 ''
1072                 ]
1073         return '\n'.join(script)
1074  def apt_get(b):
1075         ag = ['apt-get','-qy']
1076         for kv in b.apt_configs().iteritems():
1077                 ag += ['-o', '%s=%s' % kv]
1078         return ag
1079
1080  def reset(b):
1081         b._debug('reset')
1082         rmtree('binaries', b.dir.read())
1083         b.dir.invalidate()
1084         b.dir.write()
1085         b.install = []
1086         b.blamed = []
1087         b.registered = set()
1088
1089  def register(b, act, pkg, af, forwhat, blamed):
1090         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1091                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1092
1093         if act.ah['deb_'+forwhat] == 'ignore': return
1094
1095         b.blamed += testbed.blamed
1096
1097         leafname = pkg+'.deb'
1098         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1099
1100         try: os.remove(dest.write())
1101         except OSError, oe:
1102                 if oe.errno != errno.ENOENT: raise e
1103
1104         try: os.link(af.read(), dest.write())
1105         except OSError, oe:
1106                 if oe.errno != errno.EXDEV: raise e
1107                 shutil.copy(af.read(), dest)
1108
1109         if act.ah['deb_'+forwhat] == 'install':
1110                 b.install.append(pkg)
1111
1112         b.registered.add(pkg)
1113
1114  def publish(b):
1115         b._debug('publish')
1116
1117         script = '''
1118   cd "$1"
1119   apt-ftparchive packages . >Packages
1120   gzip <Packages >Packages.gz
1121   apt-ftparchive release . >Release
1122   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1123   gpg --homedir="$2" --batch --export >archive-key.pgp
1124                 '''
1125         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1126         debug_subprocess('ftparchive', cmdl, script)
1127         rc = subprocess.call(cmdl)
1128         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1129
1130         b.dir.invalidate(True)
1131         apt_source = b.dir.read(True)
1132
1133         so = TemporaryFile('vlds')
1134         script = '''
1135   apt-key add archive-key.pgp >&2
1136   echo "deb file://'''+apt_source+''' /" >sources.list
1137   cat /etc/apt/sources.list >>sources.list
1138   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1139     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1140   fi
1141   '''+ ' '.join(b.apt_get()) +''' update >&2
1142   cat /var/lib/dpkg/status
1143                 '''
1144         testbed.mungeing_apt()
1145         debug_subprocess('apt-key', script=script)
1146         (rc,se) = testbed.execute('apt-key',
1147                                 ['sh','-ec',script],
1148                                 so=so.write(True), cwd=b.dir.write(True))
1149         if rc: bomb('apt setup failed with exit code %d' % rc, se)
1150
1151         testbed.blamed += b.blamed
1152
1153         b._debug('publish reinstall checking...')
1154         pkgs_reinstall = set()
1155         pkg = None
1156         for l in file(so.read()):
1157                 if l.startswith('Package: '):
1158                         pkg = l[9:].rstrip()
1159                 elif l.startswith('Status: install '):
1160                         if pkg in b.registered:
1161                                 pkgs_reinstall.add(pkg)
1162                                 b._debug(' publish reinstall needs '+pkg)
1163
1164         if pkgs_reinstall:
1165                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1166                 what = 'apt-get-reinstall'
1167                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1168                         [pkg for pkg in pkgs_reinstall])
1169                 debug_subprocess(what, cmdl)
1170                 (rc,se) = testbed.execute(what, cmdl)
1171                 if rc: badpkg("installation of basic binarries failed,"
1172                                 " exit code %d" % rc, se)
1173
1174         b._debug('publish install...')
1175         for pkg in b.install:
1176                 what = 'apt-get-install-%s' % pkg
1177                 testbed.blame(pkg)
1178                 cmdl = b.apt_get() + ['install',pkg]
1179                 debug_subprocess(what, cmdl)
1180                 (rc,se) = testbed.execute(what, cmdl)
1181                 if rc: badpkg("installation of %s failed, exit code %d"
1182                                 % (pkg, rc), se)
1183
1184         b._debug('publish done')
1185
1186 #---------- processing of sources (building)
1187
1188 def source_rules_command(act,script,what,which,work,results_lines=0):
1189         if opts.debug:
1190                 trace = "%s-%s-trace" % (what,which)
1191                 script = [      "mkfifo -m600 "+trace,
1192                                 "tee <"+trace+" /dev/stderr >&4 &",
1193                                 "exec >"+trace+" 2>&1"  ] + script
1194
1195         script = [      "exec 3>&1 >&2",
1196                         "set -x"        ] + script
1197         script = '\n'.join(script)
1198         so = TemporaryFile('%s-%s-results' % (what,which))
1199         se = TemporaryFile('%s-%s-log' % (what,which))
1200         debug_subprocess('source-rules-command/%s/%s' % (act.what, which),
1201                         script=script)
1202         rc = testbed.execute('%s-%s' % (what,which),
1203                         ['sh','-xec',script],
1204                         so=so.write(True), se=se.write(True),
1205                         cwd= work.write(True), dump_fd=4)
1206         results = file(so.read()).read().rstrip('\n').split("\n")
1207         se = file(se.read()).read()
1208         if rc: badpkg("%s failed with exit code %d" % (which,rc), se)
1209         if results_lines is not None and len(results) != results_lines:
1210                 badpkg("got %d lines of results from %s where %d expected"
1211                         % (len(results), which, results_lines), se)
1212         if results_lines==1: return results[0]
1213         return results
1214
1215 def build_source(act):
1216         act.blame = 'arg:'+act.af.spec
1217         testbed.blame(act.blame)
1218         testbed.needs_reset()
1219
1220         what = act.what
1221         dsc = act.af
1222         basename = dsc.spec
1223
1224         dsc_file = open(dsc.read())
1225         in_files = False
1226         fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1227         for l in dsc_file:
1228                 l = l.rstrip('\n')
1229                 if l.startswith('Files:'): in_files = True; continue
1230                 elif l.startswith('#'): pass
1231                 elif not l.startswith(' '):
1232                         in_files = False
1233                         if l.startswith('Source:'):
1234                                 act.blame = 'dsc:'+l[7:].strip()
1235                                 testbed.blame(act.blame)
1236                 if not in_files: continue
1237
1238                 m = fre.match(l)
1239                 if not m: badpkg(".dsc contains unparseable line"
1240                                 " in Files: `%s'" % l)
1241                 leaf = m.groups(0)[0]
1242                 subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1243                                 sibling=True)
1244                 subfile.read(True)
1245         dsc.read(True)
1246
1247         script = binaries.apt_pkg_gdebi_script(
1248                 dsc.read(True), [[
1249                 'from GDebi.DscSrcPackage import DscSrcPackage',
1250                 'd = DscSrcPackage(cache, arg)',
1251                 'res = d.checkDeb()',
1252                  ],[
1253                 'from GDebi.DebPackage import DebPackage',
1254                 'd = DebPackage(cache)',
1255                 'res = d.satisfyDependsStr("build-essential")',
1256                 ]])
1257         cmdl = ['python','-c',script]
1258         whatp = what+'-builddeps'
1259         debug_subprocess(whatp, cmdl, script=script)
1260         (rc,se) = testbed.execute(what, cmdl)
1261         if rc: badpkg('build-depends install failed, exit code %d' % rc, se)
1262
1263         work = TemporaryDir(what+'-build')
1264
1265         script = [
1266                         'cd '+work.write(True),
1267                 ]
1268         if opts.user: script += [
1269                         'chown '+opts.user+' .',
1270                         'dsc=dsc.read(True) '+
1271                                 opts.user_wrap('dpkg-source -x $dsc')
1272                 ]
1273         else: script += [
1274                         'dpkg-source -x '+dsc.read(True),
1275                 ]
1276         script += [
1277                         'cd */.',
1278                         'dpkg-checkbuilddeps',
1279                         'pwd >&3',
1280                         opts.user_wrap('debian/rules build'),
1281                 ]
1282         result_pwd = source_rules_command(act,script,what,'build',work,1)
1283
1284         if os.path.dirname(result_pwd)+'/' != work.read(True):
1285                 badpkg("results dir `%s' is not in expected parent dir `%s'"
1286                         % (result_pwd, work.read(True)))
1287
1288         act.work = work
1289         act.tests_tree = InputDir(what+'-tests-tree',
1290                                 work.read(True)+os.path.basename(result_pwd),
1291                                 True)
1292         if act.ah['dsc_tests']:
1293                 act.tests_tree.read()
1294                 testbed.register_ephemeral(act.work)
1295                 testbed.register_ephemeral(act.tests_tree)
1296
1297         act.blamed = copy.copy(testbed.blamed)
1298
1299         def debug_b(m): debug('* <dsc:%s> %s' % (act.what, m))
1300         act.binaries = []
1301         filter = act.ah['dsc_filter']
1302         debug_b('filter=%s' % filter)
1303         if filter != '_':
1304                 script = [
1305                         'cd '+work.write(True)+'/*/.',
1306                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1307                         'cd ..',
1308                         'echo *.deb >&3',
1309                         ]
1310                 result_debs = source_rules_command(act,script,what,
1311                                 'binary',work,1)
1312                 if result_debs == '*': debs = []
1313                 else: debs = result_debs.split(' ')
1314                 debug_b('debs='+`debs`)
1315                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1316                 for deb in debs:
1317                         m = re.match(deb)
1318                         if not m: badpkg("badly-named binary `%s'" % deb)
1319                         pkg = m.groups()[0]
1320                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1321                         for pat in filter.split(','):
1322                                 debug_b('  pat=%s' % pat)
1323                                 if not fnmatch.fnmatchcase(pkg,pat):
1324                                         debug_b('   no match')
1325                                         continue
1326                                 deb_what = pkg+'_'+what+'.deb'
1327                                 bin = RelativeInputFile(deb_what,work,deb,True)
1328                                 debug_b('  deb_what=%s, bin=%s' %
1329                                         (deb_what, str(bin)))
1330                                 binaries.register(act,pkg,bin,
1331                                         'forbuilds',testbed.blamed)
1332                                 act.binaries.append((pkg,bin))
1333                                 break
1334                 debug_b('all done.')
1335
1336 #---------- main processing loop and main program
1337
1338 def process_actions():
1339         global binaries
1340
1341         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1342         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1343         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1344
1345         debug_a1('starting')
1346         testbed.open()
1347         binaries = Binaries()
1348
1349         binaries.reset()
1350
1351         debug_a1('builds ...')
1352         for act in opts.actions:
1353                 debug_a2('%s %s' %
1354                         (act.kind, act.what))
1355
1356                 testbed.prepare([])
1357                 if act.kind == 'deb':
1358                         blame('arg:'+act.af.spec)
1359                         determine_package(act)
1360                         blame('deb:'+act.pkg)
1361                         binaries.register(act,act.pkg,act.af,
1362                                 'forbuilds',testbed.blamed)
1363                 if act.kind == 'dsc':
1364                         build_source(act)
1365                 if act.kind == 'tree':
1366                         act.binaries = []
1367
1368         debug_a1('builds done.')
1369
1370         binaries.reset()
1371         control_override = None
1372
1373         debug_a1('tests ...')
1374         for act in opts.actions:
1375                 debug_a2('test %s %s' % (act.kind, act.what))
1376
1377                 testbed.needs_reset()
1378                 if act.kind == 'control':
1379                         control_override = act.af
1380                 if act.kind == 'deb':
1381                         binaries.register(act,act.pkg,act.af,'fortests',
1382                                 ['deb:'+act.pkg])
1383                 if act.kind == 'dsc':
1384                         for (pkg,bin) in act.binaries:
1385                                 binaries.register(act,pkg,bin,'fortests',
1386                                         act.blamed)
1387                         if act.ah['dsc_tests']:
1388                                 debug_a3('read control ...')
1389                                 stanzas = read_control(act, act.tests_tree,
1390                                                 control_override)
1391                                 testbed.blamed += act.blamed
1392                                 debug_a3('run_tests ...')
1393                                 run_tests(stanzas, act.tests_tree)
1394                         control_override = None
1395                 if act.kind == 'tree':
1396                         testbed.blame('arg:'+act.af.spec)
1397                         stanzas = read_control(act, act.af, control_override)
1398                         debug_a3('run_tests ...')
1399                         run_tests(stanzas, act.af)
1400                         control_override = None
1401         debug_a1('tests done.')
1402
1403 def main():
1404         global testbed
1405         global tmpdir
1406         try:
1407                 parse_args()
1408         except SystemExit, se:
1409                 os._exit(20)
1410         try:
1411                 testbed = Testbed()
1412                 testbed.start()
1413                 finalise_options()
1414                 process_actions()
1415         except:
1416                 ec = print_exception(sys.exc_info(), '')
1417                 cleanup()
1418                 os._exit(ec)
1419         cleanup()
1420         os._exit(errorcode)
1421
1422 main()