runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37
  38 from optparse import OptionParser
  39
  40 tmpdir = None
  41 testbed = None
  42 errorcode = 0
  43
  44 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  45
  46 class Quit:
  47         def __init__(q,ec,m): q.ec = ec; q.m = m
  48
  49 def bomb(m): raise Quit(20, "unexpected error: %s" % m)
  50 def badpkg(m): raise Quit(12, "erroneous package: %s" % m)
  51 def report(tname, result): print '%-20s %s' % (tname, result)
  52
  53 class Unsupported:
  54  def __init__(u, lno, m):
  55         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
  56         else: u.m = m
  57  def report(u, tname):
  58         global errorcode
  59         errorcode != 2
  60         report(tname, 'SKIP %s' % u.m)
  61
  62 def debug(m):
  63         global opts
  64         if not opts.debug: return
  65         print >>sys.stderr, 'atd-run: debug:', m
  66
  67 def flatten(l):
  68         return reduce((lambda a,b: a + b), l, [])
  69
  70 class Path:
  71         # p.path[tb]    None or path    not None => path known
  72         # p.file[tb]    None or path    not None => file exists
  73         # p.what
  74         # p.spec_tb
  75         # p.tb_scratch
  76
  77  def ensure_path(p, tb=False):
  78         if tb and not p.spec_tb:
  79                 if not testbed.scratch:
  80                         error "called ensure_path for `%s' when testbed closed"
  81                                 % what
  82                 if not p.tb_scratch or p.tb_scratch is not testbed.scratch:
  83
  84
  85         if p.path[tb] is not None: return
  86         if tb: p.path[tb] = p.tb_tmpdir
  87         else: p.path[tb] = tmpdir
  88         p.path[tb] += '/'+p.what
  89
  90  def ensure_file(p, tb=False):
  91         if p.file[tb] is not None: return
  92         p.ensure_path(tb)
  93         testbed.open()
  94
  95
  96  def write(p, tb=False):
  97         p.ensure_path(tb)
  98         return p.path[tb]
  99  def read(p, tb=False):
 100         p.ensure_file(tb)
 101
 102 class InputPath:
 103 class OutputPath:
 104 class OutputPath:
 105
 106  def __init__(p, path, spec_tb, what, dir=False):
 107         if p.tb:
 108                 if p.p[:1] != '/':
 109                         bomb("path %s specified as being in testbed but"
 110                                 " not absolute: `%s'" % (what, p.p))
 111         p.path[spec_tb] = p.file[spec_tb] = path
 112         p.what = what
 113
 114
 115         if spec_tb:
 116                 p.
 117 tb_path = path
 118                 p.tb_onpath = path
 119                 p.tb_onhost = None
 120
 121
 122 4 def __init__(p, tb, path, what, dir=False, tbscratch=None, xfmap=None
 123                 lpath=None):
 124         p.tb = tb
 125         p.p = path
 126         p.what = what
 127         p.dir = dir
 128         p.tbscratch = tbscratch
 129         p.lpath = None
 130         if p.tb:
 131                 if p.p[:1] != '/':
 132                         bomb("path %s specified as being in testbed but"
 133                                 " not absolute: `%s'" % (what, p.p))
 134                 p.local = None
 135                 p.down = p.p
 136         else:
 137                 p.local = p.p
 138                 p.down = None
 139         if p.dir: p.dirsfx = '/'
 140         else: p.dirsfx = ''
 141  def path(p):
 142         return p.p + p.dirsfx
 143  def append(p, suffix, what, dir=False):
 144         return Path(p.tb, p.path() + suffix, what=what, dir=dir,
 145                         tbscratch=p.tbscratch)
 146  def __str__(p):
 147         if p.tb: pfx = '/VIRT'
 148         elif p.p[:1] == '/': pfx = '/HOST'
 149         else: pfx = './'
 150         return pfx + p.p
 151
 152  def xfmapcopy(p, cud, dstdir):
 153         if p.xfmap is None: return
 154         srcdir = os.path.dirname(p.path()+'/')
 155         dstdir = p.xfmapdstdir+'/'
 156         for f in p.xfmap(file(p.local)):
 157                 if '/' in f: bomb("control file %s mentions other filename"
 158                                 "containing slash" % p.what)
 159                 testbed.command(cud, (srcdir+f, dstdir+f))
 160
 161  def onhost(p, lpath = None):
 162         if lpath is not None:
 163                 if p.lpath is not None: assert(p.lpath == lpath)
 164                 p.lpath = lpath
 165         if p.local is not None:
 166                 if p.lpath is not None: assert(p.local == p.lpath)
 167                 return p.local
 168         testbed.open()
 169
 170         if p.xfmap is None:
 171                 p.local = p.lpath
 172                 if p.local is None: p.local = tmpdir + '/tb-' + p.what
 173         else:
 174                 assert(p.lpath is None)
 175                 assert(not p.dir)
 176                 p.xfmapdstdir = tmpdir + '/tbd-' + p.what
 177                 os.mkdir(p.xfmapdstdir)
 178                 p.local = p.xfmapdstdir + '/' + os.path.basename(p.down)
 179
 180         testbed.command('copyup', (p.path(), p.local + p.dirsfx))
 181         p.xfmapcopy('copyup')
 182
 183         return p.local
 184
 185  def maybe_onhost(p):
 186         if p.lpath is None: return None
 187         return p.onhost()
 188
 189  def ontb(p):
 190         testbed.open()
 191
 192         if p.tbscratch is not None:
 193                 if p.tbscratch != testbed.scratch:
 194                         p.down = None
 195         if p.down is not None: return p.down
 196         if p.tb:
 197                 bomb("testbed scratch path " + str(p) + " survived testbed")
 198
 199         if p.xfmap is None:
 200                 p.down = testbed.scratch.p + '/host-' + p.what
 201         else:
 202                 assert(not p.dir)
 203                 p.xfmapdstdir = testbed.scratch.p + '/hostd-' + p.what
 204                 testbed.command('mkdir '+p.xfmapdstdir)
 205                 p.down = p.xfmapdstdir + '/' + os.path.basename(p.local)
 206
 207         p.tbscratch = testbed.scratch
 208         testbed.command('copydown', (p.path(), p.down + p.dirsfx))
 209         p.xfmapcopy('copydown')
 210         return p.down
 211
 212 def Action:
 213  def __init__(a, kind, path, arghandling, ix):
 214         a.kind = kind
 215         a.path = path
 216         a.ah = arghandling
 217         a.what = '%s%s' % (kind,ix); ix++
 218
 219 def parse_args():
 220         global opts
 221         usage = "%prog <options> -- <virt-server>..."
 222         parser = OptionParser(usage=usage)
 223         pa = parser.add_option
 224         pe = parser.add_option
 225
 226         arghandling = {
 227                 'dsc_tests': True,
 228                 'dsc_filter': '*',
 229                 'deb_forbuilds': 'auto',
 230                 'deb_fortests': 'auto',
 231                 'tb': False,
 232                 'override_control': None
 233         }
 234         initial_arghandling = arghandling.copy()
 235         n_actions = 0
 236
 237         #----------
 238         # actions (ie, test sets to run, sources to build, binaries to use):
 239
 240         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 241                 parser.largs.append((value,kindpath))
 242                 n_actions += is_act
 243
 244         def pa_action(long, metavar, kindpath, help, is_act=True):
 245                 pa('','--'+long, action='callback', callback=cb_action,
 246                         nargs=1, type='string',
 247                         callback_args=(long,kindpath,is_act), help=help)
 248
 249         pa_action('build-tree',         'TREE', '@/',
 250                 help='run tests from build tree TREE')
 251
 252         pa_action('source',             'DSC', '@.dsc',
 253                 help='build DSC and use its tests and/or'
 254                     ' generated binary packages')
 255
 256         pa_action('binary',             'DEB', '@.deb',
 257                help='use binary package DEB according'
 258                     ' to most recent --binaries-* settings')
 259
 260         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 261                help='run tests from control file CONTROL instead,
 262                     ' (applies to next test suite only)')
 263
 264         #----------
 265         # argument handling settings (what ways to use action
 266         #  arguments, and pathname processing):
 267
 268         def cb_setah(option, opt_str, value, parser, toset,setval):
 269                 if type(setval) == list:
 270                         if not value in setval:
 271                                 parser.error('value for %s option (%s) is not '
 272                                  'one of the permitted values (%s)' %
 273                                  (value, opt_str, setval.join(' ')))
 274                 elif setval is not None:
 275                         value = setval
 276                 for v in toset:
 277                         arghandling[v] = value
 278                 parser.largs.append(arghandling.copy())
 279
 280         def pa_setah(long, affected,effect, **kwargs):
 281                 type = metavar; if type: type = 'string'
 282                 pa('',long, action='callback', callback=cb_setah,
 283                    callback_args=(affected,effect), **kwargs)
 284                      ' according to most recent --binaries-* settings')
 285
 286         #---- paths: host or testbed:
 287         #
 288         pa_setah('--paths-testbed', ['tb'],True,
 289                 help='subsequent path specifications refer to the testbed')
 290         pa_setah('--paths-host', ['tb'],False,
 291                 help='subsequent path specifications refer to the host')
 292
 293         #---- source processing settings:
 294
 295         pa_setah('--sources-tests', ['dsc_tests'],True,
 296                 help='run tests from builds of subsequent sources')
 297         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 298                 help='do not run tests from builds of subsequent sources')
 299
 300         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 301                 type=string, metavar='PATTERN-LIST',
 302                 help='from subsequent sources, use binaries matching'
 303                      ' PATTERN-LIST (comma-separated glob patterns)'
 304                      ' according to most recent --binaries-* settings')
 305         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 306                 help='from subsequent sources, do not use any binaries')
 307
 308         #---- binary package processing settings:
 309
 310         def pa_setahbins(long,toset,how):
 311          pa_setah(long, toset,['ignore','auto','install'],
 312                 type=string, metavar='IGNORE|AUTO|INSTALL', default='auto',
 313                 help=how+' ignore binaries, install them as needed'
 314                         ' for dependencies, or unconditionally install'
 315                         ' them, respectively')
 316         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 317         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 318         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 319
 320         #----------
 321         # general options:
 322
 323         def cb_vserv(op,optstr,value,parser):
 324                 parser.values.vserver = list(parser.rargs)
 325                 del parser.rargs[:]
 326
 327         def cb_path(op,optstr,value,parser, long,dir,xfmap):
 328                 name = long.replace('-','_')
 329                 path = Path(arghandling['tb'], value, long, dir, xfmap=xfmap)
 330                 setattr(parser.values, name, path)
 331
 332         def pa_path(long, help, dir=False, xfmap=None):
 333                 pa('','--'+long, action='callback', callback=cb_path,
 334                         nargs=1, type='string', callback_args=(long,dir,xfmap),
 335                         help=, metavar='PATH')
 336
 337         pa_path('output-dir', 'write stderr/out files in PATH', dir=True)
 338
 339         pa('','--user',                 type='string', dest='user',
 340                 help='run tests as USER (needs root on testbed)')
 341         pa('','--fakeroot',             type='string', dest='fakeroot',
 342                 help='prefix debian/rules build with FAKEROOT')
 343         pa('-d', '--debug', action='store_true', dest='debug');
 344
 345         #----------
 346         # actual meat:
 347
 348         class SpecialOption(optparse.Option): pass
 349         vs_op = SpecialOption('','--VSERVER-DUMMY')
 350         vs_op.action = 'callback'
 351         vs_op.type = None
 352         vs_op.default = None
 353         vs_op.nargs = 0
 354         vs_op.callback = cb_vserv
 355         vs_op.callback_args = ( )
 356         vs_op.callback_kwargs = { }
 357         vs_op.help = 'introduces virtualisation server and args'
 358         vs_op._short_opts = []
 359         vs_op._long_opts = ['---']
 360
 361         pa(vs_op)
 362
 363         (opts,args) = parser.parse_args()
 364         if not hasattr(opts,'vserver'):
 365                 parser.error('you must specifiy --- <virt-server>...')
 366         if not n_actions:
 367                 parser.error('nothing to do specified')
 368
 369         arghandling = initial_arghandling
 370         opts.actions = []
 371         ix = 0
 372         for act in args:
 373                 if type(act) == dict:
 374                         arghandling = act
 375                         continue
 376                 elif type(act) == tuple:
 377                         pass
 378                 elif type(act) == string:
 379                         act = (act,act)
 380                 else:
 381                         error "unknown action in list `%s' having"
 382                               "type `%s' % (act, type(act))
 383                 (path, kindpath) = act
 384
 385                 if type(kindpath) is tuple:             kind = kindpath[0]
 386                 elif kindpath.endswith('/'):            kind = 'tree'
 387                 elif kindpath.endswith('.deb'):         kind = 'deb'
 388                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 389                 else: parser.error("do not know how to handle filename \`%s';"
 390                         " specify --source --binary or --build-tree")
 391
 392                 opts.actions.append(Action(kind, path, arghandling, ix))
 393                 ix++
 394
 395 def finalise_options():
 396         global opts, testbed
 397
 398         if opts.user is None and 'root-on-testbed' not in caps:
 399                 opts.user = ''
 400
 401         if opts.user is None:
 402                 su = 'suggested-normal-user='
 403                 ul = [
 404                         e[length(su):]
 405                         for e in caps
 406                         if e.startswith(su)
 407                         ]
 408                 if len(ul) > 1:
 409                         print >>sys.stderr, "warning: virtualisation"
 410                                 " system offers several suggested-normal-user"
 411                                 " values: "+('/'.join(ul))+", using "+ul[0]
 412                 if ul:
 413                         opts.user = ul[0]
 414                 else:
 415                         opts.user = ''
 416
 417         if opts.user:
 418                 if 'root-on-testbed' not in caps:
 419                         print >>sys.stderr, "warning: virtualisation"
 420                                 " system does not offer root on testbed,"
 421                                 " but --user option specified: failure likely"
 422                 opts.user_wrap = lambda x: 'su %s -c "%s"' % (opts.user, x)
 423         else:
 424                 opts.user_wrap = lambda x: x
 425
 426         if opts.fakeroot is None:
 427                 opts.fakeroot = ''
 428                 if opts.user or
 429                    'root-on-testbed' not in testbed.caps:
 430                         opts.fakeroot = 'fakeroot'
 431
 432 logpath_counters = {}
 433
 434 def logpath(idstr):
 435         # if idstr ends with `-' then a counter is appended
 436         if idstr.endswith('-'):
 437                 if not logpath_counters.has_key(idstr):
 438                         logpath_counters[idstr] = 1
 439                 else:
 440                         logpath_counters[idstr] += 1
 441                 idstr.append(`logpath_counters[idstr]`)
 442         idstr = 'log-' + idstr
 443         if opts.output_dir is None:
 444                 return testbed.scratch.append(idstr, idstr)
 445         elif opts.output_dir.tb:
 446                 return opts.output_dir.append(idstr, idstr)
 447         else:
 448                 return Path(True, testbed.scratch.p, idstr,
 449                         lpath=opts.output_dir.p+'/'+idstr)
 450
 451 class Testbed:
 452  def __init__(tb):
 453         tb.sp = None
 454         tb.lastsend = None
 455         tb.scratch = None
 456  def start(tb):
 457         p = subprocess.PIPE
 458         tb.sp = subprocess.Popen(opts.vserver,
 459                 stdin=p, stdout=p, stderr=None)
 460         tb.expect('ok')
 461         tb.caps = tb.command('capabilities')
 462  def stop(tb):
 463         tb.close()
 464         if tb.sp is None: return
 465         ec = tb.sp.returncode
 466         if ec is None:
 467                 tb.sp.stdout.close()
 468                 tb.send('quit')
 469                 tb.sp.stdin.close()
 470                 ec = tb.sp.wait()
 471         if ec:
 472                 tb.bomb('testbed gave exit status %d after quit' % ec)
 473  def open(tb):
 474         if tb.scratch is not None: return
 475         p = tb.commandr1('open')
 476         tb.scratch = Path(True, p, 'tb-scratch', dir=True)
 477         tb.scratch.tbscratch = tb.scratch
 478  def close(tb):
 479         if tb.scratch is None: return
 480         tb.scratch = None
 481         if tb.sp is None: return
 482         tb.command('close')
 483  def bomb(tb, m):
 484         if tb.sp is not None:
 485                 tb.sp.stdout.close()
 486                 tb.sp.stdin.close()
 487                 ec = tb.sp.wait()
 488                 if ec: print >>sys.stderr, ('adt-run: testbed failing,'
 489                         ' exit status %d' % ec)
 490         tb.sp = None
 491         raise Quit(16, 'testbed failed: %s' % m)
 492  def send(tb, string):
 493         tb.sp.stdin
 494         try:
 495                 debug('>> '+string)
 496                 print >>tb.sp.stdin, string
 497                 tb.sp.stdin.flush()
 498                 tb.lastsend = string
 499         except:
 500                 (type, value, dummy) = sys.exc_info()
 501                 tb.bomb('cannot send to testbed: %s' % traceback.
 502                         format_exception_only(type, value))
 503  def expect(tb, keyword, nresults=-1):
 504         l = tb.sp.stdout.readline()
 505         if not l: tb.bomb('unexpected eof from the testbed')
 506         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 507         l = l.rstrip('\n')
 508         debug('<< '+l)
 509         ll = l.split()
 510         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 511         if ll[0] != keyword:
 512                 if tb.lastsend is None:
 513                         tb.bomb("got banner `%s', expected `%s...'" %
 514                                 (l, keyword))
 515                 else:
 516                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 517                                 (tb.lastsend, l, keyword))
 518         ll = ll[1:]
 519         if nresults >= 0 and len(ll) != nresults:
 520                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 521                         " expected %d result parameters" %
 522                         (string, l, len(ll), nresults))
 523         return ll
 524  def commandr(tb, cmd, nresults, args=()):
 525         if type(cmd) is str: cmd = [cmd]
 526         al = cmd + map(urllib.quote, args)
 527         tb.send(string.join(al))
 528         ll = tb.expect('ok')
 529         rl = map(urllib.unquote, ll)
 530         return rl
 531  def command(tb, cmd, args=()):
 532         tb.commandr(cmd, 0, args)
 533  def commandr1(tb, cmd, args=()):
 534         rl = tb.commandr(cmd, 1, args)
 535         return rl[0]
 536
 537 class FieldBase:
 538  def __init__(f, fname, stz, base, tnames, vl):
 539         assert(vl)
 540         f.stz = stz
 541         f.base = base
 542         f.tnames = tnames
 543         f.vl = vl
 544  def words(f):
 545         def distribute(vle):
 546                 (lno, v) = vle
 547                 r = v.split()
 548                 r = map((lambda w: (lno, w)), r)
 549                 return r
 550         return flatten(map(distribute, f.vl))
 551  def atmostone(f):
 552         if len(vl) == 1:
 553                 (f.lno, f.v) = vl[0]
 554         else:
 555                 raise Unsupported(f.vl[1][0],
 556                         'only one %s field allowed' % fn)
 557         return f.v
 558
 559
 560 def acquire_built_source():
 561         global opts
 562
 563         if opts.build_source:
 564                 assert(opts.tests_tree is None)
 565                 bss = build_some_source('t', opts.build_source)
 566                 opts.tests_tree = bss[0]
 567
 568 class FieldIgnore(FieldBase):
 569  def parse(f): pass
 570
 571 class Restriction:
 572  def __init__(r,rname,base): pass
 573
 574 class Restriction_rw_tests_tree(Restriction): pass
 575
 576 class Field_Restrictions(FieldBase):
 577  def parse(f):
 578         for wle in f.words():
 579                 (lno, rname) = wle
 580                 rname = rname.replace('-','_')
 581                 try: rclass = globals()['Restriction_'+rname]
 582                 except KeyError: raise Unsupported(lno,
 583                         'unknown restriction %s' % rname)
 584                 r = rclass(rname, f.base)
 585                 f.base['restrictions'].append(r)
 586
 587 class Field_Tests(FieldIgnore): pass
 588
 589 class Field_Tests_directory(FieldBase):
 590  def parse(f):
 591         td = atmostone(f)
 592         if td.startswith('/'): raise Unspported(f.lno,
 593                 'Tests-Directory may not be absolute')
 594         base['testsdir'] = td
 595
 596 def run_tests():
 597         for t in tests:
 598                 t.run()
 599         if not tests:
 600                 global errorcode
 601                 report('*', 'SKIP no tests in this package')
 602                 errorcode |= 8
 603
 604 class Test:
 605  def __init__(t, tname, base):
 606         if '/' in tname: raise Unsupported(base[' lno'],
 607                 'test name may not contain / character')
 608         for k in base: setattr(t,k,base[k])
 609         t.tname = tname
 610         if len(base['testsdir']): tpath = base['testsdir'] + '/' + tname
 611         else: tpath = tname
 612         t.p = opts.tests_tree.append(tpath, 'test-'+tname)
 613  def report(t, m):
 614         report(t.tname, m)
 615  def reportfail(t, m):
 616         global errorcode
 617         errorcode |= 4
 618         report(t.tname, 'FAIL ' + m)
 619  def run(t):
 620         testbed.open()
 621         def stdouterr(oe):
 622                 idstr = oe + '-' + t.tname
 623                 if opts.output_dir is not None and opts.output_dir.tb:
 624                         return opts.output_dir.append(idstr)
 625                 else:
 626                         return testbed.scratch.append(idstr, idstr)
 627         def stdouterrh(p, oe):
 628                 idstr = oe + '-' + t.tname
 629                 if opts.output_dir is None or opts.output_dir.tb:
 630                         return p.onhost()
 631                 else:
 632                         return p.onhost(opts.output_dir.onhost() + '/' + idstr)
 633         so = stdouterr('stdout')
 634         se = stdouterr('stderr')
 635         rc = testbed.commandr1('execute',(t.p.ontb(),
 636                 '/dev/null', so.ontb(), se.ontb(), opts.tests_tree.ontb()))
 637         soh = stdouterrh(so, 'stdout')
 638         seh = stdouterrh(se, 'stderr')
 639         rc = int(rc)
 640         stab = os.stat(seh)
 641         if stab.st_size != 0:
 642                 l = file(seh).readline()
 643                 l = l.rstrip('\n \t\r')
 644                 if len(l) > 40: l = l[:40] + '...'
 645                 t.reportfail('stderr: %s' % l)
 646         elif rc != 0:
 647                 t.reportfail('non-zero exit status %d' % rc)
 648         else:
 649                 t.report('PASS')
 650
 651 def read_control():
 652         global tests
 653         try:
 654                 control = file(opts.control.onhost(), 'r')
 655         except IOError, oe:
 656                 if oe[0] != errno.ENOENT: raise
 657                 tests = []
 658                 return
 659         lno = 0
 660         def badctrl(m): testbed.badpkg('tests/control line %d: %s' % (lno, m))
 661         stz = None # stz[field_name][index] = (lno, value)
 662
 663         stanzas = [ ]
 664         stz = None
 665
 666         def end_stanza(stz):
 667                 if stz is None: return
 668                 stz[' errs'] = 0
 669                 stanzas.append(stz)
 670                 stz = None
 671                 hcurrent = None
 672
 673         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
 674         while 1:
 675                 l = control.readline()
 676                 if not l: break
 677                 lno += 1
 678                 if not l.endswith('\n'): badctrl('unterminated line')
 679                 if regexp.compile('\s*\#').match(l): continue
 680                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
 681                 initmat = initre.match(l)
 682                 if initmat:
 683                         (fname, l) = initmat.groups()
 684                         fname = string.capwords(fname)
 685                         if stz is None:
 686                                 stz = { ' lno': lno }
 687                         if not stz.has_key(fname): stz[fname] = [ ]
 688                         hcurrent = stz[fname]
 689                 elif regexp.compile('\s').match(l):
 690                         if not hcurrent: badctrl('unexpected continuation')
 691                 else:
 692                         badctrl('syntax error')
 693                 hcurrent.append((lno, l))
 694         end_stanza(stz)
 695
 696         def testbadctrl(stz, lno, m):
 697                 report_badctrl(lno, m)
 698                 stz[' errs'] += 1
 699
 700         for stz in stanzas:
 701                 try:
 702                         try: tnames = stz['Tests']
 703                         except KeyError:
 704                                 tnames = ['*']
 705                                 raise Unsupported(stz[' lno'],
 706                                         'no Tests field')
 707                         tnames = map((lambda lt: lt[1]), tnames)
 708                         tnames = string.join(tnames).split()
 709                         base = {
 710                                 'restrictions': [],
 711                                 'testsdir': 'debian/tests'
 712                         }
 713                         for fname in stz.keys():
 714                                 if fname.startswith(' '): continue
 715                                 vl = stz[fname]
 716                                 try: fclass = globals()['Field_'+
 717                                         fname.replace('-','_')]
 718                                 except KeyError: raise Unsupported(vl[0][0],
 719                                         'unknown metadata field %s' % fname)
 720                                 f = fclass(stz, fname, base, tnames, vl)
 721                                 f.parse()
 722                         tests = []
 723                         for tname in tnames:
 724                                 t = Test(tname, base)
 725                                 tests.append(t)
 726                 except Unsupported, u:
 727                         for tname in tnames: u.report(tname)
 728                         continue
 729
 730 def print_exception(ei, msgprefix=''):
 731         if msgprefix: print >>sys.stderr, msgprefix
 732         (et, q, tb) = ei
 733         if et is Quit:
 734                 print >>sys.stderr, 'adt-run:', q.m
 735                 return q.ec
 736         else:
 737                 print >>sys.stderr, "adt-run: unexpected, exceptional, error:"
 738                 traceback.print_exc()
 739                 return 20
 740
 741 def cleanup():
 742         try:
 743                 rm_ec = 0
 744                 if tmpdir is not None:
 745                         rm_ec = subprocess.call(['rm','-rf','--',tmpdir])
 746                 if testbed is not None:
 747                         testbed.stop()
 748                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
 749         except:
 750                 print_exception(sys.exc_info(),
 751                         '\nadt-run: error cleaning up:\n')
 752                 os._exit(20)
 753
 754 def source_rules_command(act,script,which,work,results_lines=0):
 755         script = "exec 3>&1 >&2\n" + '\n'.join(script)
 756         so = TemporaryPath('%s-%s-results' % (what,which))
 757         se = TemporaryPath('%s-%s-log' & (what,which))
 758         rc = testbed.commandr1(['execute',
 759                 ','.join(map(urllib.quote, ['sh','-xec',script]))],
 760                 '/dev/null', so.write(True), se.write(True), work.write(True))
 761         results = file(so.read()).read().split("\n")
 762         if rc:
 763                 act.bomb("%s failed with exit code %d" % (which,rc), se)
 764         if results_lines is not None and len(results) != results_lines:
 765                 act.bomb("got %d lines of results from %s where %d expected"
 766                         % (len(results), which, results_lines), se)
 767         if results_lines==1: return results[0]
 768         return results
 769
 770 def build_source(act,ah):
 771         prepare_testbed_for_action()
 772
 773         what = act.ah['what']
 774         dsc_what = what+'/'+os.path.basename(act.path)
 775         dsc = InputPath(dsc_what, act.path, arghandling['tb'])
 776
 777         if not dsc.spec_tb:
 778                 dsc_file = open(dsc.read())
 779                 in_files = False
 780                 re = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
 781                 for l in dsc_file():
 782                         if l.startswith('Files:'): in_files = True
 783                         elif l.startswith('#'): pass
 784                         elif not l.startswith(' '): in_files = False
 785                         elif not in_files: pass
 786
 787                         m = re.match(l)
 788                         if not m: act.bomb(".dsc contains unparseable line"
 789                                         " in Files: `%s'" % (`dsc`,l))
 790
 791                         subfile = dsc.enclosingdir().append('/'+m.groups(0))
 792                         subfile.ensure_file(True)
 793         dsc.ensure_file(True)
 794
 795         work = AccumulationPath(what+'/build', dir=True)
 796
 797         script = [
 798                         'cd '+work.write(True),
 799                         'gdebi '+dsc.read(True),
 800                         'dpkg-source -x '+dsc.read(True),
 801                         'cd */.',
 802                         'pwd >&3',
 803                         opts.user_wrap('debian/rules build'),
 804         ]
 805         result_pwd = source_rules_command(act,script,what,'build',work,1)
 806
 807         if os.path.dirname(result_pwd) != work.read(True):
 808                 act.bomb("results dir `%s' is not in expected parent dir `%s'"
 809                         % (results[0], work.read(True)), se)
 810
 811         ah.tests_tree = work.append('/'+os.path.basename(results[0]))
 812         if ah['dsc_tests']:
 813                 ah.tests_tree.preserve_now()
 814
 815         ah.binaries = []
 816         if ah['dsc_filter'] != '_':
 817                 script = [
 818                         'cd '+work.write(True)+'/*/.',
 819                         opts.user_wrap(opts.fakeroot+' debian/rules binary'),
 820                         'cd ..',
 821                         'echo *.deb >&3',
 822                         ]
 823                 result_debs = source_rules_command(act,script,what,
 824                                 'debian/rules binary',work,1)
 825                 if result_debs == '*': debs = []
 826                 else: debs = debs.split(' ')
 827                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
 828                 for deb in debs:
 829                         m = re.match(deb)
 830                         if not m: act.bomb("badly-named binary `%s'" % deb, se)
 831                         package = m.groups(deb)
 832                         for pat in ah['dsc_filter'].split(','):
 833                                 if fnmatch.fnmatchcase(package,pat):
 834                                         deb_path = work.read()+'/'+deb
 835                                         deb_what = package+'_'+what+'.deb'
 836                                         bin = InputPath(deb_what,deb_path,True)
 837                                         bin.preserve_now()
 838                                         record_binary(bin,'builds')
 839                                         ah.binaries.append(bin)
 840                                         break
 841
 842 def record_binary(
 843
 844 def process_actions():
 845         global binaries
 846
 847         binaries = {}
 848         ix = 0
 849         for act opts.actions:
 850                 if act.kind == 'deb':
 851                         record_binary(act,'builds')
 852                 if act.kind == 'dsc':
 853                         build_source(act)
 854                         # build_source records tree location in ah
 855
 856         binaries = {}
 857         control_override = None
 858         for (kind,path,ah) in opts.actions:
 859                 if kind == 'control':
 860                         control_override = act
 861                 if kind == 'deb':
 862                         record_binary(act,'tests')
 863                 if kind == 'dsc':
 864                         for bin in act.binaries: record_binary(bin,'tests')
 865                         if not act.ah['dsc_tests']: continue
 866                         run_tests(act,control_override)
 867                         control_override = None
 868                 if kind == 'tree':
 869                         run_tests(act,control_override)
 870                         control_override = None
 871
 872 def main():
 873         global testbed
 874         global tmpdir
 875         try:
 876                 parse_args()
 877         except SystemExit, se:
 878                 os._exit(20)
 879         try:
 880                 tmpdir = tempfile.mkdtemp()
 881                 testbed = Testbed()
 882                 testbed.start()
 883                 testbed.open()
 884                 finalise_options()
 885                 process_actions()
 886         except:
 887                 ec = print_exception(sys.exc_info(), '')
 888                 cleanup()
 889                 os._exit(ec)
 890         cleanup()
 891         os._exit(errorcode)
 892
 893 main()