runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 try: our_base = os.environ['AUTOPKGTEST_BASE']+'/lib'
  44 except KeyError: our_base = '/usr/share/autopkgtest/python';
  45 sys.path.insert(1, our_base)
  46
  47 from Autopkgtest import *
  48
  49 #---------- global variables
  50
  51 tmpdir = None           # pathstring on host
  52 testbed = None          # Testbed
  53 errorcode = 0           # exit status that we are going to use
  54 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  55 binaries = None         # Binaries (.debs we have registered)
  56 build_essential = ["build-essential"]
  57 paths = []
  58
  59 #---------- output handling
  60 #
  61 #  There are at least the following kinds of output:
  62 #
  63 #   1. stderr output which consists of
  64 #   1a. our errors
  65 #   1b. stuff printed to stderr by the virtualisation server
  66 #   1c. stuff printed to stderr by our short-lived subprocesseses
  67 #           which we don't expect to fail
  68 #
  69 #   2. trace information, which consists of
  70 #   2a. our trace information from calls to debug()
  71 #   2b. progress information and stderr output from
  72 #        general scripts we run on the host
  73 #   2c. progress information and stderr output from things
  74 #        we run on the testbed including builds
  75 #   2d. stderr and stdout output from actual tests
  76 #
  77 #   xxxx
  78 #   3. actual test results (printed to our stdout)
  79 #
  80 # Cloning of 1a and 2a, where necessary, is done by us writing
  81 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  82 # is done by forking off a copy of ourselves to do plumbing,
  83 # which copy we wait for at the appropriate point.
  84
  85 class DummyOpts:
  86  def __init__(do): do.debuglevel = 0
  87
  88 opts = DummyOpts()
  89 trace_stream = None
  90 summary_stream = None
  91
  92 def pstderr(m):
  93         print >>sys.stderr, m
  94         if trace_stream is not None: print >>trace_stream, m
  95
  96 def debug(m, minlevel=0):
  97         if opts.debuglevel < minlevel: return
  98         if opts.quiet and trace_stream is None: return
  99         p = 'adt-run: trace'
 100         if minlevel: p += `minlevel`
 101         p += ': '
 102         for l in m.rstrip('\n').split('\n'):
 103                 s = p + l
 104                 if not opts.quiet: print >>sys.stderr, s
 105                 if trace_stream is not None: print >>trace_stream, s
 106
 107 def debug_file(hp, minlevel=0):
 108         if opts.debuglevel < minlevel: return
 109         def do_copy(stream, what):
 110                 rc = subprocess.call(['cat',hp], stdout=stream)
 111                 if rc: bomb('cat failed copying data from %s'
 112                             ' to %s, exit code %d' % (hp, what, rc))
 113         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 114         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 115
 116 class Errplumb:
 117  def __init__(ep, critical=False):
 118         to_stderr = critical or not opts.quiet
 119         count = to_stderr + (trace_stream is not None)
 120         if count == 0:
 121                 ep.stream = open('/dev/null','w')
 122                 ep._sp = None
 123         elif count == 1:
 124                 if to_stderr: ep.stream = os.dup(2)
 125                 else: ep.stream = trace_stream
 126                 ep._sp = None
 127         else:
 128                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 129                         stdin=subprocess.PIPE, stdout=trace_stream,
 130                         close_fds=True)
 131                 ep.stream = ep._sp.stdin
 132  def wait(ep):
 133         if ep._sp is None: return
 134         if type(ep.stream) == type(2):
 135                 os.close(ep.stream)
 136                 ep.stream = ()
 137         ep._sp.stdin.close()
 138         rc = ep._sp.wait()
 139         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 140         ep._sp = None
 141
 142 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 143         if dbg is not None:
 144                 if isinstance(dbg,tuple): (what,script) = dbg
 145                 else: (what,script) = (dbg,None)
 146                 debug_subprocess(what, cmdl, script=script)
 147         ep = Errplumb(critical)
 148         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 149         output = running.communicate()[0]
 150         rc = running.wait()
 151         ep.wait()
 152         return (rc, output)
 153
 154 def psummary(m):
 155         if summary_stream is not None: print >>summary_stream, m
 156
 157 def preport(m):
 158         print m
 159         sys.stdout.flush()
 160         if trace_stream is not None: print >>trace_stream, m
 161         psummary(m)
 162
 163 def report(tname, result):
 164         preport('%-20s %s' % (tname, result))
 165
 166 #---------- errors we define
 167
 168 class Quit:
 169         def __init__(q,ec,m): q.ec = ec; q.m = m
 170
 171 def bomb(m):
 172         raise Quit(20, "unexpected error: %s" % m)
 173
 174 def badpkg(m):
 175         preport('blame: ' + ' '.join(testbed.blamed))
 176         preport('badpkg: ' + m)
 177         raise Quit(12, "erroneous package: %s" % m)
 178
 179 class Unsupported:
 180  def __init__(u, lno, m):
 181         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 182         else: u.m = m
 183  def report(u, tname):
 184         global errorcode
 185         errorcode != 2
 186         report(tname, 'SKIP %s' % u.m)
 187
 188 #---------- convenience function
 189
 190 def mkdir_okexist(pathname, mode=02755):
 191         try:
 192                 os.mkdir(pathname, mode)
 193         except (IOError,OSError), oe:
 194                 if oe.errno != errno.EEXIST: raise
 195
 196 def rmtree(what, pathname):
 197         debug('/ %s rmtree %s' % (what, pathname), 2)
 198         try: shutil.rmtree(pathname)
 199         except (IOError,OSError), oe:
 200                 if oe.errno != errno.EEXIST: raise
 201
 202 def debug_subprocess(what, cmdl=None, script=None):
 203         o = '$ '+what+':'
 204         if cmdl is not None:
 205                 ol = []
 206                 for x in cmdl:
 207                         if x is script: x = '<SCRIPT>'
 208                         ol.append(x.    replace('\\','\\\\').
 209                                         replace(' ','\\ ')      )
 210                 o += ' '+ ' '.join(ol)
 211         debug(o)
 212         if script is not None and opts.debuglevel >= 1:
 213                 o = ''
 214                 for l in script.rstrip('\n').split('\n'):
 215                         o += '$     '+l+'\n'
 216                 debug(o,1)
 217
 218 def flatten(l):
 219         return reduce((lambda a,b: a + b), l, [])
 220
 221 #---------- fancy automatic file-copying class
 222
 223 class AutoFile:
 224         # p.what
 225         # p.path[tb]    None or path    not None => path known
 226         # p.file[tb]    None or path    not None => file exists
 227         # p.ephem[tb]   boolean         True => destroyed by tb reset
 228         # p.spec        string or None
 229         # p.spec_tbp    True or False, or not set if spec is None
 230         # p.dir         '' or '/'
 231
 232  def __init__(p, what):
 233         p.what = what
 234         p.path = [None,None]
 235         p.file = [None,None]
 236         p.ephem = [False,False]
 237         p.spec = None
 238         p.spec_tbp = None
 239         p.dir = ''
 240
 241  def __repr__(p):
 242         return "<AF@%s>" % p.__str__()
 243  def __str__(p):
 244         def ptbp(tbp):
 245                 if p.path[tbp] is None: return '-'+p.dir
 246                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 247                 else: return p.path[tbp]+p.dir+'!'
 248         out = p.what
 249         if p.spec is not None:
 250                 if not hasattr(p,'spec_tb'): out += '~'
 251                 elif p.spec_tbp: out += '#'
 252                 else: out += '='
 253                 out += p.spec
 254         out += ':'
 255         out += ptbp(False)
 256         out += '|'
 257         out += ptbp(True)
 258         return out
 259
 260  def _wrong(p, how):
 261         xtra = ''
 262         if p.spec is not None:
 263                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 264         raise ("internal error: %s (%s)" % (how, str(p)))
 265
 266  def _ensure_path(p, tbp):
 267         if p.path[tbp] is None:
 268                 if '/' in p.what:
 269                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 270                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 271                 if not tbp:
 272                         p.path[tbp] = tmpdir+'/'+p.what
 273                 else:
 274                         p.ephem[tbp] = True
 275                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 276
 277  def write(p, tbp=False):
 278         p._debug('write %s...' % 'HT'[tbp])
 279         p._ensure_path(tbp)
 280
 281         if p.dir and not p.file[tbp]:
 282                 if not tbp:
 283                         p._debug('mkdir H')
 284                         mkdir_okexist(p.path[tbp])
 285                 else:
 286                         cmdl = ['sh','-ec',
 287                                 'test -d "$1" || mkdir -p "$1"',
 288                                 'x', p.path[tbp]]
 289                         tf_what = urllib.quote(p.what).replace('/','%2F')
 290                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 291                         if rc: bomb('failed to create directory %s' %
 292                                 p.path[tbp])
 293
 294         p.file[tbp] = p.path[tbp]
 295         return p.path[tbp]
 296
 297  def read(p, tbp=False):
 298         p._debug('read %s...' % 'HT'[tbp])
 299         p._ensure_path(tbp)
 300
 301         if p.file[tbp] is None:
 302                 if p.file[not tbp] is None:
 303                         p._wrong("requesting read but nonexistent")
 304                 cud = ['copyup','copydown'][tbp]
 305                 src = p.file[not tbp] + p.dir
 306                 dst = p.path[tbp] + p.dir
 307                 testbed.command(cud, (src, dst))
 308                 p.file[tbp] = p.path[tbp]
 309
 310         return p.file[tbp] + p.dir
 311
 312  def invalidate(p, tbp=False):
 313         if p.path[tbp] is not None:
 314                 p._debug('invalidating %s' % 'HT'[tbp])
 315         p.file[tbp] = None
 316         if p.spec_tbp != tbp or p.spec is None:
 317                 p.path[tbp] = None
 318
 319  def invalidate_ephem(p, tbp=False):
 320         if p.ephem[tbp]:
 321                 p.invalidate(tbp)
 322
 323  def _debug(p, m):
 324         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 325
 326  def _constructed(p):
 327         p._debug('constructed: '+str(p))
 328         p._check()
 329         paths.append(p)
 330
 331  def _check(p):
 332         for tbp in [False,True]:
 333          for pf in [p.path, p.file]:
 334                 if pf[tbp] is None: continue
 335                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 336                 if p.dir and pf[tbp].endswith('/'):
 337                         pf[tbp] = pf[tbp].rstrip('/')
 338                         if not pf[tbp]: pf[tbp] = '/'
 339                 if not p.dir and pf[tbp].endswith('/'):
 340                         bomb("directory `%s' specified for "
 341                              "non-directory %s" % (pf[tbp], p.what))
 342
 343  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 344         AutoFile.__init__(p,what)
 345         sh_on = ''; sh_sibl = ''
 346         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 347         if sibling: sh_sibl=' (sibling)'
 348         parent._debug('using as base: %s: %s%s%s' %
 349                         (str(parent), leaf, sh_on, sh_sibl))
 350         if not sibling and not parent.dir:
 351                 parent._wrong('asked for non-sibling relative path of non-dir')
 352         if sibling: trim = os.path.dirname
 353         else: trim = lambda x: x
 354         for tbp in [False,True]:
 355                 p.ephem[tbp] = parent.ephem[tbp]
 356                 if parent.path[tbp] is None: continue
 357                 trimmed = trim(parent.path[tbp])
 358                 if trimmed: trimmed += '/'
 359                 p.path[tbp] = trimmed + leaf
 360                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 361                         p.file[tbp] = p.path[tbp]
 362
 363 class InputFile(AutoFile):
 364  def _init(p, what, spec, spec_tbp=False):
 365         AutoFile.__init__(p, what)
 366         p.spec = spec
 367         p.spec_tbp = spec_tbp
 368         p.path[spec_tbp] = spec
 369         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 370  def __init__(p, what, spec, spec_tbp=False):
 371         p._init(what,spec,spec_tbp)
 372         p._constructed()
 373
 374 class InputDir(InputFile):
 375  def __init__(p, what, spec, spec_tbp=False):
 376         InputFile._init(p,what,spec,spec_tbp)
 377         p.dir = '/'
 378         p._constructed()
 379
 380 class OutputFile(AutoFile):
 381  def _init(p, what, spec, spec_tbp=False):
 382         AutoFile.__init__(p, what)
 383         p.spec = spec
 384         p.spec_tbp = spec_tbp
 385         p.path[spec_tbp] = spec
 386  def __init__(p, what, spec, spec_tbp=False):
 387         p._init(what,spec,spec_tbp)
 388         p._constructed()
 389
 390 class OutputDir(OutputFile):
 391  def __init__(p, what, spec, spec_tbp=False):
 392         OutputFile._init(p,what,spec,spec_tbp)
 393         p.dir = '/'
 394         p._constructed()
 395
 396 class RelativeInputDir(AutoFile):
 397  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 398         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 399         p.dir = '/'
 400         p._constructed()
 401
 402 class RelativeInputFile(AutoFile):
 403  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 404         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 405         p._constructed()
 406
 407 class RelativeOutputFile(AutoFile):
 408  def __init__(p, what, parent, leaf, sibling=False):
 409         p._relative_init(what, parent, leaf, None, False, sibling)
 410         p._constructed()
 411
 412 class TemporaryFile(AutoFile):
 413  def __init__(p, what):
 414         AutoFile.__init__(p, what)
 415         p._constructed()
 416
 417 class TemporaryDir(AutoFile):
 418  def __init__(p, what):
 419         AutoFile.__init__(p, what)
 420         p.dir = '/'
 421         p._constructed()
 422
 423 #---------- parsing and representation of the arguments
 424
 425 class Action:
 426  def __init__(a, kind, af, arghandling, what):
 427         # extra attributes get added during processing
 428         a.kind = kind
 429         a.af = af
 430         a.ah = arghandling
 431         a.what = what
 432         a.missing_tests_control = False
 433  def __repr__(a):
 434         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 435
 436 def parse_args():
 437         global opts, timeouts
 438         global n_non_actions # argh, stupid python scoping rules
 439         usage = "%prog <options> --- <virt-server>..."
 440         parser = OptionParser(usage=usage)
 441         pa = parser.add_option
 442         pe = parser.add_option
 443
 444         arghandling = {
 445                 'dsc_tests': True,
 446                 'dsc_filter': '*',
 447                 'deb_forbuilds': 'auto',
 448                 'deb_fortests': 'auto',
 449                 'tb': False,
 450                 'override_control': None,
 451                 'set_lang': 'C'
 452         }
 453         initial_arghandling = arghandling.copy()
 454         n_non_actions = 0
 455
 456         #----------
 457         # actions (ie, test sets to run, sources to build, binaries to use):
 458
 459         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 460                 global n_non_actions
 461                 parser.largs.append((value,kindpath))
 462                 n_non_actions += not(is_act)
 463
 464         def pa_action(long, metavar, kindpath, help, is_act=True):
 465                 pa('','--'+long, action='callback', callback=cb_action,
 466                         nargs=1, type='string',
 467                         callback_args=(long,kindpath,is_act), help=help)
 468
 469         pa_action('built-tree',         'TREE', '@/',
 470                 help='run tests from build tree TREE')
 471
 472         pa_action('unbuilt-tree',       'TREE', '@//',
 473                 help='run tests from build tree TREE')
 474
 475         pa_action('source',             'DSC', '@.dsc',
 476                 help='build DSC and use its tests and/or'
 477                     ' generated binary packages')
 478
 479         pa_action('binary',             'DEB', '@.deb',
 480                help='use binary package DEB according'
 481                     ' to most recent --binaries-* settings')
 482
 483         def cb_actnoarg(op,optstr,value,parser, largsentry):
 484                 parser.largs.append(largsentry)
 485         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 486                 callback_args=((None, ('instantiate',)),),
 487                 help='instantiate testbed now (during testing phase)'
 488                      ' and install packages'
 489                      ' selected for automatic installation, even'
 490                      ' if this might apparently not be required otherwise')
 491
 492         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 493                help='run tests from control file CONTROL instead,'
 494                     ' (applies to next test suite only)')
 495
 496         #----------
 497         # argument handling settings (what ways to use action
 498         #  arguments, and pathname processing):
 499
 500         def cb_setah(option, opt_str, value, parser, toset,setval):
 501                 if type(setval) == list:
 502                         if not value in setval:
 503                                 parser.error('value for %s option (%s) is not '
 504                                  'one of the permitted values (%s)' %
 505                                  (value, opt_str, setval.join(' ')))
 506                 elif setval is not None:
 507                         value = setval
 508                 for v in toset:
 509                         arghandling[v] = value
 510                 parser.largs.append(arghandling.copy())
 511
 512         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 513                 type = metavar
 514                 if type is not None: type = 'string'
 515                 pa('',long, action='callback', callback=cb_setah,
 516                    callback_args=(affected,effect), **kwargs)
 517
 518         #---- paths: host or testbed:
 519         #
 520         pa_setah('--paths-testbed', ['tb'],True,
 521                 help='subsequent path specifications refer to the testbed')
 522         pa_setah('--paths-host', ['tb'],False,
 523                 help='subsequent path specifications refer to the host')
 524
 525         #---- source processing settings:
 526
 527         pa_setah('--sources-tests', ['dsc_tests'],True,
 528                 help='run tests from builds of subsequent sources')
 529         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 530                 help='do not run tests from builds of subsequent sources')
 531
 532         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 533                 type='string', metavar='PATTERN-LIST',
 534                 help='from subsequent sources, use binaries matching'
 535                      ' PATTERN-LIST (comma-separated glob patterns)'
 536                      ' according to most recent --binaries-* settings')
 537         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 538                 help='from subsequent sources, do not use any binaries')
 539         #---- binary package processing settings:
 540
 541         def pa_setahbins(long,toset,how):
 542          pa_setah(long, toset,['ignore','auto','install'],
 543                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 544                 help=how+' ignore binaries, install them as needed'
 545                         ' for dependencies, or unconditionally install'
 546                         ' them, respectively')
 547         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 548         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 549         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 550
 551         #----------
 552         # general options:
 553
 554         def cb_vserv(op,optstr,value,parser):
 555                 parser.values.vserver = list(parser.rargs)
 556                 del parser.rargs[:]
 557
 558         def cb_path(op,optstr,value,parser, constructor,long,dir):
 559                 name = long.replace('-','_')
 560                 af = constructor(long, value,arghandling['tb'])
 561                 setattr(parser.values, name, af)
 562
 563         def pa_path(long, constructor, help, dir=False):
 564                 pa('','--'+long, action='callback', callback=cb_path,
 565                         callback_args=(constructor,long,dir),
 566                         nargs=1, type='string',
 567                         help=help, metavar='PATH')
 568
 569         pa_path('output-dir', OutputDir, dir=True,
 570                 help='write stderr/out files in PATH')
 571
 572         pa('--leave-lang', dest='set_lang', action='store_false',
 573                 help="leave LANG on testbed set to testbed's default")
 574         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 575                 help='set LANG on testbed to LANGVAL', default='C')
 576
 577         pa('','--tmp-dir',              type='string', dest='tmpdir',
 578                 help='write temporary files to TMPDIR, emptying it'
 579                      ' beforehand and leaving it behind at the end')
 580         pa('','--log-file',             type='string', dest='logfile',
 581                 help='write the log LOGFILE, emptying it beforehand,'
 582                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 583         pa('','--summary-file',         type='string', dest='summary',
 584                 help='write a summary report to SUMMARY,'
 585                      ' emptying it beforehand')
 586
 587         for k in timeouts.keys():
 588                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 589                         metavar='T', help='set %s timeout to T')
 590         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 591                         metavar='FACTOR', default=1.0,
 592                         help='multiply all default timeouts by FACTOR')
 593
 594         pa('','--user',                 type='string', dest='user',
 595                 help='run tests as USER (needs root on testbed)')
 596         pa('','--gain-root',            type='string', dest='gainroot',
 597                 help='prefix debian/rules binary with GAINROOT')
 598         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 599         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 600         pa('','--gnupg-home',           type='string', dest='gnupghome',
 601                 default='~/.autopkgtest/gpg',
 602                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 603                         " signing private apt archive);"
 604                         " `fresh' means generate new key each time.")
 605
 606         #----------
 607         # actual meat:
 608
 609         class SpecialOption(optparse.Option): pass
 610         vs_op = SpecialOption('','--VSERVER-DUMMY')
 611         vs_op.action = 'callback'
 612         vs_op.type = None
 613         vs_op.default = None
 614         vs_op.nargs = 0
 615         vs_op.callback = cb_vserv
 616         vs_op.callback_args = ( )
 617         vs_op.callback_kwargs = { }
 618         vs_op.help = 'introduces virtualisation server and args'
 619         vs_op._short_opts = []
 620         vs_op._long_opts = ['---']
 621
 622         pa(vs_op)
 623
 624         (opts,args) = parser.parse_args()
 625         if not hasattr(opts,'vserver'):
 626                 parser.error('you must specifiy --- <virt-server>...')
 627         if n_non_actions >= len(parser.largs):
 628                 parser.error('nothing to do specified')
 629
 630         for k in timeouts.keys():
 631                 t = getattr(opts,'timeout_'+k)
 632                 if t is None: t = timeouts[k] * opts.timeout_factor
 633                 timeouts[k] = int(t)
 634
 635         arghandling = initial_arghandling
 636         opts.actions = []
 637         ix = 0
 638         for act in args:
 639                 if type(act) == dict:
 640                         arghandling = act
 641                         continue
 642                 elif type(act) == tuple:
 643                         pass
 644                 elif type(act) == str:
 645                         act = (act,act)
 646                 else:
 647                         raise ("unknown action in list `%s' having"
 648                               " type `%s'" % (act, type(act)))
 649                 (pathstr, kindpath) = act
 650
 651                 constructor = InputFile
 652                 if type(kindpath) is tuple:             kind = kindpath[0]
 653                 elif kindpath.endswith('.deb'):         kind = 'deb'
 654                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 655                 elif kindpath.endswith('//'):
 656                         kind = 'ubtree'
 657                         constructor = InputDir
 658                 elif kindpath.endswith('/'):
 659                         kind = 'tree'
 660                         constructor = InputDir
 661                 else: parser.error("do not know how to handle filename `%s';"
 662                         " specify --source --binary or --build-tree" %
 663                         kindpath)
 664
 665                 what = '%s%s' % (kind,ix); ix += 1
 666
 667                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 668                 else: fwhatx = '-'+kind
 669
 670                 if pathstr is None: af = None
 671                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 672                 opts.actions.append(Action(kind, af, arghandling, what))
 673
 674 def setup_trace():
 675         global trace_stream, tmpdir, summary_stream
 676
 677         if opts.tmpdir is not None:
 678                 rmtree('tmpdir(specified)',opts.tmpdir)
 679                 mkdir_okexist(opts.tmpdir, 0700)
 680                 tmpdir = opts.tmpdir
 681         else:
 682                 assert(tmpdir is None)
 683                 tmpdir = tempfile.mkdtemp()
 684
 685         if opts.logfile is None:
 686                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 687                         opts.logfile = opts.output_dir.spec + '/log'
 688                 elif opts.tmpdir is not None:
 689                         opts.logfile = opts.tmpdir + '/log'
 690         if opts.logfile is not None:
 691                 trace_stream = open(opts.logfile, 'w', 0)
 692         if opts.summary is not None:
 693                 summary_stream = open(opts.summary, 'w', 0)
 694
 695         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 696
 697 def finalise_options():
 698         global opts, tb, build_essential
 699
 700         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 701                 opts.user = ''
 702
 703         if opts.user is None:
 704                 su = 'suggested-normal-user='
 705                 ul = [
 706                         e[len(su):]
 707                         for e in testbed.caps
 708                         if e.startswith(su)
 709                         ]
 710                 if ul:
 711                         opts.user = ul[0]
 712                 else:
 713                         opts.user = ''
 714
 715         if opts.user:
 716                 if 'root-on-testbed' not in testbed.caps:
 717                         pstderr("warning: virtualisation"
 718                                 " system does not offer root on testbed,"
 719                                 " but --user option specified: failure likely")
 720                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 721         else:
 722                 opts.user_wrap = lambda x: x
 723
 724         if opts.gainroot is None:
 725                 opts.gainroot = ''
 726                 if (opts.user or
 727                     'root-on-testbed' not in testbed.caps):
 728                         opts.gainroot = 'fakeroot'
 729                         build_essential += ['fakeroot']
 730
 731         if opts.gnupghome.startswith('~/'):
 732                 try: home = os.environ['HOME']
 733                 except KeyError:
 734                         parser.error("HOME environment variable"
 735                                 " not set, needed for --gnupghome=`%s"
 736                                 % opts.gnupghome)
 737                 opts.gnupghome = home + opts.gnupghome[1:]
 738         elif opts.gnupghome == 'fresh':
 739                 opts.gnupghome = None
 740
 741 #---------- testbed management - the Testbed class
 742
 743 class Testbed:
 744  def __init__(tb):
 745         tb.sp = None
 746         tb.lastsend = None
 747         tb.scratch = None
 748         tb.modified = False
 749         tb.blamed = []
 750         tb._debug('init')
 751         tb._need_reset_apt = False
 752  def _debug(tb, m, minlevel=0):
 753         debug('** '+m, minlevel)
 754  def start(tb):
 755         tb._debug('start')
 756         p = subprocess.PIPE
 757         debug_subprocess('vserver', opts.vserver)
 758         tb._errplumb = Errplumb(True)
 759         tb.sp = subprocess.Popen(opts.vserver,
 760                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 761         tb.expect('ok')
 762         tb.caps = tb.commandr('capabilities')
 763  def stop(tb):
 764         tb._debug('stop')
 765         tb.close()
 766         if tb.sp is None: return
 767         ec = tb.sp.returncode
 768         if ec is None:
 769                 tb.sp.stdout.close()
 770                 tb.send('quit')
 771                 tb.sp.stdin.close()
 772                 ec = tb.sp.wait()
 773         if ec:
 774                 tb.bomb('testbed gave exit status %d after quit' % ec)
 775         tb._errplumb.wait()
 776  def open(tb):
 777         tb._debug('open, scratch=%s' % tb.scratch)
 778         if tb.scratch is not None: return
 779         pl = tb.commandr('open')
 780         tb._opened(pl)
 781  def _opened(tb, pl):
 782         tb.scratch = InputDir('tb-scratch', pl[0], True)
 783         tb.deps_processed = []
 784         for af in paths: af.invalidate_ephem(True)
 785         tb._auxverbscript_make()
 786  def _auxverbscript_make(tb):
 787         pec = tb.commandr('print-auxverb-command')
 788         if len(pec) < 1: tb.bomb('too few results from print-execute-command')
 789         cmdl = map(urllib.unquote, pec[0].split(','))
 790
 791         tb._debug('cmdl = %s' % (`cmdl`))
 792
 793         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 794         print >>open(tb.ec_auxverbscript.write(),'w'), (
 795 '''#!/bin/sh
 796 set -e
 797 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 798         # This is a pretty nasty hack.  Hopefully it can go away
 799         #  eventually.  See #635763.
 800         set -- dpkg --print-architecture
 801 fi
 802 exec '''+shellquote_cmdl(cmdl)+' "$@"'+"\n"
 803                 )
 804         os.chmod(tb.ec_auxverbscript.write(), 0755)
 805
 806  def mungeing_apt(tb):
 807         if not 'revert' in tb.caps:
 808                 tb._need_reset_apt = True
 809  def reset_apt(tb):
 810         if not tb._need_reset_apt: return
 811         what = 'aptget-update-reset'
 812         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 813         rc = tb.execute(what, cmdl, kind='install')
 814         if rc:
 815                 pstderr("\n" "warning: failed to restore"
 816                         " testbed apt cache, exit code %d" % rc)
 817         tb._need_reset_apt = False
 818  def close(tb):
 819         tb._debug('close, scratch=%s' % tb.scratch)
 820         if tb.scratch is None: return
 821         tb.scratch = None
 822         if tb.sp is None: return
 823         tb.command('close')
 824  def prepare1(tb, deps_new):
 825         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 826                 (tb.modified, tb.deps_processed, deps_new), 1)
 827         if 'revert' in tb.caps and (tb.modified or
 828             [d for d in tb.deps_processed if d not in deps_new]):
 829                 for af in paths: af.read(False)
 830                 tb._debug('reset **')
 831                 pl = tb.commandr('revert')
 832                 tb._opened(pl)
 833         tb.modified = False
 834  def prepare2(tb, deps_new):
 835         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 836         binaries.publish()
 837         tb._install_deps(deps_new)
 838  def prepare(tb, deps_new):
 839         tb.prepare1(deps_new)
 840         tb.prepare2(deps_new)
 841  def _install_deps(tb, deps_new):
 842         tb._debug(' installing dependencies '+`deps_new`, 1)
 843         tb.deps_processed = deps_new
 844         if not deps_new: return
 845         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 846  def needs_reset(tb):
 847         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 848         tb.modified = True
 849  def blame(tb, m):
 850         tb._debug('blame += %s' % m, 1)
 851         tb.blamed.append(m)
 852  def bomb(tb, m):
 853         tb._debug('bomb %s' % m)
 854         if tb.sp is not None:
 855                 tb.sp.stdout.close()
 856                 tb.sp.stdin.close()
 857                 ec = tb.sp.wait()
 858                 if ec: pstderr('adt-run: testbed failing,'
 859                                 ' exit status %d' % ec)
 860         tb.sp = None
 861         raise Quit(16, 'testbed failed: %s' % m)
 862  def send(tb, string):
 863         tb.sp.stdin
 864         try:
 865                 debug('>> '+string, 2)
 866                 print >>tb.sp.stdin, string
 867                 tb.sp.stdin.flush()
 868                 tb.lastsend = string
 869         except:
 870                 (type, value, dummy) = sys.exc_info()
 871                 tb.bomb('cannot send to testbed: %s' % traceback.
 872                         format_exception_only(type, value))
 873  def expect(tb, keyword, nresults=None):
 874         l = tb.sp.stdout.readline()
 875         if not l: tb.bomb('unexpected eof from the testbed')
 876         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 877         l = l.rstrip('\n')
 878         debug('<< '+l, 2)
 879         ll = l.split()
 880         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 881         if ll[0] != keyword:
 882                 if tb.lastsend is None:
 883                         tb.bomb("got banner `%s', expected `%s...'" %
 884                                 (l, keyword))
 885                 else:
 886                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 887                                 (tb.lastsend, l, keyword))
 888         ll = ll[1:]
 889         if nresults is not None and len(ll) != nresults:
 890                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 891                         " expected %d result parameters" %
 892                         (tb.lastsend, l, len(ll), nresults))
 893         return ll
 894  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 895         # pass args=[None,...] or =(None,...) to avoid more url quoting
 896         if type(cmd) is str: cmd = [cmd]
 897         if len(args) and args[0] is None: args = args[1:]
 898         else: args = map(urllib.quote, args)
 899         al = cmd + args
 900         tb.send(string.join(al))
 901         ll = tb.expect('ok', nresults)
 902         if unquote: ll = map(urllib.unquote, ll)
 903         return ll
 904  def command(tb, cmd, args=()):
 905         tb.commandr(cmd, args, 0)
 906  def commandr1(tb, cmd, args=()):
 907         rl = tb.commandr(cmd, args, 1)
 908         return rl[0]
 909  def execute(tb, what, cmdl,
 910                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 911                 script=False, tmpdir=None, kind='short'):
 912         # Options for script:
 913         #   False - do not call debug_subprocess, no synch. reporting required
 914         #   None or string - call debug_subprocess with that value,
 915         #                       plumb stderr through synchronously if possible
 916         # Options for se:
 917         #   None - usual Errplumb (output is of kind 2c)
 918         #   string - path on testbed (output is of kind 2d)
 919
 920         timeout = timeouts[kind]
 921
 922         if script is not False: debug_subprocess(what, cmdl, script=script)
 923         if cwd is None: cwd = tb.scratch.write(True)
 924
 925         xdump = None
 926         if se is None:
 927                 ep = Errplumb()
 928                 se_catch = TemporaryFile(what+'-xerr')
 929                 se_use = se_catch.write(True)
 930                 if not opts.quiet: xdump = 'debug=2-2'
 931                 elif trace_stream is not None:
 932                         xdump = 'debug=2-%d' % trace_stream.fileno()
 933         else:
 934                 ep = None
 935                 se_catch = None
 936                 se_use = se
 937
 938         cmdl = [None,
 939                 ','.join(map(urllib.quote, cmdl)),
 940                 si, so, se_use, cwd]
 941
 942         if timeout is not None and timeout > 0:
 943                 cmdl.append('timeout=%d' % timeout)
 944
 945         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 946         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 947         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 948         if opts.set_lang is not False:
 949                 cmdl.append('env=LANG=%s' % opts.set_lang)
 950
 951         rc = tb.commandr1('execute', cmdl)
 952         try: rc = int(rc)
 953         except ValueError: bomb("execute for %s gave invalid response `%s'"
 954                                         % (what,rc))
 955
 956         if se_catch is not None:
 957                 debug_file(se_catch.read())
 958         if ep is not None:
 959                 ep.wait()
 960
 961         return rc
 962
 963  def satisfy_dependencies_string(tb, deps, what):
 964         # Must have called Binaries.configure_apt
 965         debug('dependencies: %s: satisfying %s' % (what,deps))
 966         dsc = TemporaryFile('deps.dsc')
 967         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 968         # pbuilder-satisfydepends has a bug where it ignores the
 969         #  Build-Depends if it's the last line in the dsc (#635696)
 970         tb.satisfy_dependencies_dsc(dsc, what)
 971
 972  def satisfy_dependencies_dsc(tb, dsc, what):
 973         # Must have called Binaries.configure_apt
 974         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 975                 '--binary-all', # --check-key
 976                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 977                 '-c',dsc.read()
 978         ]
 979         # The --internal-chrootexec option is really handy but
 980         #  perhaps we are not supposed to use it ?  See also #635697.
 981         debug('dependencies: %s: running %s' % (what,`cmdl`))
 982         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 983         if rc: badpkg('dependency install failed, exit code %d' % rc)
 984
 985 #---------- representation of test control files: Field*, Test, etc.
 986
 987 class FieldBase:
 988  def __init__(f, fname, stz, base, tnames, vl):
 989         assert(vl)
 990         f.stz = stz
 991         f.base = base
 992         f.tnames = tnames
 993         f.vl = vl
 994  def words(f):
 995         def distribute(vle):
 996                 (lno, v) = vle
 997                 r = v.split()
 998                 r = map((lambda w: (lno, w)), r)
 999                 return r
1000         return flatten(map(distribute, f.vl))
1001  def atmostone(f):
1002         if len(vl) == 1:
1003                 (f.lno, f.v) = vl[0]
1004         else:
1005                 raise Unsupported(f.vl[1][0],
1006                         'only one %s field allowed' % fn)
1007         return f.v
1008
1009 class FieldIgnore(FieldBase):
1010  def parse(f): pass
1011
1012 class Restriction:
1013  def __init__(r,rname,base): pass
1014
1015 class Restriction_rw_build_tree(Restriction): pass
1016 class Restriction_breaks_testbed(Restriction):
1017  def __init__(r, rname, base):
1018         if 'revert' not in testbed.caps:
1019                 raise Unsupported(f.lno,
1020                         'Test breaks testbed but testbed cannot revert')
1021 class Restriction_needs_root(Restriction):
1022  def __init__(r, rname, base):
1023         if 'root-on-testbed' not in testbed.caps:
1024                 raise Unsupported(f.lno,
1025                         'Test needs root on testbed which is not available')
1026
1027 class Field_Restrictions(FieldBase):
1028  def parse(f):
1029         for wle in f.words():
1030                 (lno, rname) = wle
1031                 nrname = rname.replace('-','_')
1032                 try: rclass = globals()['Restriction_'+nrname]
1033                 except KeyError: raise Unsupported(lno,
1034                         'unknown restriction %s' % rname)
1035                 r = rclass(nrname, f.base)
1036                 f.base['restriction_names'].append(rname)
1037                 f.base['restrictions'].append(r)
1038
1039 class Field_Features(FieldIgnore):
1040  def parse(f):
1041         for wle in f.words():
1042                 (lno, fname) = wle
1043                 f.base['feature_names'].append(fname)
1044                 nfname = fname.replace('-','_')
1045                 try: fclass = globals()['Feature_'+nfname]
1046                 except KeyError: continue
1047                 ft = fclass(nfname, f.base)
1048                 f.base['features'].append(ft)
1049
1050 class Field_Tests(FieldIgnore): pass
1051
1052 class Field_Depends(FieldBase):
1053  def parse(f):
1054         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
1055         dl = map(lambda x: x.strip(),
1056                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1057         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1058         for d in dl:
1059                 if re.search(d):
1060                         badpkg("Test Depends field contains dependency"
1061                                " `%s' with invalid characters" % d)
1062         f.base['depends'] = dl
1063
1064 class Field_Tests_directory(FieldBase):
1065  def parse(f):
1066         td = atmostone(f)
1067         if td.startswith('/'): raise Unspported(f.lno,
1068                 'Tests-Directory may not be absolute')
1069         f.base['testsdir'] = td
1070
1071 def run_tests(stanzas, tree):
1072         global errorcode, testbed
1073         if stanzas == ():
1074                 report('*', 'SKIP no tests in this package')
1075                 errorcode |= 8
1076         for stanza in stanzas:
1077                 tests = stanza[' tests']
1078                 if not tests:
1079                         report('*', 'SKIP package has metadata but no tests')
1080                         errorcode |= 8
1081                 for t in tests:
1082                         t.prepare()
1083                         t.run(tree)
1084                         if 'breaks-testbed' in t.restriction_names:
1085                                 testbed.needs_reset()
1086                 testbed.needs_reset()
1087
1088 class Test:
1089  def __init__(t, tname, base, act):
1090         if '/' in tname: raise Unsupported(base[' lno'],
1091                 'test name may not contain / character')
1092         for k in base: setattr(t,k,base[k])
1093         t.tname = tname
1094         t.act = act
1095         t.what = act.what+'t-'+tname
1096         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1097         else: t.path = tname
1098         t._debug('constructed; path=%s' % t.path)
1099         t._debug(' .depends=%s' % t.depends)
1100  def _debug(t, m):
1101         debug('& %s: %s' % (t.what, m))
1102  def report(t, m):
1103         report(t.what, m)
1104  def reportfail(t, m):
1105         global errorcode
1106         errorcode |= 4
1107         report(t.what, 'FAIL ' + m)
1108  def prepare(t):
1109         t._debug('preparing')
1110         dn = []
1111         for d in t.depends:
1112                 t._debug(' processing dependency '+d)
1113                 if not '@' in d:
1114                         t._debug('  literal dependency '+d)
1115                         dn.append(d)
1116                 else:
1117                         for (pkg,bin) in t.act.binaries:
1118                                 d = d.replace('@',pkg)
1119                                 t._debug('  synthesised dependency '+d)
1120                                 dn.append(d)
1121         testbed.prepare(dn)
1122  def run(t, tree):
1123         t._debug('[----------------------------------------')
1124         def stdouterr(oe):
1125                 idstr = t.what + '-' + oe
1126                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1127                         use_dir = opts.output_dir
1128                 else:
1129                         use_dir = testbed.scratch
1130                 return RelativeOutputFile(idstr, use_dir, idstr)
1131
1132         if hasattr(t.act,'work'): t.act.work.write(True)
1133         tree.read(True)
1134
1135         af = RelativeInputFile(t.what, tree, t.path)
1136         so = stdouterr('stdout')
1137         se = stdouterr('stderr')
1138
1139         tf = af.read(True)
1140         tmpdir = None
1141
1142         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1143         if rc: bomb('failed to chmod +x %s' % tf)
1144
1145         if 'needs-root' not in t.restriction_names and opts.user is not None:
1146                 tfl = ['su',opts.user,'-c',tf]
1147                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1148                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1149                 if opts.user:
1150                         script += '; chown %s "$1"' % opts.user
1151                         if 'rw-build-tree' in t.restriction_names:
1152                                 script += '; chown -R %s "$2"' % opts.user
1153                 rc = testbed.execute('mktmpdir-'+t.what,
1154                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1155                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1156                                 % (tmpdir, rc))
1157         else:
1158                 tfl = [tf]
1159
1160         rc = testbed.execute('test-'+t.what, tfl,
1161                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1162                 tmpdir=tmpdir, kind='test')
1163
1164         so_read = so.read()
1165         se_read = se.read()
1166
1167         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1168         stab = os.stat(se_read)
1169         if stab.st_size != 0:
1170                 l = open(se_read).readline()
1171                 l = l.rstrip('\n \t\r')
1172                 if len(l) > 35: l = l[:35] + '...'
1173                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1174                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1175                 debug_file(se_read)
1176         elif rc != 0:
1177                 t.reportfail('non-zero exit status %d' % rc)
1178         else:
1179                 t.report('PASS')
1180
1181         stab = os.stat(so_read)
1182         if stab.st_size != 0:
1183                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1184                 debug_file(so_read)
1185
1186         t._debug('----------------------------------------]')
1187
1188 def read_control(act, tree, control_override):
1189         stanzas = [ ]
1190
1191         if control_override is not None:
1192                 control_af = control_override
1193                 testbed.blame('arg:'+control_override.spec)
1194         else:
1195                 if act.missing_tests_control:
1196                         return ()
1197                 control_af = RelativeInputFile(act.what+'-testcontrol',
1198                         tree, 'debian/tests/control')
1199         try:
1200                 control = open(control_af.read(), 'r')
1201         except (IOError,OSError), oe:
1202                 if oe[0] != errno.ENOENT: raise
1203                 return []
1204
1205         lno = 0
1206         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1207         stz = None      # stz[field_name][index] = (lno, value)
1208                         # special field names:
1209                         # stz[' lno'] = number
1210                         # stz[' tests'] = list of Test objects
1211         def end_stanza(stz):
1212                 if stz is None: return
1213                 stz[' errs'] = 0
1214                 stanzas.append(stz)
1215                 stz = None
1216                 hcurrent = None
1217
1218         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1219         while 1:
1220                 l = control.readline()
1221                 if not l: break
1222                 lno += 1
1223                 if not l.endswith('\n'): badctrl('unterminated line')
1224                 if regexp.compile('\s*\#').match(l): continue
1225                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1226                 initmat = initre.match(l)
1227                 if initmat:
1228                         (fname, l) = initmat.groups()
1229                         fname = string.capwords(fname)
1230                         if stz is None:
1231                                 stz = { ' lno': lno, ' tests': [] }
1232                         if not stz.has_key(fname): stz[fname] = [ ]
1233                         hcurrent = stz[fname]
1234                 elif regexp.compile('\s').match(l):
1235                         if not hcurrent: badctrl('unexpected continuation')
1236                 else:
1237                         badctrl('syntax error')
1238                 hcurrent.append((lno, l))
1239         end_stanza(stz)
1240
1241         def testbadctrl(stz, lno, m):
1242                 report_badctrl(lno, m)
1243                 stz[' errs'] += 1
1244
1245         for stz in stanzas:
1246                 try:
1247                         try: tnames = stz['Tests']
1248                         except KeyError:
1249                                 tnames = ['*']
1250                                 raise Unsupported(stz[' lno'],
1251                                         'no Tests field')
1252                         tnames = map((lambda lt: lt[1]), tnames)
1253                         tnames = string.join(tnames).split()
1254                         base = {
1255                                 'restriction_names': [],
1256                                 'restrictions': [],
1257                                 'feature_names': [],
1258                                 'features': [],
1259                                 'testsdir': 'debian/tests',
1260                                 'depends' : '@'
1261                         }
1262                         for fname in stz.keys():
1263                                 if fname.startswith(' '): continue
1264                                 vl = stz[fname]
1265                                 try: fclass = globals()['Field_'+
1266                                         fname.replace('-','_')]
1267                                 except KeyError: raise Unsupported(vl[0][0],
1268                                         'unknown metadata field %s' % fname)
1269                                 f = fclass(stz, fname, base, tnames, vl)
1270                                 f.parse()
1271                         for tname in tnames:
1272                                 t = Test(tname, base, act)
1273                                 stz[' tests'].append(t)
1274                 except Unsupported, u:
1275                         for tname in tnames: u.report(tname)
1276                         continue
1277
1278         return stanzas
1279
1280 def print_exception(ei, msgprefix=''):
1281         if msgprefix: pstderr(msgprefix)
1282         (et, q, tb) = ei
1283         if et is Quit:
1284                 pstderr('adt-run: ' + q.m)
1285                 psummary('quitting: '+q.m)
1286                 return q.ec
1287         else:
1288                 pstderr("adt-run: unexpected, exceptional, error:")
1289                 psummary('quitting: unexpected error, consult transcript')
1290                 traceback.print_exc(None, sys.stderr)
1291                 if trace_stream is not None:
1292                         traceback.print_exc(None, trace_stream)
1293                 return 20
1294
1295 def cleanup():
1296         global trace_stream
1297         try:
1298                 if testbed is not None:
1299                         testbed.reset_apt()
1300                         testbed.stop()
1301                 if opts.tmpdir is None and tmpdir is not None:
1302                         rmtree('tmpdir', tmpdir)
1303                 if trace_stream is not None:
1304                         trace_stream.close()
1305                         trace_stream = None
1306         except:
1307                 print_exception(sys.exc_info(),
1308                         '\nadt-run: error cleaning up:\n')
1309                 os._exit(20)
1310
1311 #---------- registration, installation etc. of .deb's: Binaries
1312
1313 def determine_package(act):
1314         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1315         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1316         if rc: badpkg('failed to parse binary package, code %d' % rc)
1317         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1318         act.pkg = None
1319         for l in output.split('\n'):
1320                 m = re.match(l)
1321                 if not m: continue
1322                 if act.pkg: badpkg('two Package: lines in control file')
1323                 act.pkg = m.groups()[0]
1324         if not act.pkg: badpkg('no good Package: line in control file')
1325
1326 class Binaries:
1327  def __init__(b, tb):
1328         b.dir = TemporaryDir('binaries')
1329         b.dir.write()
1330         ok = False
1331
1332         if opts.gnupghome is None:
1333                 opts.gnupghome = tmpdir+'/gnupg'
1334
1335         b._debug('initialising')
1336         try:
1337                 for x in ['pubring','secring']:
1338                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1339                 ok = True
1340         except (IOError,OSError), oe:
1341                 if oe.errno != errno.ENOENT: raise
1342
1343         if ok: b._debug('no key generation needed')
1344         else: b.genkey()
1345
1346  def _debug(b, s):
1347         debug('* '+s)
1348
1349  def genkey(b):
1350         b._debug('preparing for key generation')
1351
1352         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1353         mkdir_okexist(opts.gnupghome, 0700)
1354
1355         script = '''
1356   exec >&2
1357   cd "$1"
1358   cat <<"END" >key-gen-params
1359 Key-Type: DSA
1360 Key-Length: 1024
1361 Key-Usage: sign
1362 Name-Real: autopkgtest per-run key
1363 Name-Comment: do not trust this key
1364 Name-Email: autopkgtest@example.com
1365 END
1366   set -x
1367   gpg --homedir="$1" --batch --gen-key key-gen-params
1368                 '''
1369         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1370         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1371         if rc: bomb('key generation failed, code %d' % rc)
1372
1373  def apt_configs(b):
1374         return {
1375                 "Debug::pkgProblemResolver": "true",
1376                 "APT::Get::force-yes" : "true",
1377                 "APT::Get::Assume-Yes" : "true",
1378                 "quiet" : "true",
1379         }
1380
1381  def _configure_apt(b, tb):
1382         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1383                         True)
1384         f = open(config.write(),'w')
1385         for (k,v) in b.apt_configs().iteritems():
1386                 print >>f, '%s { "%s"; };' % (k, v)
1387         f.close()
1388         config.read(True)
1389
1390         prefs = OutputFile('apt-prefs','/etc/apt/preferences.d/90autopkgtest',
1391                         True)
1392         print >>open(prefs.write(),'w'), '''
1393 Package: *
1394 Pin: origin ""
1395 Pin-Priority: 1002
1396 '''
1397         prefs.read(True)
1398
1399  def _apt_get(b):
1400         ag = ['apt-get','-q']
1401         for kv in b.apt_configs().iteritems():
1402                 ag += ['-o', '%s=%s' % kv]
1403         return ' '.join(ag)
1404
1405  def reset(b):
1406         b._debug('reset')
1407         rmtree('binaries', b.dir.read())
1408         b.dir.invalidate()
1409         b.dir.write()
1410         b.install = []
1411         b.blamed = []
1412         b.registered = set()
1413
1414  def register(b, act, pkg, af, forwhat, blamed):
1415         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1416                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1417
1418         if act.ah['deb_'+forwhat] == 'ignore': return
1419
1420         b.blamed += testbed.blamed
1421
1422         leafname = pkg+'.deb'
1423         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1424
1425         try: os.remove(dest.write())
1426         except (IOError,OSError), oe:
1427                 if oe.errno != errno.ENOENT: raise e
1428
1429         try: os.link(af.read(), dest.write())
1430         except (IOError,OSError), oe:
1431                 if oe.errno != errno.EXDEV: raise e
1432                 shutil.copy(af.read(), dest)
1433
1434         if act.ah['deb_'+forwhat] == 'install':
1435                 b.install.append(pkg)
1436
1437         b.registered.add(pkg)
1438
1439  def publish(b):
1440         b._debug('publish')
1441
1442         b._configure_apt(testbed)
1443
1444         script = '''
1445   exec >&2
1446   cd "$1"
1447   apt-ftparchive packages . >Packages
1448   gzip <Packages >Packages.gz
1449   apt-ftparchive release . >Release
1450   rm -f Release.gpg
1451   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1452   gpg --homedir="$2" --batch --export >archive-key.pgp
1453                 '''
1454         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1455         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1456         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1457
1458         b.dir.invalidate(True)
1459         apt_source = b.dir.read(True)
1460
1461         so = TemporaryFile('vlds')
1462         script = '''
1463   exec 3>&1 >&2
1464   apt-key add archive-key.pgp
1465   echo "deb file://'''+apt_source+''' /" >/etc/apt/sources.list.d/autopkgtest.list
1466   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1467     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1468   fi
1469   '''+ b._apt_get() +''' update >&2
1470   cat /var/lib/dpkg/status >&3
1471                 '''
1472         testbed.mungeing_apt()
1473         rc = testbed.execute('apt-key', ['sh','-ec',script],
1474                                 so=so.write(True), cwd=b.dir.write(True),
1475                                 script=script, kind='install')
1476         if rc: bomb('apt setup failed with exit code %d' % rc)
1477
1478         testbed.blamed += b.blamed
1479
1480         b._debug('publish reinstall checking...')
1481         pkgs_reinstall = set()
1482         pkg = None
1483         for l in open(so.read()):
1484                 if l.startswith('Package: '):
1485                         pkg = l[9:].rstrip()
1486                 elif l.startswith('Status: install '):
1487                         if pkg in b.registered:
1488                                 pkgs_reinstall.add(pkg)
1489                                 b._debug(' publish reinstall needs '+pkg)
1490
1491         if pkgs_reinstall:
1492                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1493                 what = 'apt-get-reinstall'
1494                 cmdl = (b._apt_get() + ' --reinstall install '+
1495                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1496                 cmdl = ['sh','-c',cmdl]
1497                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1498                 if rc: badpkg("installation of basic binarries failed,"
1499                                 " exit code %d" % rc)
1500
1501         b._debug('publish install...')
1502         for pkg in b.install:
1503                 what = 'apt-get-install-%s' % pkg
1504                 testbed.blame(pkg)
1505                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1506                 cmdl = ['sh','-c',cmdl]
1507                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1508                 if rc: badpkg("installation of %s failed, exit code %d"
1509                                 % (pkg, rc))
1510
1511         b._debug('publish done')
1512
1513 #---------- processing of sources (building)
1514
1515 def source_rules_command(act,script,what,which,work,cwd,
1516                                 results_lines=0,xargs=[]):
1517         script = [      "exec 3>&1 >&2",
1518                         "set -x"        ] + script
1519         script = '\n'.join(script)
1520         so = TemporaryFile('%s-%s-results' % (what,which))
1521         rc = testbed.execute('%s-%s' % (what,which),
1522                         ['sh','-ec',script]+xargs, script=script,
1523                         so=so.write(True), cwd=cwd, kind='build')
1524         results = open(so.read()).read().rstrip('\n')
1525         if len(results): results = results.split("\n")
1526         else: results = []
1527         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1528         if results_lines is not None and len(results) != results_lines:
1529                 badpkg("got %d lines of results from %s where %d expected"
1530                         % (len(results), which, results_lines))
1531         if results_lines==1: return results[0]
1532         return results
1533
1534 def build_source(act, control_override):
1535         act.blame = 'arg:'+act.af.spec
1536         testbed.blame(act.blame)
1537         testbed.prepare1([])
1538         testbed.needs_reset()
1539
1540         what = act.what
1541         basename = act.af.spec
1542         debiancontrol = None
1543         act.binaries = []
1544
1545         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1546
1547         if act.kind == 'dsc':
1548                 dsc = act.af
1549                 dsc_file = open(dsc.read())
1550                 in_files = False
1551                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1552                 for l in dsc_file:
1553                         l = l.rstrip('\n')
1554                         if l.startswith('Files:'): in_files = True; continue
1555                         elif l.startswith('#'): pass
1556                         elif not l.startswith(' '):
1557                                 in_files = False
1558                                 if l.startswith('Source:'):
1559                                         act.blame = 'dsc:'+l[7:].strip()
1560                                         testbed.blame(act.blame)
1561                         if not in_files: continue
1562
1563                         m = fre.match(l)
1564                         if not m: badpkg(".dsc contains unparseable line"
1565                                         " in Files: `%s'" % l)
1566                         leaf = m.groups(0)[0]
1567                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1568                                         sibling=True)
1569                         subfile.read(True)
1570                 dsc.read(True)
1571
1572         if act.kind == 'ubtree':
1573                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1574                         act.af, 'debian/control')
1575                 dsc = TemporaryFile(what+'-fakedsc')
1576                 dsc_w = open(dsc.write(), 'w')
1577                 for l in open(debiancontrol.read()):
1578                         l = l.rstrip('\n')
1579                         if not len(l): break
1580                         print >>dsc_w, l
1581                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1582                 dsc_w.close()
1583
1584         if act.kind == 'dsc':
1585                 testbed.prepare2([])
1586                 testbed.satisfy_dependencies_string('dpkg-dev',
1587                                                 'install dpkg-dev')
1588
1589         work = TemporaryDir(what+'-build')
1590         act.work = work
1591
1592         tmpdir = work.write(True)+'/tmpdir'
1593         tmpdir_script = [
1594                         'TMPDIR="$1"',
1595                         'rm -rf -- "$TMPDIR"',
1596                         'export TMPDIR',
1597                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1598                 ]
1599
1600         if act.kind == 'ubtree':
1601                 spec = '%s/real-tree' % work.write(True)
1602                 create_command = '''
1603                         rm -rf "$spec"
1604                         mkdir "$spec"
1605                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1606                         '''
1607                 initcwd = act.af.read(True)
1608
1609         if act.kind == 'dsc':
1610                 spec = dsc.read(True)
1611                 create_command = '''
1612                         dpkg-source -x $spec
1613                         '''
1614                 initcwd = work.write(True)
1615
1616         script = [
1617                 'spec="$2"',
1618                 'origpwd=`pwd`',
1619                 'cd '+work.write(True)
1620         ]
1621
1622         if opts.user:
1623                 script += ([ 'chown '+opts.user+' .' ] +
1624                         tmpdir_script +
1625                         [ 'spec="$spec" origpwd="$origpwd" '
1626                                 +opts.user_wrap(create_command) ])
1627         else:
1628                 script += (tmpdir_script +
1629                         [ create_command ])
1630
1631         script += [
1632                         'cd [a-z0-9]*-*/.',
1633                         'pwd >&3',
1634                         'set +e; test -f debian/tests/control; echo $? >&3'
1635                 ]
1636         (result_pwd, control_test_rc) = source_rules_command(
1637                         act,script,what,'extract',work,
1638                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1639
1640         filter = act.ah['dsc_filter']
1641
1642         if control_test_rc == '1': act.missing_tests_control = True
1643
1644         # For optional builds:
1645         #
1646         # We might need to build the package because:
1647         #   - we want its binaries (filter isn't _ and at least one of the
1648         #       deb_... isn't ignore)
1649         #   - the test control file says so
1650         #       (assuming we have any tests)
1651
1652         class NeedBuildException: pass
1653         def build_needed(m):
1654                 debug_b('build needed for %s' % m)
1655                 raise NeedBuildException()
1656
1657         try:
1658                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1659                                       act.ah['deb_fortests'] != 'ignore'):
1660                         build_needed('binaries')
1661
1662                 result_pwd_af = InputDir(what+'-treeforcontrol',
1663                         result_pwd, True)
1664                 stanzas = read_control(act, result_pwd_af, control_override)
1665                 for stanza in stanzas:
1666                         for t in stanza[' tests']:
1667                                 if 'build-needed' not in t.restriction_names:
1668                                         build_needed('test %s' % t.tname)
1669                                 for d in t.depends:
1670                                         if '@' in d:
1671                                                 build_needed('test %s '
1672                                                  'dependency %s' % (t.tname,d))
1673
1674                 debug_b('build not needed')
1675                 built = False
1676
1677         except NeedBuildException:
1678
1679                 if act.kind != 'dsc':
1680                         testbed.prepare2([])
1681
1682                 testbed.satisfy_dependencies_string('build-essential',
1683                                 'install build-essential')
1684                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1685
1686                 script = tmpdir_script + [
1687                         'cd "$2"',
1688                         'dpkg-checkbuilddeps',
1689                         opts.user_wrap('debian/rules build'),
1690                         ]
1691                 source_rules_command(act,script,what,'build',work,
1692                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1693
1694                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1695                         badpkg("results dir `%s' is not in expected parent"
1696                                " dir `%s'" % (result_pwd, work.read(True)))
1697
1698                 built = True
1699
1700         act.tests_tree = RelativeInputDir(what+'-tests-tree',
1701                                 work, os.path.basename(result_pwd),
1702                                 True)
1703
1704         if not built:
1705                 act.blamed = []
1706                 return
1707
1708         act.blamed = copy.copy(testbed.blamed)
1709
1710         debug_b('filter=%s' % filter)
1711         if filter != '_':
1712                 script = tmpdir_script + [
1713                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1714                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1715                         'cd ..',
1716                         'echo *.deb >&3',
1717                         ]
1718                 result_debs = source_rules_command(act,script,what,
1719                                 'binary',work,work.write(True),
1720                                 results_lines=1, xargs=['x',tmpdir])
1721                 if result_debs == '*.deb': debs = []
1722                 else: debs = result_debs.split(' ')
1723                 debug_b('debs='+`debs`)
1724                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1725                 for deb in debs:
1726                         m = re.match(deb)
1727                         if not m: badpkg("badly-named binary `%s'" % deb)
1728                         pkg = m.groups()[0]
1729                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1730                         for pat in filter.split(','):
1731                                 debug_b('  pat=%s' % pat)
1732                                 if not fnmatch.fnmatchcase(pkg,pat):
1733                                         debug_b('   no match')
1734                                         continue
1735                                 deb_what = pkg+'_'+what+'.deb'
1736                                 bin = RelativeInputFile(deb_what,work,deb,True)
1737                                 debug_b('  deb_what=%s, bin=%s' %
1738                                         (deb_what, str(bin)))
1739                                 binaries.register(act,pkg,bin,
1740                                         'forbuilds',testbed.blamed)
1741                                 act.binaries.append((pkg,bin))
1742                                 break
1743                 debug_b('all done.')
1744
1745 #---------- main processing loop and main program
1746
1747 def process_actions():
1748         global binaries
1749
1750         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1751         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1752         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1753
1754         debug_a1('starting')
1755         testbed.open()
1756         binaries = Binaries(testbed)
1757
1758         binaries.reset()
1759         control_override = None
1760
1761         debug_a1('builds ...')
1762         for act in opts.actions:
1763                 debug_a2('%s %s' %
1764                         (act.kind, act.what))
1765
1766                 if act.kind == 'control':
1767                         control_override = act.af
1768                 if act.kind == 'deb':
1769                         testbed.blame('arg:'+act.af.spec)
1770                         determine_package(act)
1771                         testbed.blame('deb:'+act.pkg)
1772                         binaries.register(act,act.pkg,act.af,
1773                                 'forbuilds',testbed.blamed)
1774                 if act.kind == 'dsc' or act.kind == 'ubtree':
1775                         build_source(act, control_override)
1776                 if act.kind == 'tree':
1777                         act.binaries = []
1778                 if act.kind.endswith('tree') or act.kind == 'dsc':
1779                         control_override = None
1780                 if act.kind == 'instantiate':
1781                         pass
1782
1783         debug_a1('builds done.')
1784
1785         binaries.reset()
1786         control_override = None
1787
1788         debug_a1('tests ...')
1789         for act in opts.actions:
1790                 debug_a2('test %s %s' % (act.kind, act.what))
1791
1792                 testbed.needs_reset()
1793                 if act.kind == 'control':
1794                         control_override = act.af
1795                 if act.kind == 'deb':
1796                         binaries.register(act,act.pkg,act.af,'fortests',
1797                                 ['deb:'+act.pkg])
1798                 if act.kind == 'dsc' or act.kind == 'ubtree':
1799                         for (pkg,bin) in act.binaries:
1800                                 binaries.register(act,pkg,bin,'fortests',
1801                                         act.blamed)
1802                 if act.kind == 'dsc':
1803                         if act.ah['dsc_tests']:
1804                                 debug_a3('read control ...')
1805                                 stanzas = read_control(act, act.tests_tree,
1806                                                 control_override)
1807                                 testbed.blamed += act.blamed
1808                                 debug_a3('run_tests ...')
1809                                 run_tests(stanzas, act.tests_tree)
1810                         control_override = None
1811                 if act.kind == 'tree' or act.kind == 'ubtree':
1812                         testbed.blame('arg:'+act.af.spec)
1813                         stanzas = read_control(act, act.af, control_override)
1814                         debug_a3('run_tests ...')
1815                         run_tests(stanzas, act.af)
1816                         control_override = None
1817                 if act.kind == 'instantiate':
1818                         testbed.prepare([])
1819         debug_a1('tests done.')
1820
1821 def main():
1822         global testbed
1823         global tmpdir
1824         try:
1825                 parse_args()
1826         except SystemExit, se:
1827                 os._exit(20)
1828         try:
1829                 setup_trace()
1830                 testbed = Testbed()
1831                 testbed.start()
1832                 finalise_options()
1833                 process_actions()
1834         except:
1835                 ec = print_exception(sys.exc_info(), '')
1836                 cleanup()
1837                 os._exit(ec)
1838         cleanup()
1839         os._exit(errorcode)
1840
1841 main()