runner/adt-run

   1 #!/usr/bin/python2.6
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 try: our_base = os.environ['AUTOPKGTEST_BASE']+'/lib'
  44 except KeyError: our_base = '/usr/share/autopkgtest/python';
  45 sys.path.insert(1, our_base)
  46
  47 from Autopkgtest import *
  48
  49 #---------- global variables
  50
  51 tmpdir = None           # pathstring on host
  52 testbed = None          # Testbed
  53 errorcode = 0           # exit status that we are going to use
  54 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  55 binaries = None         # Binaries (.debs we have registered)
  56 build_essential = ["build-essential"]
  57 paths = []
  58
  59 #---------- output handling
  60 #
  61 #  There are at least the following kinds of output:
  62 #
  63 #   1. stderr output which consists of
  64 #   1a. our errors
  65 #   1b. stuff printed to stderr by the virtualisation server
  66 #   1c. stuff printed to stderr by our short-lived subprocesseses
  67 #           which we don't expect to fail
  68 #
  69 #   2. trace information, which consists of
  70 #   2a. our trace information from calls to debug()
  71 #   2b. progress information and stderr output from
  72 #        general scripts we run on the host
  73 #   2c. progress information and stderr output from things
  74 #        we run on the testbed including builds
  75 #   2d. stderr and stdout output from actual tests
  76 #
  77 #   xxxx
  78 #   3. actual test results (printed to our stdout)
  79 #
  80 # Cloning of 1a and 2a, where necessary, is done by us writing
  81 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  82 # is done by forking off a copy of ourselves to do plumbing,
  83 # which copy we wait for at the appropriate point.
  84
  85 class DummyOpts:
  86  def __init__(do): do.debuglevel = 0
  87
  88 opts = DummyOpts()
  89 trace_stream = None
  90 summary_stream = None
  91
  92 def pstderr(m):
  93         print >>sys.stderr, m
  94         if trace_stream is not None: print >>trace_stream, m
  95
  96 def debug(m, minlevel=0):
  97         if opts.debuglevel < minlevel: return
  98         if opts.quiet and trace_stream is None: return
  99         p = 'adt-run: trace'
 100         if minlevel: p += `minlevel`
 101         p += ': '
 102         for l in m.rstrip('\n').split('\n'):
 103                 s = p + l
 104                 if not opts.quiet: print >>sys.stderr, s
 105                 if trace_stream is not None: print >>trace_stream, s
 106
 107 def debug_file(hp, minlevel=0):
 108         if opts.debuglevel < minlevel: return
 109         def do_copy(stream, what):
 110                 rc = subprocess.call(['cat',hp], stdout=stream)
 111                 if rc: bomb('cat failed copying data from %s'
 112                             ' to %s, exit code %d' % (hp, what, rc))
 113         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 114         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 115
 116 class Errplumb:
 117  def __init__(ep, critical=False):
 118         to_stderr = critical or not opts.quiet
 119         count = to_stderr + (trace_stream is not None)
 120         if count == 0:
 121                 ep.stream = open('/dev/null','w')
 122                 ep._sp = None
 123         elif count == 1:
 124                 if to_stderr: ep.stream = os.dup(2)
 125                 else: ep.stream = trace_stream
 126                 ep._sp = None
 127         else:
 128                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 129                         stdin=subprocess.PIPE, stdout=trace_stream,
 130                         close_fds=True)
 131                 ep.stream = ep._sp.stdin
 132  def wait(ep):
 133         if ep._sp is None: return
 134         if type(ep.stream) == type(2):
 135                 os.close(ep.stream)
 136                 ep.stream = ()
 137         ep._sp.stdin.close()
 138         rc = ep._sp.wait()
 139         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 140         ep._sp = None
 141
 142 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 143         if dbg is not None:
 144                 if isinstance(dbg,tuple): (what,script) = dbg
 145                 else: (what,script) = (dbg,None)
 146                 debug_subprocess(what, cmdl, script=script)
 147         ep = Errplumb(critical)
 148         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 149         output = running.communicate()[0]
 150         rc = running.wait()
 151         ep.wait()
 152         return (rc, output)
 153
 154 def psummary(m):
 155         if summary_stream is not None: print >>summary_stream, m
 156
 157 def preport(m):
 158         print m
 159         sys.stdout.flush()
 160         if trace_stream is not None: print >>trace_stream, m
 161         psummary(m)
 162
 163 def report(tname, result):
 164         preport('%-20s %s' % (tname, result))
 165
 166 #---------- errors we define
 167
 168 class Quit:
 169         def __init__(q,ec,m): q.ec = ec; q.m = m
 170
 171 def bomb(m):
 172         raise Quit(20, "unexpected error: %s" % m)
 173
 174 def badpkg(m):
 175         preport('blame: ' + ' '.join(testbed.blamed))
 176         preport('badpkg: ' + m)
 177         raise Quit(12, "erroneous package: %s" % m)
 178
 179 class Unsupported:
 180  def __init__(u, lno, m):
 181         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 182         else: u.m = m
 183  def report(u, tname):
 184         global errorcode
 185         errorcode != 2
 186         report(tname, 'SKIP %s' % u.m)
 187
 188 #---------- convenience function
 189
 190 def mkdir_okexist(pathname, mode=02755):
 191         try:
 192                 os.mkdir(pathname, mode)
 193         except (IOError,OSError), oe:
 194                 if oe.errno != errno.EEXIST: raise
 195
 196 def rmtree(what, pathname):
 197         debug('/ %s rmtree %s' % (what, pathname), 2)
 198         try: shutil.rmtree(pathname)
 199         except (IOError,OSError), oe:
 200                 if oe.errno != errno.EEXIST: raise
 201
 202 def debug_subprocess(what, cmdl=None, script=None):
 203         o = '$ '+what+':'
 204         if cmdl is not None:
 205                 ol = []
 206                 for x in cmdl:
 207                         if x is script: x = '<SCRIPT>'
 208                         ol.append(x.    replace('\\','\\\\').
 209                                         replace(' ','\\ ')      )
 210                 o += ' '+ ' '.join(ol)
 211         debug(o)
 212         if script is not None and opts.debuglevel >= 1:
 213                 o = ''
 214                 for l in script.rstrip('\n').split('\n'):
 215                         o += '$     '+l+'\n'
 216                 debug(o,1)
 217
 218 def flatten(l):
 219         return reduce((lambda a,b: a + b), l, [])
 220
 221 #---------- fancy automatic file-copying class
 222
 223 class AutoFile:
 224         # p.what
 225         # p.path[tb]    None or path    not None => path known
 226         # p.file[tb]    None or path    not None => file exists
 227         # p.ephem[tb]   boolean         True => destroyed by tb reset
 228         # p.spec        string or None
 229         # p.spec_tbp    True or False, or not set if spec is None
 230         # p.dir         '' or '/'
 231
 232  def __init__(p, what):
 233         p.what = what
 234         p.path = [None,None]
 235         p.file = [None,None]
 236         p.ephem = [False,False]
 237         p.spec = None
 238         p.spec_tbp = None
 239         p.dir = ''
 240
 241  def __repr__(p):
 242         return "<AF@%s>" % p.__str__()
 243  def __str__(p):
 244         def ptbp(tbp):
 245                 if p.path[tbp] is None: return '-'+p.dir
 246                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 247                 else: return p.path[tbp]+p.dir+'!'
 248         out = p.what
 249         if p.spec is not None:
 250                 if not hasattr(p,'spec_tb'): out += '~'
 251                 elif p.spec_tbp: out += '#'
 252                 else: out += '='
 253                 out += p.spec
 254         out += ':'
 255         out += ptbp(False)
 256         out += '|'
 257         out += ptbp(True)
 258         return out
 259
 260  def _wrong(p, how):
 261         xtra = ''
 262         if p.spec is not None:
 263                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 264         raise Exception("internal error: %s (%s)" % (how, str(p)))
 265
 266  def _ensure_path(p, tbp):
 267         if p.path[tbp] is None:
 268                 if '/' in p.what:
 269                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 270                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 271                 if not tbp:
 272                         p.path[tbp] = tmpdir+'/'+p.what
 273                 else:
 274                         p.ephem[tbp] = True
 275                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 276
 277  def write(p, tbp=False):
 278         p._debug('write %s...' % 'HT'[tbp])
 279         p._ensure_path(tbp)
 280
 281         if p.dir and not p.file[tbp]:
 282                 if not tbp:
 283                         p._debug('mkdir H')
 284                         mkdir_okexist(p.path[tbp])
 285                 else:
 286                         cmdl = ['sh','-ec',
 287                                 'test -d "$1" || mkdir -p "$1"',
 288                                 'x', p.path[tbp]]
 289                         tf_what = urllib.quote(p.what).replace('/','%2F')
 290                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 291                         if rc: bomb('failed to create directory %s' %
 292                                 p.path[tbp])
 293
 294         p.file[tbp] = p.path[tbp]
 295         return p.path[tbp]
 296
 297  def read(p, tbp=False):
 298         p._debug('read %s...' % 'HT'[tbp])
 299         p._ensure_path(tbp)
 300
 301         if p.file[tbp] is None:
 302                 if p.file[not tbp] is None:
 303                         p._wrong("requesting read but nonexistent")
 304                 cud = ['copyup','copydown'][tbp]
 305                 src = p.file[not tbp] + p.dir
 306                 dst = p.path[tbp] + p.dir
 307                 testbed.command(cud, (src, dst))
 308                 p.file[tbp] = p.path[tbp]
 309
 310         return p.file[tbp] + p.dir
 311
 312  def invalidate(p, tbp=False):
 313         if p.path[tbp] is not None:
 314                 p._debug('invalidating %s' % 'HT'[tbp])
 315         p.file[tbp] = None
 316         if p.spec_tbp != tbp or p.spec is None:
 317                 p.path[tbp] = None
 318
 319  def invalidate_ephem(p, tbp=False):
 320         if p.ephem[tbp]:
 321                 p.invalidate(tbp)
 322
 323  def _debug(p, m):
 324         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 325
 326  def _constructed(p):
 327         p._debug('constructed: '+str(p))
 328         p._check()
 329         paths.append(p)
 330
 331  def _check(p):
 332         for tbp in [False,True]:
 333          for pf in [p.path, p.file]:
 334                 if pf[tbp] is None: continue
 335                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 336                 if p.dir and pf[tbp].endswith('/'):
 337                         pf[tbp] = pf[tbp].rstrip('/')
 338                         if not pf[tbp]: pf[tbp] = '/'
 339                 if not p.dir and pf[tbp].endswith('/'):
 340                         bomb("directory `%s' specified for "
 341                              "non-directory %s" % (pf[tbp], p.what))
 342
 343  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 344         AutoFile.__init__(p,what)
 345         sh_on = ''; sh_sibl = ''
 346         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 347         if sibling: sh_sibl=' (sibling)'
 348         parent._debug('using as base: %s: %s%s%s' %
 349                         (str(parent), leaf, sh_on, sh_sibl))
 350         if not sibling and not parent.dir:
 351                 parent._wrong('asked for non-sibling relative path of non-dir')
 352         if sibling: trim = os.path.dirname
 353         else: trim = lambda x: x
 354         for tbp in [False,True]:
 355                 p.ephem[tbp] = parent.ephem[tbp]
 356                 if parent.path[tbp] is None: continue
 357                 trimmed = trim(parent.path[tbp])
 358                 if trimmed: trimmed += '/'
 359                 p.path[tbp] = trimmed + leaf
 360                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 361                         p.file[tbp] = p.path[tbp]
 362
 363 class InputFile(AutoFile):
 364  def _init(p, what, spec, spec_tbp=False):
 365         AutoFile.__init__(p, what)
 366         p.spec = spec
 367         p.spec_tbp = spec_tbp
 368         p.path[spec_tbp] = spec
 369         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 370  def __init__(p, what, spec, spec_tbp=False):
 371         p._init(what,spec,spec_tbp)
 372         p._constructed()
 373
 374 class InputDir(InputFile):
 375  def __init__(p, what, spec, spec_tbp=False):
 376         InputFile._init(p,what,spec,spec_tbp)
 377         p.dir = '/'
 378         p._constructed()
 379
 380 class OutputFile(AutoFile):
 381  def _init(p, what, spec, spec_tbp=False):
 382         AutoFile.__init__(p, what)
 383         p.spec = spec
 384         p.spec_tbp = spec_tbp
 385         p.path[spec_tbp] = spec
 386  def __init__(p, what, spec, spec_tbp=False):
 387         p._init(what,spec,spec_tbp)
 388         p._constructed()
 389
 390 class OutputDir(OutputFile):
 391  def __init__(p, what, spec, spec_tbp=False):
 392         OutputFile._init(p,what,spec,spec_tbp)
 393         p.dir = '/'
 394         p._constructed()
 395
 396 class RelativeInputDir(AutoFile):
 397  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 398         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 399         p.dir = '/'
 400         p._constructed()
 401
 402 class RelativeInputFile(AutoFile):
 403  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 404         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 405         p._constructed()
 406
 407 class RelativeOutputFile(AutoFile):
 408  def __init__(p, what, parent, leaf, sibling=False):
 409         p._relative_init(what, parent, leaf, None, False, sibling)
 410         p._constructed()
 411
 412 class TemporaryFile(AutoFile):
 413  def __init__(p, what):
 414         AutoFile.__init__(p, what)
 415         p._constructed()
 416
 417 class TemporaryDir(AutoFile):
 418  def __init__(p, what):
 419         AutoFile.__init__(p, what)
 420         p.dir = '/'
 421         p._constructed()
 422
 423 #---------- parsing and representation of the arguments
 424
 425 class Action:
 426  def __init__(a, kind, af, arghandling, what):
 427         # extra attributes get added during processing
 428         a.kind = kind
 429         a.af = af
 430         a.ah = arghandling
 431         a.what = what
 432         a.missing_tests_control = False
 433  def __repr__(a):
 434         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 435
 436 def parse_args():
 437         global opts, timeouts
 438         global n_non_actions # argh, stupid python scoping rules
 439         usage = "%prog <options> --- <virt-server>..."
 440         parser = OptionParser(usage=usage)
 441         pa = parser.add_option
 442         pe = parser.add_option
 443
 444         arghandling = {
 445                 'dsc_tests': True,
 446                 'dsc_filter': '*',
 447                 'deb_forbuilds': 'auto',
 448                 'deb_fortests': 'auto',
 449                 'tb': False,
 450                 'override_control': None,
 451                 'set_lang': 'C'
 452         }
 453         initial_arghandling = arghandling.copy()
 454         n_non_actions = 0
 455
 456         #----------
 457         # actions (ie, test sets to run, sources to build, binaries to use):
 458
 459         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 460                 global n_non_actions
 461                 parser.largs.append((value,kindpath))
 462                 n_non_actions += not(is_act)
 463
 464         def pa_action(long, metavar, kindpath, help, is_act=True):
 465                 pa('','--'+long, action='callback', callback=cb_action,
 466                         nargs=1, type='string',
 467                         callback_args=(long,kindpath,is_act), help=help)
 468
 469         pa_action('built-tree',         'TREE', '@/',
 470                 help='run tests from build tree TREE')
 471
 472         pa_action('unbuilt-tree',       'TREE', '@//',
 473                 help='run tests from build tree TREE')
 474
 475         pa_action('source',             'DSC', '@.dsc',
 476                 help='build DSC and use its tests and/or'
 477                     ' generated binary packages')
 478
 479         pa_action('binary',             'DEB', '@.deb',
 480                help='use binary package DEB according'
 481                     ' to most recent --binaries-* settings')
 482
 483         def cb_actnoarg(op,optstr,value,parser, largsentry):
 484                 parser.largs.append(largsentry)
 485         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 486                 callback_args=((None, ('instantiate',)),),
 487                 help='instantiate testbed now (during testing phase)'
 488                      ' and install packages'
 489                      ' selected for automatic installation, even'
 490                      ' if this might apparently not be required otherwise')
 491
 492         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 493                help='run tests from control file CONTROL instead,'
 494                     ' (applies to next test suite only)')
 495
 496         #----------
 497         # argument handling settings (what ways to use action
 498         #  arguments, and pathname processing):
 499
 500         def cb_setah(option, opt_str, value, parser, toset,setval):
 501                 if type(setval) == list:
 502                         if not value in setval:
 503                                 parser.error('value for %s option (%s) is not '
 504                                  'one of the permitted values (%s)' %
 505                                  (value, opt_str, setval.join(' ')))
 506                 elif setval is not None:
 507                         value = setval
 508                 for v in toset:
 509                         arghandling[v] = value
 510                 parser.largs.append(arghandling.copy())
 511
 512         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 513                 type = metavar
 514                 if type is not None: type = 'string'
 515                 pa('',long, action='callback', callback=cb_setah,
 516                    callback_args=(affected,effect), **kwargs)
 517
 518         #---- paths: host or testbed:
 519         #
 520         pa_setah('--paths-testbed', ['tb'],True,
 521                 help='subsequent path specifications refer to the testbed')
 522         pa_setah('--paths-host', ['tb'],False,
 523                 help='subsequent path specifications refer to the host')
 524
 525         #---- source processing settings:
 526
 527         pa_setah('--sources-tests', ['dsc_tests'],True,
 528                 help='run tests from builds of subsequent sources')
 529         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 530                 help='do not run tests from builds of subsequent sources')
 531
 532         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 533                 type='string', metavar='PATTERN-LIST',
 534                 help='from subsequent sources, use binaries matching'
 535                      ' PATTERN-LIST (comma-separated glob patterns)'
 536                      ' according to most recent --binaries-* settings')
 537         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 538                 help='from subsequent sources, do not use any binaries')
 539         #---- binary package processing settings:
 540
 541         def pa_setahbins(long,toset,how):
 542          pa_setah(long, toset,['ignore','auto','install'],
 543                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 544                 help=how+' ignore binaries, install them as needed'
 545                         ' for dependencies, or unconditionally install'
 546                         ' them, respectively')
 547         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 548         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 549         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 550
 551         #----------
 552         # general options:
 553
 554         def cb_vserv(op,optstr,value,parser):
 555                 parser.values.vserver = list(parser.rargs)
 556                 del parser.rargs[:]
 557
 558         def cb_path(op,optstr,value,parser, constructor,long,dir):
 559                 name = long.replace('-','_')
 560                 af = constructor(long, value,arghandling['tb'])
 561                 setattr(parser.values, name, af)
 562
 563         def pa_path(long, constructor, help, dir=False):
 564                 pa('','--'+long, action='callback', callback=cb_path,
 565                         callback_args=(constructor,long,dir),
 566                         nargs=1, type='string',
 567                         help=help, metavar='PATH')
 568
 569         pa_path('output-dir', OutputDir, dir=True,
 570                 help='write stderr/out files in PATH')
 571
 572         pa('--leave-lang', dest='set_lang', action='store_false',
 573                 help="leave LANG on testbed set to testbed's default")
 574         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 575                 help='set LANG on testbed to LANGVAL', default='C')
 576
 577         pa('','--tmp-dir',              type='string', dest='tmpdir',
 578                 help='write temporary files to TMPDIR, emptying it'
 579                      ' beforehand and leaving it behind at the end')
 580         pa('','--log-file',             type='string', dest='logfile',
 581                 help='write the log LOGFILE, emptying it beforehand,'
 582                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 583         pa('','--summary-file',         type='string', dest='summary',
 584                 help='write a summary report to SUMMARY,'
 585                      ' emptying it beforehand')
 586
 587         for k in timeouts.keys():
 588                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 589                         metavar='T', help='set %s timeout to T')
 590         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 591                         metavar='FACTOR', default=1.0,
 592                         help='multiply all default timeouts by FACTOR')
 593
 594         pa('','--user',                 type='string', dest='user',
 595                 help='run tests as USER (needs root on testbed)')
 596         pa('','--gain-root',            type='string', dest='gainroot',
 597                 help='prefix debian/rules binary with GAINROOT')
 598         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 599         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 600         pa('','--gnupg-home',           type='string', dest='gnupghome',
 601                 default='~/.autopkgtest/gpg',
 602                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 603                         " signing private apt archive);"
 604                         " `fresh' means generate new key each time.")
 605
 606         #----------
 607         # actual meat:
 608
 609         class SpecialOption(optparse.Option): pass
 610         vs_op = SpecialOption('','--VSERVER-DUMMY')
 611         vs_op.action = 'callback'
 612         vs_op.type = None
 613         vs_op.default = None
 614         vs_op.nargs = 0
 615         vs_op.callback = cb_vserv
 616         vs_op.callback_args = ( )
 617         vs_op.callback_kwargs = { }
 618         vs_op.help = 'introduces virtualisation server and args'
 619         vs_op._short_opts = []
 620         vs_op._long_opts = ['---']
 621
 622         pa(vs_op)
 623
 624         (opts,args) = parser.parse_args()
 625         if not hasattr(opts,'vserver'):
 626                 parser.error('you must specifiy --- <virt-server>...')
 627         if n_non_actions >= len(parser.largs):
 628                 parser.error('nothing to do specified')
 629
 630         for k in timeouts.keys():
 631                 t = getattr(opts,'timeout_'+k)
 632                 if t is None: t = timeouts[k] * opts.timeout_factor
 633                 timeouts[k] = int(t)
 634
 635         arghandling = initial_arghandling
 636         opts.actions = []
 637         ix = 0
 638         for act in args:
 639                 if type(act) == dict:
 640                         arghandling = act
 641                         continue
 642                 elif type(act) == tuple:
 643                         pass
 644                 elif type(act) == str:
 645                         act = (act,act)
 646                 else:
 647                         raise Exception("unknown action in list `%s' having"
 648                               " type `%s'" % (act, type(act)))
 649                 (pathstr, kindpath) = act
 650
 651                 constructor = InputFile
 652                 if type(kindpath) is tuple:             kind = kindpath[0]
 653                 elif kindpath.endswith('.deb'):         kind = 'deb'
 654                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 655                 elif kindpath.endswith('//'):
 656                         kind = 'ubtree'
 657                         constructor = InputDir
 658                 elif kindpath.endswith('/'):
 659                         kind = 'tree'
 660                         constructor = InputDir
 661                 else: parser.error("do not know how to handle filename `%s';"
 662                         " specify --source --binary or --build-tree" %
 663                         kindpath)
 664
 665                 what = '%s%s' % (kind,ix); ix += 1
 666
 667                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 668                 else: fwhatx = '-'+kind
 669
 670                 if pathstr is None: af = None
 671                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 672                 opts.actions.append(Action(kind, af, arghandling, what))
 673
 674 def setup_trace():
 675         global trace_stream, tmpdir, summary_stream
 676
 677         if opts.tmpdir is not None:
 678                 rmtree('tmpdir(specified)',opts.tmpdir)
 679                 mkdir_okexist(opts.tmpdir, 0700)
 680                 tmpdir = opts.tmpdir
 681         else:
 682                 assert(tmpdir is None)
 683                 tmpdir = tempfile.mkdtemp()
 684
 685         if opts.logfile is None:
 686                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 687                         opts.logfile = opts.output_dir.spec + '/log'
 688                 elif opts.tmpdir is not None:
 689                         opts.logfile = opts.tmpdir + '/log'
 690         if opts.logfile is not None:
 691                 trace_stream = open(opts.logfile, 'w', 0)
 692         if opts.summary is not None:
 693                 summary_stream = open(opts.summary, 'w', 0)
 694
 695         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 696
 697 def finalise_options():
 698         global opts, tb, build_essential
 699
 700         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 701                 opts.user = ''
 702
 703         if opts.user is None:
 704                 su = 'suggested-normal-user='
 705                 ul = [
 706                         e[len(su):]
 707                         for e in testbed.caps
 708                         if e.startswith(su)
 709                         ]
 710                 if ul:
 711                         opts.user = ul[0]
 712                 else:
 713                         opts.user = ''
 714
 715         if opts.user:
 716                 if 'root-on-testbed' not in testbed.caps:
 717                         pstderr("warning: virtualisation"
 718                                 " system does not offer root on testbed,"
 719                                 " but --user option specified: failure likely")
 720                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 721         else:
 722                 opts.user_wrap = lambda x: x
 723
 724         if opts.gainroot is None:
 725                 opts.gainroot = ''
 726                 if (opts.user or
 727                     'root-on-testbed' not in testbed.caps):
 728                         opts.gainroot = 'fakeroot'
 729                         build_essential += ['fakeroot']
 730
 731         if opts.gnupghome.startswith('~/'):
 732                 try: home = os.environ['HOME']
 733                 except KeyError:
 734                         parser.error("HOME environment variable"
 735                                 " not set, needed for --gnupghome=`%s"
 736                                 % opts.gnupghome)
 737                 opts.gnupghome = home + opts.gnupghome[1:]
 738         elif opts.gnupghome == 'fresh':
 739                 opts.gnupghome = None
 740
 741 #---------- testbed management - the Testbed class
 742
 743 class Testbed:
 744  def __init__(tb):
 745         tb.sp = None
 746         tb.lastsend = None
 747         tb.scratch = None
 748         tb.modified = False
 749         tb.blamed = []
 750         tb._debug('init')
 751         tb._need_reset_apt = False
 752  def _debug(tb, m, minlevel=0):
 753         debug('** '+m, minlevel)
 754  def start(tb):
 755         tb._debug('start')
 756         p = subprocess.PIPE
 757         debug_subprocess('vserver', opts.vserver)
 758         tb._errplumb = Errplumb(True)
 759         tb.sp = subprocess.Popen(opts.vserver,
 760                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 761         tb.expect('ok')
 762         tb.caps = tb.commandr('capabilities')
 763  def stop(tb):
 764         tb._debug('stop')
 765         tb.close()
 766         if tb.sp is None: return
 767         ec = tb.sp.returncode
 768         if ec is None:
 769                 tb.sp.stdout.close()
 770                 tb.send('quit')
 771                 tb.sp.stdin.close()
 772                 ec = tb.sp.wait()
 773         if ec:
 774                 tb.bomb('testbed gave exit status %d after quit' % ec)
 775         tb._errplumb.wait()
 776  def open(tb):
 777         tb._debug('open, scratch=%s' % tb.scratch)
 778         if tb.scratch is not None: return
 779         pl = tb.commandr('open')
 780         tb._opened(pl)
 781  def _opened(tb, pl):
 782         tb.scratch = InputDir('tb-scratch', pl[0], True)
 783         tb.deps_processed = []
 784         for af in paths: af.invalidate_ephem(True)
 785         tb._auxverbscript_make()
 786  def _auxverbscript_make(tb):
 787         pec = tb.commandr('print-auxverb-command')
 788         if len(pec) < 1: tb.bomb('too few results from print-execute-command')
 789         cmdl = map(urllib.unquote, pec[0].split(','))
 790
 791         tb._debug('cmdl = %s' % (`cmdl`))
 792
 793         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 794         print >>open(tb.ec_auxverbscript.write(),'w'), (
 795 '''#!/bin/sh
 796 set -e
 797 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 798         # This is a pretty nasty hack.  Hopefully it can go away
 799         #  eventually.  See #635763.
 800         set -- dpkg --print-architecture
 801 fi
 802 exec '''+shellquote_cmdl(cmdl)+' "$@"'+"\n"
 803                 )
 804         os.chmod(tb.ec_auxverbscript.write(), 0755)
 805
 806  def mungeing_apt(tb):
 807         if not 'revert' in tb.caps:
 808                 tb._need_reset_apt = True
 809  def reset_apt(tb):
 810         if not tb._need_reset_apt: return
 811         what = 'aptget-update-reset'
 812         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 813         rc = tb.execute(what, cmdl, kind='install')
 814         if rc:
 815                 pstderr("\n" "warning: failed to restore"
 816                         " testbed apt cache, exit code %d" % rc)
 817         tb._need_reset_apt = False
 818  def close(tb):
 819         tb._debug('close, scratch=%s' % tb.scratch)
 820         if tb.scratch is None: return
 821         tb.scratch = None
 822         if tb.sp is None: return
 823         tb.command('close')
 824  def prepare1(tb, deps_new):
 825         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 826                 (tb.modified, tb.deps_processed, deps_new), 1)
 827         if 'revert' in tb.caps and (tb.modified or
 828             [d for d in tb.deps_processed if d not in deps_new]):
 829                 for af in paths: af.read(False)
 830                 tb._debug('reset **')
 831                 pl = tb.commandr('revert')
 832                 tb._opened(pl)
 833         tb.modified = False
 834  def prepare2(tb, deps_new):
 835         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 836         binaries.publish()
 837         tb._install_deps(deps_new)
 838  def prepare(tb, deps_new):
 839         tb.prepare1(deps_new)
 840         tb.prepare2(deps_new)
 841  def _install_deps(tb, deps_new):
 842         tb._debug(' installing dependencies '+`deps_new`, 1)
 843         tb.deps_processed = deps_new
 844         if not deps_new: return
 845         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 846  def needs_reset(tb):
 847         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 848         tb.modified = True
 849  def blame(tb, m):
 850         tb._debug('blame += %s' % m, 1)
 851         tb.blamed.append(m)
 852  def bomb(tb, m):
 853         tb._debug('bomb %s' % m)
 854         if tb.sp is not None:
 855                 tb.sp.stdout.close()
 856                 tb.sp.stdin.close()
 857                 ec = tb.sp.wait()
 858                 if ec: pstderr('adt-run: testbed failing,'
 859                                 ' exit status %d' % ec)
 860         tb.sp = None
 861         raise Quit(16, 'testbed failed: %s' % m)
 862  def send(tb, string):
 863         tb.sp.stdin
 864         try:
 865                 debug('>> '+string, 2)
 866                 print >>tb.sp.stdin, string
 867                 tb.sp.stdin.flush()
 868                 tb.lastsend = string
 869         except:
 870                 (type, value, dummy) = sys.exc_info()
 871                 tb.bomb('cannot send to testbed: %s' % traceback.
 872                         format_exception_only(type, value))
 873  def expect(tb, keyword, nresults=None):
 874         l = tb.sp.stdout.readline()
 875         if not l: tb.bomb('unexpected eof from the testbed')
 876         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 877         l = l.rstrip('\n')
 878         debug('<< '+l, 2)
 879         ll = l.split()
 880         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 881         if ll[0] != keyword:
 882                 if tb.lastsend is None:
 883                         tb.bomb("got banner `%s', expected `%s...'" %
 884                                 (l, keyword))
 885                 else:
 886                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 887                                 (tb.lastsend, l, keyword))
 888         ll = ll[1:]
 889         if nresults is not None and len(ll) != nresults:
 890                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 891                         " expected %d result parameters" %
 892                         (tb.lastsend, l, len(ll), nresults))
 893         return ll
 894  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 895         # pass args=[None,...] or =(None,...) to avoid more url quoting
 896         if type(cmd) is str: cmd = [cmd]
 897         if len(args) and args[0] is None: args = args[1:]
 898         else: args = map(urllib.quote, args)
 899         al = cmd + args
 900         tb.send(string.join(al))
 901         ll = tb.expect('ok', nresults)
 902         if unquote: ll = map(urllib.unquote, ll)
 903         return ll
 904  def command(tb, cmd, args=()):
 905         tb.commandr(cmd, args, 0)
 906  def commandr1(tb, cmd, args=()):
 907         rl = tb.commandr(cmd, args, 1)
 908         return rl[0]
 909  def execute(tb, what, cmdl,
 910                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 911                 script=False, tmpdir=None, kind='short'):
 912         # Options for script:
 913         #   False - do not call debug_subprocess, no synch. reporting required
 914         #   None or string - call debug_subprocess with that value,
 915         #                       plumb stderr through synchronously if possible
 916         # Options for se:
 917         #   None - usual Errplumb (output is of kind 2c)
 918         #   string - path on testbed (output is of kind 2d)
 919
 920         timeout = timeouts[kind]
 921
 922         if script is not False: debug_subprocess(what, cmdl, script=script)
 923         if cwd is None: cwd = tb.scratch.write(True)
 924
 925         xdump = None
 926         if se is None:
 927                 ep = Errplumb()
 928                 se_catch = TemporaryFile(what+'-xerr')
 929                 se_use = se_catch.write(True)
 930                 if not opts.quiet: xdump = 'debug=2-2'
 931                 elif trace_stream is not None:
 932                         xdump = 'debug=2-%d' % trace_stream.fileno()
 933         else:
 934                 ep = None
 935                 se_catch = None
 936                 se_use = se
 937
 938         cmdl = [None,
 939                 ','.join(map(urllib.quote, cmdl)),
 940                 si, so, se_use, cwd]
 941
 942         if timeout is not None and timeout > 0:
 943                 cmdl.append('timeout=%d' % timeout)
 944
 945         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 946         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 947         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 948         if opts.set_lang is not False:
 949                 cmdl.append('env=LANG=%s' % opts.set_lang)
 950
 951         rc = tb.commandr1('execute', cmdl)
 952         try: rc = int(rc)
 953         except ValueError: bomb("execute for %s gave invalid response `%s'"
 954                                         % (what,rc))
 955
 956         if se_catch is not None:
 957                 debug_file(se_catch.read())
 958         if ep is not None:
 959                 ep.wait()
 960
 961         return rc
 962
 963  def satisfy_dependencies_string(tb, deps, what):
 964         # Must have called Binaries.configure_apt
 965         debug('dependencies: %s: satisfying %s' % (what,deps))
 966         dsc = TemporaryFile('deps.dsc')
 967         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 968         # pbuilder-satisfydepends has a bug where it ignores the
 969         #  Build-Depends if it's the last line in the dsc (#635696)
 970         tb.satisfy_dependencies_dsc(dsc, what)
 971
 972  def satisfy_dependencies_dsc(tb, dsc, what):
 973         # Must have called Binaries.configure_apt
 974         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 975                 '--binary-all', # --check-key
 976                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 977                 '-c',dsc.read()
 978         ]
 979         # The --internal-chrootexec option is really handy but
 980         #  perhaps we are not supposed to use it ?  See also #635697.
 981         debug('dependencies: %s: running %s' % (what,`cmdl`))
 982         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 983         if rc: badpkg('dependency install failed, exit code %d' % rc)
 984
 985 #---------- representation of test control files: Field*, Test, etc.
 986
 987 class FieldBase:
 988  def __init__(f, fname, stz, base, tnames, vl):
 989         assert(vl)
 990         f.stz = stz
 991         f.base = base
 992         f.tnames = tnames
 993         f.vl = vl
 994  def words(f):
 995         def distribute(vle):
 996                 (lno, v) = vle
 997                 r = v.split()
 998                 r = map((lambda w: (lno, w)), r)
 999                 return r
1000         return flatten(map(distribute, f.vl))
1001  def atmostone(f):
1002         if len(vl) == 1:
1003                 (f.lno, f.v) = vl[0]
1004         else:
1005                 raise Unsupported(f.vl[1][0],
1006                         'only one %s field allowed' % fn)
1007         return f.v
1008
1009 class FieldIgnore(FieldBase):
1010  def parse(f): pass
1011
1012 class Restriction:
1013  def __init__(r,rname,base): pass
1014
1015 class Restriction_rw_build_tree(Restriction): pass
1016 class Restriction_build_neeeded(Restriction): pass
1017 class Restriction_breaks_testbed(Restriction):
1018  def __init__(r, rname, base):
1019         if 'revert-full-system' not in testbed.caps:
1020                 raise Unsupported(f.lno,
1021         'Test breaks testbed but testbed does not advertise revert-full-system')
1022 class Restriction_needs_root(Restriction):
1023  def __init__(r, rname, base):
1024         if 'root-on-testbed' not in testbed.caps:
1025                 raise Unsupported(f.lno,
1026                         'Test needs root on testbed which is not available')
1027
1028 class Field_Restrictions(FieldBase):
1029  def parse(f):
1030         for wle in f.words():
1031                 (lno, rname) = wle
1032                 nrname = rname.replace('-','_')
1033                 try: rclass = globals()['Restriction_'+nrname]
1034                 except KeyError: raise Unsupported(lno,
1035                         'unknown restriction %s' % rname)
1036                 r = rclass(nrname, f.base)
1037                 f.base['restriction_names'].append(rname)
1038                 f.base['restrictions'].append(r)
1039
1040 class Field_Features(FieldIgnore):
1041  def parse(f):
1042         for wle in f.words():
1043                 (lno, fname) = wle
1044                 f.base['feature_names'].append(fname)
1045                 nfname = fname.replace('-','_')
1046                 try: fclass = globals()['Feature_'+nfname]
1047                 except KeyError: continue
1048                 ft = fclass(nfname, f.base)
1049                 f.base['features'].append(ft)
1050
1051 class Field_Tests(FieldIgnore): pass
1052
1053 class Field_Depends(FieldBase):
1054  def parse(f):
1055         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
1056         dl = map(lambda x: x.strip(),
1057                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1058         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1059         for d in dl:
1060                 if re.search(d):
1061                         badpkg("Test Depends field contains dependency"
1062                                " `%s' with invalid characters" % d)
1063         f.base['depends'] = dl
1064
1065 class Field_Tests_directory(FieldBase):
1066  def parse(f):
1067         td = atmostone(f)
1068         if td.startswith('/'): raise Unspported(f.lno,
1069                 'Tests-Directory may not be absolute')
1070         f.base['testsdir'] = td
1071
1072 def run_tests(stanzas, tree):
1073         global errorcode, testbed
1074         if stanzas == ():
1075                 report('*', 'SKIP no tests in this package')
1076                 errorcode |= 8
1077         for stanza in stanzas:
1078                 tests = stanza[' tests']
1079                 if not tests:
1080                         report('*', 'SKIP package has metadata but no tests')
1081                         errorcode |= 8
1082                 for t in tests:
1083                         t.prepare()
1084                         t.run(tree)
1085                         if 'breaks-testbed' in t.restriction_names:
1086                                 testbed.needs_reset()
1087                 testbed.needs_reset()
1088
1089 class Test:
1090  def __init__(t, tname, base, act):
1091         if '/' in tname: raise Unsupported(base[' lno'],
1092                 'test name may not contain / character')
1093         for k in base: setattr(t,k,base[k])
1094         t.tname = tname
1095         t.act = act
1096         t.what = act.what+'t-'+tname
1097         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1098         else: t.path = tname
1099         t._debug('constructed; path=%s' % t.path)
1100         t._debug(' .depends=%s' % t.depends)
1101  def _debug(t, m):
1102         debug('& %s: %s' % (t.what, m))
1103  def report(t, m):
1104         report(t.what, m)
1105  def reportfail(t, m):
1106         global errorcode
1107         errorcode |= 4
1108         report(t.what, 'FAIL ' + m)
1109  def prepare(t):
1110         t._debug('preparing')
1111         dn = []
1112         for d in t.depends:
1113                 t._debug(' processing dependency '+d)
1114                 if not '@' in d:
1115                         t._debug('  literal dependency '+d)
1116                         dn.append(d)
1117                 else:
1118                         for (pkg,bin) in t.act.binaries:
1119                                 d = d.replace('@',pkg)
1120                                 t._debug('  synthesised dependency '+d)
1121                                 dn.append(d)
1122         testbed.prepare(dn)
1123  def run(t, tree):
1124         t._debug('[----------------------------------------')
1125         def stdouterr(oe):
1126                 idstr = t.what + '-' + oe
1127                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1128                         use_dir = opts.output_dir
1129                 else:
1130                         use_dir = testbed.scratch
1131                 return RelativeOutputFile(idstr, use_dir, idstr)
1132
1133         if hasattr(t.act,'work'): t.act.work.write(True)
1134         tree.read(True)
1135
1136         af = RelativeInputFile(t.what, tree, t.path)
1137         so = stdouterr('stdout')
1138         se = stdouterr('stderr')
1139
1140         tf = af.read(True)
1141         tmpdir = None
1142
1143         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1144         if rc: bomb('failed to chmod +x %s' % tf)
1145
1146         if 'needs-root' not in t.restriction_names and opts.user is not None:
1147                 tfl = ['su',opts.user,'-c',tf]
1148                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1149                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1150                 if opts.user:
1151                         script += '; chown %s "$1"' % opts.user
1152                         if 'rw-build-tree' in t.restriction_names:
1153                                 script += '; chown -R %s "$2"' % opts.user
1154                 rc = testbed.execute('mktmpdir-'+t.what,
1155                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1156                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1157                                 % (tmpdir, rc))
1158         else:
1159                 tfl = [tf]
1160
1161         rc = testbed.execute('test-'+t.what, tfl,
1162                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1163                 tmpdir=tmpdir, kind='test')
1164
1165         so_read = so.read()
1166         se_read = se.read()
1167
1168         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1169         stab = os.stat(se_read)
1170         if stab.st_size != 0:
1171                 l = open(se_read).readline()
1172                 l = l.rstrip('\n \t\r')
1173                 if len(l) > 35: l = l[:35] + '...'
1174                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1175                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1176                 debug_file(se_read)
1177         elif rc != 0:
1178                 t.reportfail('non-zero exit status %d' % rc)
1179         else:
1180                 t.report('PASS')
1181
1182         stab = os.stat(so_read)
1183         if stab.st_size != 0:
1184                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1185                 debug_file(so_read)
1186
1187         t._debug('----------------------------------------]')
1188
1189 def read_control(act, tree, control_override):
1190         stanzas = [ ]
1191
1192         if control_override is not None:
1193                 control_af = control_override
1194                 testbed.blame('arg:'+control_override.spec)
1195         else:
1196                 if act.missing_tests_control:
1197                         return ()
1198                 control_af = RelativeInputFile(act.what+'-testcontrol',
1199                         tree, 'debian/tests/control')
1200         try:
1201                 control = open(control_af.read(), 'r')
1202         except (IOError,OSError), oe:
1203                 if oe[0] != errno.ENOENT: raise
1204                 return []
1205
1206         lno = 0
1207         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1208         stz = None      # stz[field_name][index] = (lno, value)
1209                         # special field names:
1210                         # stz[' lno'] = number
1211                         # stz[' tests'] = list of Test objects
1212         def end_stanza(stz):
1213                 if stz is None: return
1214                 stz[' errs'] = 0
1215                 stanzas.append(stz)
1216                 stz = None
1217                 hcurrent = None
1218
1219         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1220         while 1:
1221                 l = control.readline()
1222                 if not l: break
1223                 lno += 1
1224                 if not l.endswith('\n'): badctrl('unterminated line')
1225                 if regexp.compile('\s*\#').match(l): continue
1226                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1227                 initmat = initre.match(l)
1228                 if initmat:
1229                         (fname, l) = initmat.groups()
1230                         fname = string.capwords(fname)
1231                         if stz is None:
1232                                 stz = { ' lno': lno, ' tests': [] }
1233                         if not stz.has_key(fname): stz[fname] = [ ]
1234                         hcurrent = stz[fname]
1235                 elif regexp.compile('\s').match(l):
1236                         if not hcurrent: badctrl('unexpected continuation')
1237                 else:
1238                         badctrl('syntax error')
1239                 hcurrent.append((lno, l))
1240         end_stanza(stz)
1241
1242         def testbadctrl(stz, lno, m):
1243                 report_badctrl(lno, m)
1244                 stz[' errs'] += 1
1245
1246         for stz in stanzas:
1247                 try:
1248                         try: tnames = stz['Tests']
1249                         except KeyError:
1250                                 tnames = ['*']
1251                                 raise Unsupported(stz[' lno'],
1252                                         'no Tests field')
1253                         tnames = map((lambda lt: lt[1]), tnames)
1254                         tnames = string.join(tnames).split()
1255                         base = {
1256                                 'restriction_names': [],
1257                                 'restrictions': [],
1258                                 'feature_names': [],
1259                                 'features': [],
1260                                 'testsdir': 'debian/tests',
1261                                 'depends' : '@'
1262                         }
1263                         for fname in stz.keys():
1264                                 if fname.startswith(' '): continue
1265                                 vl = stz[fname]
1266                                 try: fclass = globals()['Field_'+
1267                                         fname.replace('-','_')]
1268                                 except KeyError: raise Unsupported(vl[0][0],
1269                                         'unknown metadata field %s' % fname)
1270                                 f = fclass(stz, fname, base, tnames, vl)
1271                                 f.parse()
1272                         for tname in tnames:
1273                                 t = Test(tname, base, act)
1274                                 stz[' tests'].append(t)
1275                 except Unsupported, u:
1276                         for tname in tnames: u.report(tname)
1277                         continue
1278
1279         return stanzas
1280
1281 def print_exception(ei, msgprefix=''):
1282         if msgprefix: pstderr(msgprefix)
1283         (et, q, tb) = ei
1284         if et is Quit:
1285                 pstderr('adt-run: ' + q.m)
1286                 psummary('quitting: '+q.m)
1287                 return q.ec
1288         else:
1289                 pstderr("adt-run: unexpected, exceptional, error:")
1290                 psummary('quitting: unexpected error, consult transcript')
1291                 traceback.print_exc(None, sys.stderr)
1292                 if trace_stream is not None:
1293                         traceback.print_exc(None, trace_stream)
1294                 return 20
1295
1296 def cleanup():
1297         global trace_stream
1298         try:
1299                 if testbed is not None:
1300                         testbed.reset_apt()
1301                         testbed.stop()
1302                 if opts.tmpdir is None and tmpdir is not None:
1303                         rmtree('tmpdir', tmpdir)
1304                 if trace_stream is not None:
1305                         trace_stream.close()
1306                         trace_stream = None
1307         except:
1308                 print_exception(sys.exc_info(),
1309                         '\nadt-run: error cleaning up:\n')
1310                 os._exit(20)
1311
1312 #---------- registration, installation etc. of .deb's: Binaries
1313
1314 def determine_package(act):
1315         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1316         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1317         if rc: badpkg('failed to parse binary package, code %d' % rc)
1318         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1319         act.pkg = None
1320         for l in output.split('\n'):
1321                 m = re.match(l)
1322                 if not m: continue
1323                 if act.pkg: badpkg('two Package: lines in control file')
1324                 act.pkg = m.groups()[0]
1325         if not act.pkg: badpkg('no good Package: line in control file')
1326
1327 class Binaries:
1328  def __init__(b, tb):
1329         b.dir = TemporaryDir('binaries')
1330         b.dir.write()
1331         ok = False
1332
1333         if opts.gnupghome is None:
1334                 opts.gnupghome = tmpdir+'/gnupg'
1335
1336         b._debug('initialising')
1337         try:
1338                 for x in ['pubring','secring']:
1339                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1340                 ok = True
1341         except (IOError,OSError), oe:
1342                 if oe.errno != errno.ENOENT: raise
1343
1344         if ok: b._debug('no key generation needed')
1345         else: b.genkey()
1346
1347  def _debug(b, s):
1348         debug('* '+s)
1349
1350  def genkey(b):
1351         b._debug('preparing for key generation')
1352
1353         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1354         mkdir_okexist(opts.gnupghome, 0700)
1355
1356         script = '''
1357   exec >&2
1358   cd "$1"
1359   cat <<"END" >key-gen-params
1360 Key-Type: DSA
1361 Key-Length: 1024
1362 Key-Usage: sign
1363 Name-Real: autopkgtest per-run key
1364 Name-Comment: do not trust this key
1365 Name-Email: autopkgtest@example.com
1366 END
1367   set -x
1368   gpg --homedir="$1" --batch --gen-key key-gen-params
1369                 '''
1370         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1371         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1372         if rc: bomb('key generation failed, code %d' % rc)
1373
1374  def apt_configs(b):
1375         return {
1376                 "Debug::pkgProblemResolver": "true",
1377                 "APT::Get::force-yes" : "true",
1378                 "APT::Get::Assume-Yes" : "true",
1379                 "quiet" : "true",
1380         }
1381
1382  def _configure_apt(b, tb):
1383         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1384                         True)
1385         f = open(config.write(),'w')
1386         for (k,v) in b.apt_configs().iteritems():
1387                 print >>f, '%s { "%s"; };' % (k, v)
1388         f.close()
1389         config.read(True)
1390
1391         prefs = OutputFile('apt-prefs','/etc/apt/preferences.d/90autopkgtest',
1392                         True)
1393         print >>open(prefs.write(),'w'), '''
1394 Package: *
1395 Pin: origin ""
1396 Pin-Priority: 1002
1397 '''
1398         prefs.read(True)
1399
1400  def _apt_get(b):
1401         ag = ['apt-get','-q']
1402         for kv in b.apt_configs().iteritems():
1403                 ag += ['-o', '%s=%s' % kv]
1404         return ' '.join(ag)
1405
1406  def reset(b):
1407         b._debug('reset')
1408         rmtree('binaries', b.dir.read())
1409         b.dir.invalidate()
1410         b.dir.write()
1411         b.install = []
1412         b.blamed = []
1413         b.registered = set()
1414
1415  def register(b, act, pkg, af, forwhat, blamed):
1416         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1417                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1418
1419         if act.ah['deb_'+forwhat] == 'ignore': return
1420
1421         b.blamed += testbed.blamed
1422
1423         leafname = pkg+'.deb'
1424         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1425
1426         try: os.remove(dest.write())
1427         except (IOError,OSError), oe:
1428                 if oe.errno != errno.ENOENT: raise e
1429
1430         try: os.link(af.read(), dest.write())
1431         except (IOError,OSError), oe:
1432                 if oe.errno != errno.EXDEV: raise e
1433                 shutil.copy(af.read(), dest)
1434
1435         if act.ah['deb_'+forwhat] == 'install':
1436                 b.install.append(pkg)
1437
1438         b.registered.add(pkg)
1439
1440  def publish(b):
1441         b._debug('publish')
1442
1443         b._configure_apt(testbed)
1444
1445         script = '''
1446   exec >&2
1447   cd "$1"
1448   apt-ftparchive packages . >Packages
1449   gzip <Packages >Packages.gz
1450   apt-ftparchive release . >Release
1451   rm -f Release.gpg
1452   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1453   gpg --homedir="$2" --batch --export >archive-key.pgp
1454                 '''
1455         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1456         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1457         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1458
1459         b.dir.invalidate(True)
1460         apt_source = b.dir.read(True)
1461
1462         so = TemporaryFile('vlds')
1463         script = '''
1464   exec 3>&1 >&2
1465   apt-key add archive-key.pgp
1466   echo "deb file://'''+apt_source+''' /" >/etc/apt/sources.list.d/autopkgtest.list
1467   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1468     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1469   fi
1470   '''+ b._apt_get() +''' update >&2
1471   cat /var/lib/dpkg/status >&3
1472                 '''
1473         testbed.mungeing_apt()
1474         rc = testbed.execute('apt-key', ['sh','-ec',script],
1475                                 so=so.write(True), cwd=b.dir.write(True),
1476                                 script=script, kind='install')
1477         if rc: bomb('apt setup failed with exit code %d' % rc)
1478
1479         testbed.blamed += b.blamed
1480
1481         b._debug('publish reinstall checking...')
1482         pkgs_reinstall = set()
1483         pkg = None
1484         for l in open(so.read()):
1485                 if l.startswith('Package: '):
1486                         pkg = l[9:].rstrip()
1487                 elif l.startswith('Status: install '):
1488                         if pkg in b.registered:
1489                                 pkgs_reinstall.add(pkg)
1490                                 b._debug(' publish reinstall needs '+pkg)
1491
1492         if pkgs_reinstall:
1493                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1494                 what = 'apt-get-reinstall'
1495                 cmdl = (b._apt_get() + ' --reinstall install '+
1496                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1497                 cmdl = ['sh','-c',cmdl]
1498                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1499                 if rc: badpkg("installation of basic binarries failed,"
1500                                 " exit code %d" % rc)
1501
1502         b._debug('publish install...')
1503         for pkg in b.install:
1504                 what = 'apt-get-install-%s' % pkg
1505                 testbed.blame(pkg)
1506                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1507                 cmdl = ['sh','-c',cmdl]
1508                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1509                 if rc: badpkg("installation of %s failed, exit code %d"
1510                                 % (pkg, rc))
1511
1512         b._debug('publish done')
1513
1514 #---------- processing of sources (building)
1515
1516 def source_rules_command(act,script,what,which,work,cwd,
1517                                 results_lines=0,xargs=[]):
1518         script = [      "exec 3>&1 >&2",
1519                         "set -x"        ] + script
1520         script = '\n'.join(script)
1521         so = TemporaryFile('%s-%s-results' % (what,which))
1522         rc = testbed.execute('%s-%s' % (what,which),
1523                         ['sh','-ec',script]+xargs, script=script,
1524                         so=so.write(True), cwd=cwd, kind='build')
1525         results = open(so.read()).read().rstrip('\n')
1526         if len(results): results = results.split("\n")
1527         else: results = []
1528         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1529         if results_lines is not None and len(results) != results_lines:
1530                 badpkg("got %d lines of results from %s where %d expected"
1531                         % (len(results), which, results_lines))
1532         if results_lines==1: return results[0]
1533         return results
1534
1535 def build_source(act, control_override):
1536         act.blame = 'arg:'+act.af.spec
1537         testbed.blame(act.blame)
1538         testbed.prepare1([])
1539         testbed.needs_reset()
1540
1541         what = act.what
1542         basename = act.af.spec
1543         debiancontrol = None
1544         act.binaries = []
1545
1546         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1547
1548         if act.kind == 'dsc':
1549                 dsc = act.af
1550                 dsc_file = open(dsc.read())
1551                 in_files = False
1552                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1553                 for l in dsc_file:
1554                         l = l.rstrip('\n')
1555                         if l.startswith('Files:'): in_files = True; continue
1556                         elif l.startswith('#'): pass
1557                         elif not l.startswith(' '):
1558                                 in_files = False
1559                                 if l.startswith('Source:'):
1560                                         act.blame = 'dsc:'+l[7:].strip()
1561                                         testbed.blame(act.blame)
1562                         if not in_files: continue
1563
1564                         m = fre.match(l)
1565                         if not m: badpkg(".dsc contains unparseable line"
1566                                         " in Files: `%s'" % l)
1567                         leaf = m.groups(0)[0]
1568                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1569                                         sibling=True)
1570                         subfile.read(True)
1571                 dsc.read(True)
1572
1573         if act.kind == 'ubtree':
1574                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1575                         act.af, 'debian/control')
1576                 dsc = TemporaryFile(what+'-fakedsc')
1577                 dsc_w = open(dsc.write(), 'w')
1578                 for l in open(debiancontrol.read()):
1579                         l = l.rstrip('\n')
1580                         if not len(l): break
1581                         print >>dsc_w, l
1582                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1583                 dsc_w.close()
1584
1585         if act.kind == 'dsc':
1586                 testbed.prepare2([])
1587                 testbed.satisfy_dependencies_string('dpkg-dev',
1588                                                 'install dpkg-dev')
1589
1590         work = TemporaryDir(what+'-build')
1591         act.work = work
1592
1593         tmpdir = work.write(True)+'/tmpdir'
1594         tmpdir_script = [
1595                         'TMPDIR="$1"',
1596                         'rm -rf -- "$TMPDIR"',
1597                         'export TMPDIR',
1598                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1599                 ]
1600
1601         if act.kind == 'ubtree':
1602                 spec = '%s/real-tree' % work.write(True)
1603                 create_command = '''
1604                         rm -rf "$spec"
1605                         mkdir "$spec"
1606                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1607                         '''
1608                 initcwd = act.af.read(True)
1609
1610         if act.kind == 'dsc':
1611                 spec = dsc.read(True)
1612                 create_command = '''
1613                         dpkg-source -x $spec
1614                         '''
1615                 initcwd = work.write(True)
1616
1617         script = [
1618                 'spec="$2"',
1619                 'origpwd=`pwd`',
1620                 'cd '+work.write(True)
1621         ]
1622
1623         if opts.user:
1624                 script += ([ 'chown '+opts.user+' .' ] +
1625                         tmpdir_script +
1626                         [ 'spec="$spec" origpwd="$origpwd" '
1627                                 +opts.user_wrap(create_command) ])
1628         else:
1629                 script += (tmpdir_script +
1630                         [ create_command ])
1631
1632         script += [
1633                         'cd [a-z0-9]*-*/.',
1634                         'pwd >&3',
1635                         'set +e; test -f debian/tests/control; echo $? >&3'
1636                 ]
1637         (result_pwd, control_test_rc) = source_rules_command(
1638                         act,script,what,'extract',work,
1639                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1640
1641         filter = act.ah['dsc_filter']
1642
1643         if control_test_rc == '1': act.missing_tests_control = True
1644
1645         # For optional builds:
1646         #
1647         # We might need to build the package because:
1648         #   - we want its binaries (filter isn't _ and at least one of the
1649         #       deb_... isn't ignore)
1650         #   - the test control file says so
1651         #       (assuming we have any tests)
1652
1653         class NeedBuildException: pass
1654         def build_needed(m):
1655                 debug_b('build needed for %s' % m)
1656                 raise NeedBuildException()
1657
1658         try:
1659                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1660                                       act.ah['deb_fortests'] != 'ignore'):
1661                         build_needed('binaries')
1662
1663                 result_pwd_af = InputDir(what+'-treeforcontrol',
1664                         result_pwd, True)
1665                 stanzas = read_control(act, result_pwd_af, control_override)
1666                 for stanza in stanzas:
1667                         for t in stanza[' tests']:
1668                                 if 'build-needed' in t.restriction_names:
1669                                         build_needed('test %s' % t.tname)
1670
1671                 debug_b('build not needed')
1672                 built = False
1673
1674         except NeedBuildException:
1675
1676                 if act.kind != 'dsc':
1677                         testbed.prepare2([])
1678
1679                 testbed.satisfy_dependencies_string('build-essential',
1680                                 'install build-essential')
1681                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1682
1683                 script = tmpdir_script + [
1684                         'cd "$2"',
1685                         'dpkg-checkbuilddeps',
1686                         opts.user_wrap('debian/rules build'),
1687                         ]
1688                 source_rules_command(act,script,what,'build',work,
1689                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1690
1691                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1692                         badpkg("results dir `%s' is not in expected parent"
1693                                " dir `%s'" % (result_pwd, work.read(True)))
1694
1695                 built = True
1696
1697         act.tests_tree = RelativeInputDir(what+'-tests-tree',
1698                                 work, os.path.basename(result_pwd),
1699                                 True)
1700
1701         if not built:
1702                 act.blamed = []
1703                 return
1704
1705         act.blamed = copy.copy(testbed.blamed)
1706
1707         debug_b('filter=%s' % filter)
1708         if filter != '_':
1709                 script = tmpdir_script + [
1710                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1711                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1712                         'cd ..',
1713                         'echo *.deb >&3',
1714                         ]
1715                 result_debs = source_rules_command(act,script,what,
1716                                 'binary',work,work.write(True),
1717                                 results_lines=1, xargs=['x',tmpdir])
1718                 if result_debs == '*.deb': debs = []
1719                 else: debs = result_debs.split(' ')
1720                 debug_b('debs='+`debs`)
1721                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1722                 for deb in debs:
1723                         m = re.match(deb)
1724                         if not m: badpkg("badly-named binary `%s'" % deb)
1725                         pkg = m.groups()[0]
1726                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1727                         for pat in filter.split(','):
1728                                 debug_b('  pat=%s' % pat)
1729                                 if not fnmatch.fnmatchcase(pkg,pat):
1730                                         debug_b('   no match')
1731                                         continue
1732                                 deb_what = pkg+'_'+what+'.deb'
1733                                 bin = RelativeInputFile(deb_what,work,deb,True)
1734                                 debug_b('  deb_what=%s, bin=%s' %
1735                                         (deb_what, str(bin)))
1736                                 binaries.register(act,pkg,bin,
1737                                         'forbuilds',testbed.blamed)
1738                                 act.binaries.append((pkg,bin))
1739                                 break
1740                 debug_b('all done.')
1741
1742 #---------- main processing loop and main program
1743
1744 def process_actions():
1745         global binaries
1746
1747         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1748         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1749         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1750
1751         debug_a1('starting')
1752         testbed.open()
1753         binaries = Binaries(testbed)
1754
1755         binaries.reset()
1756         control_override = None
1757
1758         debug_a1('builds ...')
1759         for act in opts.actions:
1760                 debug_a2('%s %s' %
1761                         (act.kind, act.what))
1762
1763                 if act.kind == 'control':
1764                         control_override = act.af
1765                 if act.kind == 'deb':
1766                         testbed.blame('arg:'+act.af.spec)
1767                         determine_package(act)
1768                         testbed.blame('deb:'+act.pkg)
1769                         binaries.register(act,act.pkg,act.af,
1770                                 'forbuilds',testbed.blamed)
1771                 if act.kind == 'dsc' or act.kind == 'ubtree':
1772                         build_source(act, control_override)
1773                 if act.kind == 'tree':
1774                         act.binaries = []
1775                 if act.kind.endswith('tree') or act.kind == 'dsc':
1776                         control_override = None
1777                 if act.kind == 'instantiate':
1778                         pass
1779
1780         debug_a1('builds done.')
1781
1782         binaries.reset()
1783         control_override = None
1784
1785         debug_a1('tests ...')
1786         for act in opts.actions:
1787                 debug_a2('test %s %s' % (act.kind, act.what))
1788
1789                 testbed.needs_reset()
1790                 if act.kind == 'control':
1791                         control_override = act.af
1792                 if act.kind == 'deb':
1793                         binaries.register(act,act.pkg,act.af,'fortests',
1794                                 ['deb:'+act.pkg])
1795                 if act.kind == 'dsc' or act.kind == 'ubtree':
1796                         for (pkg,bin) in act.binaries:
1797                                 binaries.register(act,pkg,bin,'fortests',
1798                                         act.blamed)
1799                 if act.kind == 'dsc':
1800                         if act.ah['dsc_tests']:
1801                                 debug_a3('read control ...')
1802                                 stanzas = read_control(act, act.tests_tree,
1803                                                 control_override)
1804                                 testbed.blamed += act.blamed
1805                                 debug_a3('run_tests ...')
1806                                 run_tests(stanzas, act.tests_tree)
1807                         control_override = None
1808                 if act.kind == 'tree' or act.kind == 'ubtree':
1809                         testbed.blame('arg:'+act.af.spec)
1810                         stanzas = read_control(act, act.af, control_override)
1811                         debug_a3('run_tests ...')
1812                         run_tests(stanzas, act.af)
1813                         control_override = None
1814                 if act.kind == 'instantiate':
1815                         testbed.prepare([])
1816         debug_a1('tests done.')
1817
1818 def main():
1819         global testbed
1820         global tmpdir
1821         try:
1822                 parse_args()
1823         except SystemExit, se:
1824                 os._exit(20)
1825         try:
1826                 setup_trace()
1827                 testbed = Testbed()
1828                 testbed.start()
1829                 finalise_options()
1830                 process_actions()
1831         except:
1832                 ec = print_exception(sys.exc_info(), '')
1833                 cleanup()
1834                 os._exit(ec)
1835         cleanup()
1836         os._exit(errorcode)
1837
1838 main()