runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51 paths = []
  52
  53 #---------- output handling
  54 #
  55 #  There are at least the following kinds of output:
  56 #
  57 #   1. stderr output which consists of
  58 #   1a. our errors
  59 #   1b. stuff printed to stderr by the virtualisation server
  60 #   1c. stuff printed to stderr by our short-lived subprocesseses
  61 #           which we don't expect to fail
  62 #
  63 #   2. trace information, which consists of
  64 #   2a. our trace information from calls to debug()
  65 #   2b. progress information and stderr output from
  66 #        general scripts we run on the host
  67 #   2c. progress information and stderr output from things
  68 #        we run on the testbed including builds
  69 #   2d. stderr and stdout output from actual tests
  70 #
  71 #   xxxx
  72 #   3. actual test results (printed to our stdout)
  73 #
  74 # Cloning of 1a and 2a, where necessary, is done by us writing
  75 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  76 # is done by forking off a copy of ourselves to do plumbing,
  77 # which copy we wait for at the appropriate point.
  78
  79 class DummyOpts:
  80  def __init__(do): do.debuglevel = 0
  81
  82 opts = DummyOpts()
  83 trace_stream = None
  84 summary_stream = None
  85
  86 def pstderr(m):
  87         print >>sys.stderr, m
  88         if trace_stream is not None: print >>trace_stream, m
  89
  90 def debug(m, minlevel=0):
  91         if opts.debuglevel < minlevel: return
  92         if opts.quiet and trace_stream is None: return
  93         p = 'adt-run: trace'
  94         if minlevel: p += `minlevel`
  95         p += ': '
  96         for l in m.rstrip('\n').split('\n'):
  97                 s = p + l
  98                 if not opts.quiet: print >>sys.stderr, s
  99                 if trace_stream is not None: print >>trace_stream, s
 100
 101 def debug_file(hp, minlevel=0):
 102         if opts.debuglevel < minlevel: return
 103         def do_copy(stream, what):
 104                 rc = subprocess.call(['cat',hp], stdout=stream)
 105                 if rc: bomb('cat failed copying data from %s'
 106                             ' to %s, exit code %d' % (hp, what, rc))
 107         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 108         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 109
 110 class Errplumb:
 111  def __init__(ep, critical=False):
 112         to_stderr = critical or not opts.quiet
 113         count = to_stderr + (trace_stream is not None)
 114         if count == 0:
 115                 ep.stream = open('/dev/null','w')
 116                 ep._sp = None
 117         elif count == 1:
 118                 if to_stderr: ep.stream = os.dup(2)
 119                 else: ep.stream = trace_stream
 120                 ep._sp = None
 121         else:
 122                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 123                         stdin=subprocess.PIPE, stdout=trace_stream,
 124                         close_fds=True)
 125                 ep.stream = ep._sp.stdin
 126  def wait(ep):
 127         if ep._sp is None: return
 128         if type(ep.stream) == type(2):
 129                 os.close(ep.stream)
 130                 ep.stream = ()
 131         ep._sp.stdin.close()
 132         rc = ep._sp.wait()
 133         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 134         ep._sp = None
 135
 136 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 137         if dbg is not None:
 138                 if isinstance(dbg,tuple): (what,script) = dbg
 139                 else: (what,script) = (dbg,None)
 140                 debug_subprocess(what, cmdl, script=script)
 141         ep = Errplumb(critical)
 142         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 143         output = running.communicate()[0]
 144         rc = running.wait()
 145         ep.wait()
 146         return (rc, output)
 147
 148 def psummary(m):
 149         if summary_stream is not None: print >>summary_stream, m
 150
 151 def preport(m):
 152         print m
 153         sys.stdout.flush()
 154         if trace_stream is not None: print >>trace_stream, m
 155         psummary(m)
 156
 157 def report(tname, result):
 158         preport('%-20s %s' % (tname, result))
 159
 160 #---------- errors we define
 161
 162 class Quit:
 163         def __init__(q,ec,m): q.ec = ec; q.m = m
 164
 165 def bomb(m):
 166         raise Quit(20, "unexpected error: %s" % m)
 167
 168 def badpkg(m):
 169         preport('blame: ' + ' '.join(testbed.blamed))
 170         preport('badpkg: ' + m)
 171         raise Quit(12, "erroneous package: %s" % m)
 172
 173 class Unsupported:
 174  def __init__(u, lno, m):
 175         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 176         else: u.m = m
 177  def report(u, tname):
 178         global errorcode
 179         errorcode != 2
 180         report(tname, 'SKIP %s' % u.m)
 181
 182 #---------- convenience function
 183
 184 def mkdir_okexist(pathname, mode=02755):
 185         try:
 186                 os.mkdir(pathname, mode)
 187         except (IOError,OSError), oe:
 188                 if oe.errno != errno.EEXIST: raise
 189
 190 def rmtree(what, pathname):
 191         debug('/ %s rmtree %s' % (what, pathname), 2)
 192         try: shutil.rmtree(pathname)
 193         except (IOError,OSError), oe:
 194                 if oe.errno != errno.EEXIST: raise
 195
 196 def debug_subprocess(what, cmdl=None, script=None):
 197         o = '$ '+what+':'
 198         if cmdl is not None:
 199                 ol = []
 200                 for x in cmdl:
 201                         if x is script: x = '<SCRIPT>'
 202                         ol.append(x.    replace('\\','\\\\').
 203                                         replace(' ','\\ ')      )
 204                 o += ' '+ ' '.join(ol)
 205         debug(o)
 206         if script is not None and opts.debuglevel >= 1:
 207                 o = ''
 208                 for l in script.rstrip('\n').split('\n'):
 209                         o += '$     '+l+'\n'
 210                 debug(o,1)
 211
 212 def flatten(l):
 213         return reduce((lambda a,b: a + b), l, [])
 214
 215 #---------- fancy automatic file-copying class
 216
 217 class AutoFile:
 218         # p.what
 219         # p.path[tb]    None or path    not None => path known
 220         # p.file[tb]    None or path    not None => file exists
 221         # p.ephem[tb]   boolean         True => destroyed by tb reset
 222         # p.spec        string or None
 223         # p.spec_tbp    True or False, or not set if spec is None
 224         # p.dir         '' or '/'
 225
 226  def __init__(p, what):
 227         p.what = what
 228         p.path = [None,None]
 229         p.file = [None,None]
 230         p.ephem = [False,False]
 231         p.spec = None
 232         p.spec_tbp = None
 233         p.dir = ''
 234
 235  def __repr__(p):
 236         return "<AF@%s>" % p.__str__()
 237  def __str__(p):
 238         def ptbp(tbp):
 239                 if p.path[tbp] is None: return '-'+p.dir
 240                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 241                 else: return p.path[tbp]+p.dir+'!'
 242         out = p.what
 243         if p.spec is not None:
 244                 if not hasattr(p,'spec_tb'): out += '~'
 245                 elif p.spec_tbp: out += '#'
 246                 else: out += '='
 247                 out += p.spec
 248         out += ':'
 249         out += ptbp(False)
 250         out += '|'
 251         out += ptbp(True)
 252         return out
 253
 254  def _wrong(p, how):
 255         xtra = ''
 256         if p.spec is not None:
 257                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 258         raise ("internal error: %s (%s)" % (how, str(p)))
 259
 260  def _ensure_path(p, tbp):
 261         if p.path[tbp] is None:
 262                 if '/' in p.what:
 263                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 264                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 265                 if not tbp:
 266                         p.path[tbp] = tmpdir+'/'+p.what
 267                 else:
 268                         p.ephem[tbp] = True
 269                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 270
 271  def write(p, tbp=False):
 272         p._debug('write %s...' % 'HT'[tbp])
 273         p._ensure_path(tbp)
 274
 275         if p.dir and not p.file[tbp]:
 276                 if not tbp:
 277                         p._debug('mkdir H')
 278                         mkdir_okexist(p.path[tbp])
 279                 else:
 280                         cmdl = ['sh','-ec',
 281                                 'test -d "$1" || mkdir -p "$1"',
 282                                 'x', p.path[tbp]]
 283                         tf_what = urllib.quote(p.what).replace('/','%2F')
 284                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 285                         if rc: bomb('failed to create directory %s' %
 286                                 p.path[tbp])
 287
 288         p.file[tbp] = p.path[tbp]
 289         return p.path[tbp]
 290
 291  def read(p, tbp=False):
 292         p._debug('read %s...' % 'HT'[tbp])
 293         p._ensure_path(tbp)
 294
 295         if p.file[tbp] is None:
 296                 if p.file[not tbp] is None:
 297                         p._wrong("requesting read but nonexistent")
 298                 cud = ['copyup','copydown'][tbp]
 299                 src = p.file[not tbp] + p.dir
 300                 dst = p.path[tbp] + p.dir
 301                 testbed.command(cud, (src, dst))
 302                 p.file[tbp] = p.path[tbp]
 303
 304         return p.file[tbp] + p.dir
 305
 306  def invalidate(p, tbp=False):
 307         if p.path[tbp] is not None:
 308                 p._debug('invalidating %s' % 'HT'[tbp])
 309         p.file[tbp] = None
 310         if p.spec_tbp != tbp or p.spec is None:
 311                 p.path[tbp] = None
 312
 313  def invalidate_ephem(p, tbp=False):
 314         if p.ephem[tbp]:
 315                 p.invalidate(tbp)
 316
 317  def _debug(p, m):
 318         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 319
 320  def _constructed(p):
 321         p._debug('constructed: '+str(p))
 322         p._check()
 323         paths.append(p)
 324
 325  def _check(p):
 326         for tbp in [False,True]:
 327          for pf in [p.path, p.file]:
 328                 if pf[tbp] is None: continue
 329                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 330                 if p.dir and pf[tbp].endswith('/'):
 331                         pf[tbp] = pf[tbp].rstrip('/')
 332                         if not pf[tbp]: pf[tbp] = '/'
 333                 if not p.dir and pf[tbp].endswith('/'):
 334                         bomb("directory `%s' specified for "
 335                              "non-directory %s" % (pf[tbp], p.what))
 336
 337  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 338         AutoFile.__init__(p,what)
 339         sh_on = ''; sh_sibl = ''
 340         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 341         if sibling: sh_sibl=' (sibling)'
 342         parent._debug('using as base: %s: %s%s%s' %
 343                         (str(parent), leaf, sh_on, sh_sibl))
 344         if not sibling and not parent.dir:
 345                 parent._wrong('asked for non-sibling relative path of non-dir')
 346         if sibling: trim = os.path.dirname
 347         else: trim = lambda x: x
 348         for tbp in [False,True]:
 349                 p.ephem[tbp] = parent.ephem[tbp]
 350                 if parent.path[tbp] is None: continue
 351                 trimmed = trim(parent.path[tbp])
 352                 if trimmed: trimmed += '/'
 353                 p.path[tbp] = trimmed + leaf
 354                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 355                         p.file[tbp] = p.path[tbp]
 356
 357 class InputFile(AutoFile):
 358  def _init(p, what, spec, spec_tbp=False):
 359         AutoFile.__init__(p, what)
 360         p.spec = spec
 361         p.spec_tbp = spec_tbp
 362         p.path[spec_tbp] = spec
 363         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 364  def __init__(p, what, spec, spec_tbp=False):
 365         p._init(what,spec,spec_tbp)
 366         p._constructed()
 367
 368 class InputDir(InputFile):
 369  def __init__(p, what, spec, spec_tbp=False):
 370         InputFile._init(p,what,spec,spec_tbp)
 371         p.dir = '/'
 372         p._constructed()
 373
 374 class OutputFile(AutoFile):
 375  def _init(p, what, spec, spec_tbp=False):
 376         AutoFile.__init__(p, what)
 377         p.spec = spec
 378         p.spec_tbp = spec_tbp
 379         p.path[spec_tbp] = spec
 380  def __init__(p, what, spec, spec_tbp=False):
 381         p._init(what,spec,spec_tbp)
 382         p._constructed()
 383
 384 class OutputDir(OutputFile):
 385  def __init__(p, what, spec, spec_tbp=False):
 386         OutputFile._init(p,what,spec,spec_tbp)
 387         p.dir = '/'
 388         p._constructed()
 389
 390 class RelativeInputDir(AutoFile):
 391  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 392         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 393         p.dir = '/'
 394         p._constructed()
 395
 396 class RelativeInputFile(AutoFile):
 397  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 398         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 399         p._constructed()
 400
 401 class RelativeOutputFile(AutoFile):
 402  def __init__(p, what, parent, leaf, sibling=False):
 403         p._relative_init(what, parent, leaf, None, False, sibling)
 404         p._constructed()
 405
 406 class TemporaryFile(AutoFile):
 407  def __init__(p, what):
 408         AutoFile.__init__(p, what)
 409         p._constructed()
 410
 411 class TemporaryDir(AutoFile):
 412  def __init__(p, what):
 413         AutoFile.__init__(p, what)
 414         p.dir = '/'
 415         p._constructed()
 416
 417 #---------- parsing and representation of the arguments
 418
 419 class Action:
 420  def __init__(a, kind, af, arghandling, what):
 421         # extra attributes get added during processing
 422         a.kind = kind
 423         a.af = af
 424         a.ah = arghandling
 425         a.what = what
 426         a.missing_tests_control = False
 427  def __repr__(a):
 428         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 429
 430 def parse_args():
 431         global opts, timeouts
 432         global n_non_actions # argh, stupid python scoping rules
 433         usage = "%prog <options> --- <virt-server>..."
 434         parser = OptionParser(usage=usage)
 435         pa = parser.add_option
 436         pe = parser.add_option
 437
 438         arghandling = {
 439                 'dsc_tests': True,
 440                 'dsc_filter': '*',
 441                 'deb_forbuilds': 'auto',
 442                 'deb_fortests': 'auto',
 443                 'tb': False,
 444                 'override_control': None,
 445                 'set_lang': 'C'
 446         }
 447         initial_arghandling = arghandling.copy()
 448         n_non_actions = 0
 449
 450         #----------
 451         # actions (ie, test sets to run, sources to build, binaries to use):
 452
 453         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 454                 global n_non_actions
 455                 parser.largs.append((value,kindpath))
 456                 n_non_actions += not(is_act)
 457
 458         def pa_action(long, metavar, kindpath, help, is_act=True):
 459                 pa('','--'+long, action='callback', callback=cb_action,
 460                         nargs=1, type='string',
 461                         callback_args=(long,kindpath,is_act), help=help)
 462
 463         pa_action('built-tree',         'TREE', '@/',
 464                 help='run tests from build tree TREE')
 465
 466         pa_action('unbuilt-tree',       'TREE', '@//',
 467                 help='run tests from build tree TREE')
 468
 469         pa_action('source',             'DSC', '@.dsc',
 470                 help='build DSC and use its tests and/or'
 471                     ' generated binary packages')
 472
 473         pa_action('binary',             'DEB', '@.deb',
 474                help='use binary package DEB according'
 475                     ' to most recent --binaries-* settings')
 476
 477         def cb_actnoarg(op,optstr,value,parser, largsentry):
 478                 parser.largs.append(largsentry)
 479         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 480                 callback_args=((None, ('instantiate',)),),
 481                 help='instantiate testbed now (during testing phase)'
 482                      ' and install packages'
 483                      ' selected for automatic installation, even'
 484                      ' if this might apparently not be required otherwise')
 485
 486         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 487                help='run tests from control file CONTROL instead,'
 488                     ' (applies to next test suite only)')
 489
 490         #----------
 491         # argument handling settings (what ways to use action
 492         #  arguments, and pathname processing):
 493
 494         def cb_setah(option, opt_str, value, parser, toset,setval):
 495                 if type(setval) == list:
 496                         if not value in setval:
 497                                 parser.error('value for %s option (%s) is not '
 498                                  'one of the permitted values (%s)' %
 499                                  (value, opt_str, setval.join(' ')))
 500                 elif setval is not None:
 501                         value = setval
 502                 for v in toset:
 503                         arghandling[v] = value
 504                 parser.largs.append(arghandling.copy())
 505
 506         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 507                 type = metavar
 508                 if type is not None: type = 'string'
 509                 pa('',long, action='callback', callback=cb_setah,
 510                    callback_args=(affected,effect), **kwargs)
 511
 512         #---- paths: host or testbed:
 513         #
 514         pa_setah('--paths-testbed', ['tb'],True,
 515                 help='subsequent path specifications refer to the testbed')
 516         pa_setah('--paths-host', ['tb'],False,
 517                 help='subsequent path specifications refer to the host')
 518
 519         #---- source processing settings:
 520
 521         pa_setah('--sources-tests', ['dsc_tests'],True,
 522                 help='run tests from builds of subsequent sources')
 523         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 524                 help='do not run tests from builds of subsequent sources')
 525
 526         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 527                 type='string', metavar='PATTERN-LIST',
 528                 help='from subsequent sources, use binaries matching'
 529                      ' PATTERN-LIST (comma-separated glob patterns)'
 530                      ' according to most recent --binaries-* settings')
 531         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 532                 help='from subsequent sources, do not use any binaries')
 533         #---- binary package processing settings:
 534
 535         def pa_setahbins(long,toset,how):
 536          pa_setah(long, toset,['ignore','auto','install'],
 537                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 538                 help=how+' ignore binaries, install them as needed'
 539                         ' for dependencies, or unconditionally install'
 540                         ' them, respectively')
 541         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 542         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 543         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 544
 545         #----------
 546         # general options:
 547
 548         def cb_vserv(op,optstr,value,parser):
 549                 parser.values.vserver = list(parser.rargs)
 550                 del parser.rargs[:]
 551
 552         def cb_path(op,optstr,value,parser, constructor,long,dir):
 553                 name = long.replace('-','_')
 554                 af = constructor(long, value,arghandling['tb'])
 555                 setattr(parser.values, name, af)
 556
 557         def pa_path(long, constructor, help, dir=False):
 558                 pa('','--'+long, action='callback', callback=cb_path,
 559                         callback_args=(constructor,long,dir),
 560                         nargs=1, type='string',
 561                         help=help, metavar='PATH')
 562
 563         pa_path('output-dir', OutputDir, dir=True,
 564                 help='write stderr/out files in PATH')
 565
 566         pa('--leave-lang', dest='set_lang', action='store_false',
 567                 help="leave LANG on testbed set to testbed's default")
 568         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 569                 help='set LANG on testbed to LANGVAL', default='C')
 570
 571         pa('','--tmp-dir',              type='string', dest='tmpdir',
 572                 help='write temporary files to TMPDIR, emptying it'
 573                      ' beforehand and leaving it behind at the end')
 574         pa('','--log-file',             type='string', dest='logfile',
 575                 help='write the log LOGFILE, emptying it beforehand,'
 576                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 577         pa('','--summary-file',         type='string', dest='summary',
 578                 help='write a summary report to SUMMARY,'
 579                      ' emptying it beforehand')
 580
 581         for k in timeouts.keys():
 582                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 583                         metavar='T', help='set %s timeout to T')
 584         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 585                         metavar='FACTOR', default=1.0,
 586                         help='multiply all default timeouts by FACTOR')
 587
 588         pa('','--user',                 type='string', dest='user',
 589                 help='run tests as USER (needs root on testbed)')
 590         pa('','--gain-root',            type='string', dest='gainroot',
 591                 help='prefix debian/rules binary with GAINROOT')
 592         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 593         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 594         pa('','--gnupg-home',           type='string', dest='gnupghome',
 595                 default='~/.autopkgtest/gpg',
 596                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 597                         " signing private apt archive);"
 598                         " `fresh' means generate new key each time.")
 599
 600         #----------
 601         # actual meat:
 602
 603         class SpecialOption(optparse.Option): pass
 604         vs_op = SpecialOption('','--VSERVER-DUMMY')
 605         vs_op.action = 'callback'
 606         vs_op.type = None
 607         vs_op.default = None
 608         vs_op.nargs = 0
 609         vs_op.callback = cb_vserv
 610         vs_op.callback_args = ( )
 611         vs_op.callback_kwargs = { }
 612         vs_op.help = 'introduces virtualisation server and args'
 613         vs_op._short_opts = []
 614         vs_op._long_opts = ['---']
 615
 616         pa(vs_op)
 617
 618         (opts,args) = parser.parse_args()
 619         if not hasattr(opts,'vserver'):
 620                 parser.error('you must specifiy --- <virt-server>...')
 621         if n_non_actions >= len(parser.largs):
 622                 parser.error('nothing to do specified')
 623
 624         for k in timeouts.keys():
 625                 t = getattr(opts,'timeout_'+k)
 626                 if t is None: t = timeouts[k] * opts.timeout_factor
 627                 timeouts[k] = int(t)
 628
 629         arghandling = initial_arghandling
 630         opts.actions = []
 631         ix = 0
 632         for act in args:
 633                 if type(act) == dict:
 634                         arghandling = act
 635                         continue
 636                 elif type(act) == tuple:
 637                         pass
 638                 elif type(act) == str:
 639                         act = (act,act)
 640                 else:
 641                         raise ("unknown action in list `%s' having"
 642                               " type `%s'" % (act, type(act)))
 643                 (pathstr, kindpath) = act
 644
 645                 constructor = InputFile
 646                 if type(kindpath) is tuple:             kind = kindpath[0]
 647                 elif kindpath.endswith('.deb'):         kind = 'deb'
 648                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 649                 elif kindpath.endswith('//'):
 650                         kind = 'ubtree'
 651                         constructor = InputDir
 652                 elif kindpath.endswith('/'):
 653                         kind = 'tree'
 654                         constructor = InputDir
 655                 else: parser.error("do not know how to handle filename `%s';"
 656                         " specify --source --binary or --build-tree" %
 657                         kindpath)
 658
 659                 what = '%s%s' % (kind,ix); ix += 1
 660
 661                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 662                 else: fwhatx = '-'+kind
 663
 664                 if pathstr is None: af = None
 665                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 666                 opts.actions.append(Action(kind, af, arghandling, what))
 667
 668 def setup_trace():
 669         global trace_stream, tmpdir, summary_stream
 670
 671         if opts.tmpdir is not None:
 672                 rmtree('tmpdir(specified)',opts.tmpdir)
 673                 mkdir_okexist(opts.tmpdir, 0700)
 674                 tmpdir = opts.tmpdir
 675         else:
 676                 assert(tmpdir is None)
 677                 tmpdir = tempfile.mkdtemp()
 678
 679         if opts.logfile is None:
 680                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 681                         opts.logfile = opts.output_dir.spec + '/log'
 682                 elif opts.tmpdir is not None:
 683                         opts.logfile = opts.tmpdir + '/log'
 684         if opts.logfile is not None:
 685                 trace_stream = open(opts.logfile, 'w', 0)
 686         if opts.summary is not None:
 687                 summary_stream = open(opts.summary, 'w', 0)
 688
 689         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 690
 691 def finalise_options():
 692         global opts, tb, build_essential
 693
 694         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 695                 opts.user = ''
 696
 697         if opts.user is None:
 698                 su = 'suggested-normal-user='
 699                 ul = [
 700                         e[len(su):]
 701                         for e in testbed.caps
 702                         if e.startswith(su)
 703                         ]
 704                 if ul:
 705                         opts.user = ul[0]
 706                 else:
 707                         opts.user = ''
 708
 709         if opts.user:
 710                 if 'root-on-testbed' not in testbed.caps:
 711                         pstderr("warning: virtualisation"
 712                                 " system does not offer root on testbed,"
 713                                 " but --user option specified: failure likely")
 714                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 715         else:
 716                 opts.user_wrap = lambda x: x
 717
 718         if opts.gainroot is None:
 719                 opts.gainroot = ''
 720                 if (opts.user or
 721                     'root-on-testbed' not in testbed.caps):
 722                         opts.gainroot = 'fakeroot'
 723                         build_essential += ['fakeroot']
 724
 725         if opts.gnupghome.startswith('~/'):
 726                 try: home = os.environ['HOME']
 727                 except KeyError:
 728                         parser.error("HOME environment variable"
 729                                 " not set, needed for --gnupghome=`%s"
 730                                 % opts.gnupghome)
 731                 opts.gnupghome = home + opts.gnupghome[1:]
 732         elif opts.gnupghome == 'fresh':
 733                 opts.gnupghome = None
 734
 735 #---------- testbed management - the Testbed class
 736
 737 class Testbed:
 738  def __init__(tb):
 739         tb.sp = None
 740         tb.lastsend = None
 741         tb.scratch = None
 742         tb.modified = False
 743         tb.blamed = []
 744         tb._debug('init')
 745         tb._need_reset_apt = False
 746  def _debug(tb, m, minlevel=0):
 747         debug('** '+m, minlevel)
 748  def start(tb):
 749         tb._debug('start')
 750         p = subprocess.PIPE
 751         debug_subprocess('vserver', opts.vserver)
 752         tb._errplumb = Errplumb(True)
 753         tb.sp = subprocess.Popen(opts.vserver,
 754                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 755         tb.expect('ok')
 756         tb.caps = tb.commandr('capabilities')
 757  def stop(tb):
 758         tb._debug('stop')
 759         tb.close()
 760         if tb.sp is None: return
 761         ec = tb.sp.returncode
 762         if ec is None:
 763                 tb.sp.stdout.close()
 764                 tb.send('quit')
 765                 tb.sp.stdin.close()
 766                 ec = tb.sp.wait()
 767         if ec:
 768                 tb.bomb('testbed gave exit status %d after quit' % ec)
 769         tb._errplumb.wait()
 770  def open(tb):
 771         tb._debug('open, scratch=%s' % tb.scratch)
 772         if tb.scratch is not None: return
 773         pl = tb.commandr('open')
 774         tb._opened(pl)
 775  def _opened(tb, pl):
 776         tb.scratch = InputDir('tb-scratch', pl[0], True)
 777         tb.deps_processed = []
 778         for af in paths: af.invalidate_ephem(True)
 779         tb._auxverbscript_make()
 780  def _auxverbscript_make(tb):
 781         pec = tb.commandr('print-auxverb-command')
 782         if len(pec) < 1: tb.bomb('too few results from print-execute-command')
 783         cmdl = map(urllib.unquote, pec[0].split(','))
 784
 785         shellquote_re = regexp.compile('"')
 786         def shellquote_arg(s): return "'" + shellquote_re.sub(r"'\''", s) + "'"
 787         def shellquote_cmdl(l): return ' '.join(map(shellquote_arg,l))
 788
 789         tb._debug('cmdl = %s' % (`cmdl`))
 790
 791         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 792         print >>open(tb.ec_auxverbscript.write(),'w'), (
 793 '''#!/bin/sh
 794 set -e
 795 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 796         # This is a pretty nasty hack.  Hopefully it can go away
 797         #  eventually.  See #635763.
 798         set -- dpkg --print-architecture
 799 fi
 800 exec '''+shellquote_cmdl(cmdl)+' "$@"'+"\n"
 801                 )
 802         os.chmod(tb.ec_auxverbscript.write(), 0755)
 803
 804  def mungeing_apt(tb):
 805         if not 'revert' in tb.caps:
 806                 tb._need_reset_apt = True
 807  def reset_apt(tb):
 808         if not tb._need_reset_apt: return
 809         what = 'aptget-update-reset'
 810         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 811         rc = tb.execute(what, cmdl, kind='install')
 812         if rc:
 813                 pstderr("\n" "warning: failed to restore"
 814                         " testbed apt cache, exit code %d" % rc)
 815         tb._need_reset_apt = False
 816  def close(tb):
 817         tb._debug('close, scratch=%s' % tb.scratch)
 818         if tb.scratch is None: return
 819         tb.scratch = None
 820         if tb.sp is None: return
 821         tb.command('close')
 822  def prepare1(tb, deps_new):
 823         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 824                 (tb.modified, tb.deps_processed, deps_new), 1)
 825         if 'revert' in tb.caps and (tb.modified or
 826             [d for d in tb.deps_processed if d not in deps_new]):
 827                 for af in paths: af.read(False)
 828                 tb._debug('reset **')
 829                 pl = tb.commandr('revert')
 830                 tb._opened(pl)
 831         tb.modified = False
 832  def prepare2(tb, deps_new):
 833         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 834         binaries.publish()
 835         tb._install_deps(deps_new)
 836  def prepare(tb, deps_new):
 837         tb.prepare1(deps_new)
 838         tb.prepare2(deps_new)
 839  def _install_deps(tb, deps_new):
 840         tb._debug(' installing dependencies '+`deps_new`, 1)
 841         tb.deps_processed = deps_new
 842         if not deps_new: return
 843         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 844  def needs_reset(tb):
 845         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 846         tb.modified = True
 847  def blame(tb, m):
 848         tb._debug('blame += %s' % m, 1)
 849         tb.blamed.append(m)
 850  def bomb(tb, m):
 851         tb._debug('bomb %s' % m)
 852         if tb.sp is not None:
 853                 tb.sp.stdout.close()
 854                 tb.sp.stdin.close()
 855                 ec = tb.sp.wait()
 856                 if ec: pstderr('adt-run: testbed failing,'
 857                                 ' exit status %d' % ec)
 858         tb.sp = None
 859         raise Quit(16, 'testbed failed: %s' % m)
 860  def send(tb, string):
 861         tb.sp.stdin
 862         try:
 863                 debug('>> '+string, 2)
 864                 print >>tb.sp.stdin, string
 865                 tb.sp.stdin.flush()
 866                 tb.lastsend = string
 867         except:
 868                 (type, value, dummy) = sys.exc_info()
 869                 tb.bomb('cannot send to testbed: %s' % traceback.
 870                         format_exception_only(type, value))
 871  def expect(tb, keyword, nresults=None):
 872         l = tb.sp.stdout.readline()
 873         if not l: tb.bomb('unexpected eof from the testbed')
 874         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 875         l = l.rstrip('\n')
 876         debug('<< '+l, 2)
 877         ll = l.split()
 878         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 879         if ll[0] != keyword:
 880                 if tb.lastsend is None:
 881                         tb.bomb("got banner `%s', expected `%s...'" %
 882                                 (l, keyword))
 883                 else:
 884                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 885                                 (tb.lastsend, l, keyword))
 886         ll = ll[1:]
 887         if nresults is not None and len(ll) != nresults:
 888                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 889                         " expected %d result parameters" %
 890                         (tb.lastsend, l, len(ll), nresults))
 891         return ll
 892  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 893         # pass args=[None,...] or =(None,...) to avoid more url quoting
 894         if type(cmd) is str: cmd = [cmd]
 895         if len(args) and args[0] is None: args = args[1:]
 896         else: args = map(urllib.quote, args)
 897         al = cmd + args
 898         tb.send(string.join(al))
 899         ll = tb.expect('ok', nresults)
 900         if unquote: ll = map(urllib.unquote, ll)
 901         return ll
 902  def command(tb, cmd, args=()):
 903         tb.commandr(cmd, args, 0)
 904  def commandr1(tb, cmd, args=()):
 905         rl = tb.commandr(cmd, args, 1)
 906         return rl[0]
 907  def execute(tb, what, cmdl,
 908                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 909                 script=False, tmpdir=None, kind='short'):
 910         # Options for script:
 911         #   False - do not call debug_subprocess, no synch. reporting required
 912         #   None or string - call debug_subprocess with that value,
 913         #                       plumb stderr through synchronously if possible
 914         # Options for se:
 915         #   None - usual Errplumb (output is of kind 2c)
 916         #   string - path on testbed (output is of kind 2d)
 917
 918         timeout = timeouts[kind]
 919
 920         if script is not False: debug_subprocess(what, cmdl, script=script)
 921         if cwd is None: cwd = tb.scratch.write(True)
 922
 923         xdump = None
 924         if se is None:
 925                 ep = Errplumb()
 926                 se_catch = TemporaryFile(what+'-xerr')
 927                 se_use = se_catch.write(True)
 928                 if not opts.quiet: xdump = 'debug=2-2'
 929                 elif trace_stream is not None:
 930                         xdump = 'debug=2-%d' % trace_stream.fileno()
 931         else:
 932                 ep = None
 933                 se_catch = None
 934                 se_use = se
 935
 936         cmdl = [None,
 937                 ','.join(map(urllib.quote, cmdl)),
 938                 si, so, se_use, cwd]
 939
 940         if timeout is not None and timeout > 0:
 941                 cmdl.append('timeout=%d' % timeout)
 942
 943         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 944         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 945         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 946         if opts.set_lang is not False:
 947                 cmdl.append('env=LANG=%s' % opts.set_lang)
 948
 949         rc = tb.commandr1('execute', cmdl)
 950         try: rc = int(rc)
 951         except ValueError: bomb("execute for %s gave invalid response `%s'"
 952                                         % (what,rc))
 953
 954         if se_catch is not None:
 955                 debug_file(se_catch.read())
 956         if ep is not None:
 957                 ep.wait()
 958
 959         return rc
 960
 961  def satisfy_dependencies_string(tb, deps, what):
 962         # Must have called Binaries.configure_apt
 963         debug('dependencies: %s: satisfying %s' % (what,deps))
 964         dsc = TemporaryFile('deps.dsc')
 965         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 966         # pbuilder-satisfydepends has a bug where it ignores the
 967         #  Build-Depends if it's the last line in the dsc (#635696)
 968         tb.satisfy_dependencies_dsc(dsc, what)
 969
 970  def satisfy_dependencies_dsc(tb, dsc, what):
 971         # Must have called Binaries.configure_apt
 972         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 973                 '--binary-all', # --check-key
 974                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 975                 '-c',dsc.read()
 976         ]
 977         # The --internal-chrootexec option is really handy but
 978         #  perhaps we are not supposed to use it ?  See also #635697.
 979         debug('dependencies: %s: running %s' % (what,`cmdl`))
 980         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 981         if rc: badpkg('dependency install failed, exit code %d' % rc)
 982
 983 #---------- representation of test control files: Field*, Test, etc.
 984
 985 class FieldBase:
 986  def __init__(f, fname, stz, base, tnames, vl):
 987         assert(vl)
 988         f.stz = stz
 989         f.base = base
 990         f.tnames = tnames
 991         f.vl = vl
 992  def words(f):
 993         def distribute(vle):
 994                 (lno, v) = vle
 995                 r = v.split()
 996                 r = map((lambda w: (lno, w)), r)
 997                 return r
 998         return flatten(map(distribute, f.vl))
 999  def atmostone(f):
1000         if len(vl) == 1:
1001                 (f.lno, f.v) = vl[0]
1002         else:
1003                 raise Unsupported(f.vl[1][0],
1004                         'only one %s field allowed' % fn)
1005         return f.v
1006
1007 class FieldIgnore(FieldBase):
1008  def parse(f): pass
1009
1010 class Restriction:
1011  def __init__(r,rname,base): pass
1012
1013 class Restriction_rw_build_tree(Restriction): pass
1014 class Restriction_breaks_testbed(Restriction):
1015  def __init__(r, rname, base):
1016         if 'revert' not in testbed.caps:
1017                 raise Unsupported(f.lno,
1018                         'Test breaks testbed but testbed cannot revert')
1019 class Restriction_needs_root(Restriction):
1020  def __init__(r, rname, base):
1021         if 'root-on-testbed' not in testbed.caps:
1022                 raise Unsupported(f.lno,
1023                         'Test needs root on testbed which is not available')
1024
1025 class Field_Restrictions(FieldBase):
1026  def parse(f):
1027         for wle in f.words():
1028                 (lno, rname) = wle
1029                 nrname = rname.replace('-','_')
1030                 try: rclass = globals()['Restriction_'+nrname]
1031                 except KeyError: raise Unsupported(lno,
1032                         'unknown restriction %s' % rname)
1033                 r = rclass(nrname, f.base)
1034                 f.base['restriction_names'].append(rname)
1035                 f.base['restrictions'].append(r)
1036
1037 class Field_Features(FieldIgnore):
1038  def parse(f):
1039         for wle in f.words():
1040                 (lno, fname) = wle
1041                 f.base['feature_names'].append(fname)
1042                 nfname = fname.replace('-','_')
1043                 try: fclass = globals()['Feature_'+nfname]
1044                 except KeyError: continue
1045                 ft = fclass(nfname, f.base)
1046                 f.base['features'].append(ft)
1047
1048 class Field_Tests(FieldIgnore): pass
1049
1050 class Field_Depends(FieldBase):
1051  def parse(f):
1052         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
1053         dl = map(lambda x: x.strip(),
1054                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1055         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1056         for d in dl:
1057                 if re.search(d):
1058                         badpkg("Test Depends field contains dependency"
1059                                " `%s' with invalid characters" % d)
1060         f.base['depends'] = dl
1061
1062 class Field_Tests_directory(FieldBase):
1063  def parse(f):
1064         td = atmostone(f)
1065         if td.startswith('/'): raise Unspported(f.lno,
1066                 'Tests-Directory may not be absolute')
1067         f.base['testsdir'] = td
1068
1069 def run_tests(stanzas, tree):
1070         global errorcode, testbed
1071         if stanzas == ():
1072                 report('*', 'SKIP no tests in this package')
1073                 errorcode |= 8
1074         for stanza in stanzas:
1075                 tests = stanza[' tests']
1076                 if not tests:
1077                         report('*', 'SKIP package has metadata but no tests')
1078                         errorcode |= 8
1079                 for t in tests:
1080                         t.prepare()
1081                         t.run(tree)
1082                         if 'breaks-testbed' in t.restriction_names:
1083                                 testbed.needs_reset()
1084                 testbed.needs_reset()
1085
1086 class Test:
1087  def __init__(t, tname, base, act):
1088         if '/' in tname: raise Unsupported(base[' lno'],
1089                 'test name may not contain / character')
1090         for k in base: setattr(t,k,base[k])
1091         t.tname = tname
1092         t.act = act
1093         t.what = act.what+'t-'+tname
1094         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1095         else: t.path = tname
1096         t._debug('constructed; path=%s' % t.path)
1097         t._debug(' .depends=%s' % t.depends)
1098  def _debug(t, m):
1099         debug('& %s: %s' % (t.what, m))
1100  def report(t, m):
1101         report(t.what, m)
1102  def reportfail(t, m):
1103         global errorcode
1104         errorcode |= 4
1105         report(t.what, 'FAIL ' + m)
1106  def prepare(t):
1107         t._debug('preparing')
1108         dn = []
1109         for d in t.depends:
1110                 t._debug(' processing dependency '+d)
1111                 if not '@' in d:
1112                         t._debug('  literal dependency '+d)
1113                         dn.append(d)
1114                 else:
1115                         for (pkg,bin) in t.act.binaries:
1116                                 d = d.replace('@',pkg)
1117                                 t._debug('  synthesised dependency '+d)
1118                                 dn.append(d)
1119         testbed.prepare(dn)
1120  def run(t, tree):
1121         t._debug('[----------------------------------------')
1122         def stdouterr(oe):
1123                 idstr = t.what + '-' + oe
1124                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1125                         use_dir = opts.output_dir
1126                 else:
1127                         use_dir = testbed.scratch
1128                 return RelativeOutputFile(idstr, use_dir, idstr)
1129
1130         if hasattr(t.act,'work'): t.act.work.write(True)
1131         tree.read(True)
1132
1133         af = RelativeInputFile(t.what, tree, t.path)
1134         so = stdouterr('stdout')
1135         se = stdouterr('stderr')
1136
1137         tf = af.read(True)
1138         tmpdir = None
1139
1140         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1141         if rc: bomb('failed to chmod +x %s' % tf)
1142
1143         if 'needs-root' not in t.restriction_names and opts.user is not None:
1144                 tfl = ['su',opts.user,'-c',tf]
1145                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1146                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1147                 if opts.user:
1148                         script += '; chown %s "$1"' % opts.user
1149                         if 'rw-build-tree' in t.restriction_names:
1150                                 script += '; chown -R %s "$2"' % opts.user
1151                 rc = testbed.execute('mktmpdir-'+t.what,
1152                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1153                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1154                                 % (tmpdir, rc))
1155         else:
1156                 tfl = [tf]
1157
1158         rc = testbed.execute('test-'+t.what, tfl,
1159                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1160                 tmpdir=tmpdir, kind='test')
1161
1162         so_read = so.read()
1163         se_read = se.read()
1164
1165         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1166         stab = os.stat(se_read)
1167         if stab.st_size != 0:
1168                 l = open(se_read).readline()
1169                 l = l.rstrip('\n \t\r')
1170                 if len(l) > 35: l = l[:35] + '...'
1171                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1172                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1173                 debug_file(se_read)
1174         elif rc != 0:
1175                 t.reportfail('non-zero exit status %d' % rc)
1176         else:
1177                 t.report('PASS')
1178
1179         stab = os.stat(so_read)
1180         if stab.st_size != 0:
1181                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1182                 debug_file(so_read)
1183
1184         t._debug('----------------------------------------]')
1185
1186 def read_control(act, tree, control_override):
1187         stanzas = [ ]
1188
1189         if control_override is not None:
1190                 control_af = control_override
1191                 testbed.blame('arg:'+control_override.spec)
1192         else:
1193                 if act.missing_tests_control:
1194                         return ()
1195                 control_af = RelativeInputFile(act.what+'-testcontrol',
1196                         tree, 'debian/tests/control')
1197         try:
1198                 control = open(control_af.read(), 'r')
1199         except (IOError,OSError), oe:
1200                 if oe[0] != errno.ENOENT: raise
1201                 return []
1202
1203         lno = 0
1204         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1205         stz = None      # stz[field_name][index] = (lno, value)
1206                         # special field names:
1207                         # stz[' lno'] = number
1208                         # stz[' tests'] = list of Test objects
1209         def end_stanza(stz):
1210                 if stz is None: return
1211                 stz[' errs'] = 0
1212                 stanzas.append(stz)
1213                 stz = None
1214                 hcurrent = None
1215
1216         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1217         while 1:
1218                 l = control.readline()
1219                 if not l: break
1220                 lno += 1
1221                 if not l.endswith('\n'): badctrl('unterminated line')
1222                 if regexp.compile('\s*\#').match(l): continue
1223                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1224                 initmat = initre.match(l)
1225                 if initmat:
1226                         (fname, l) = initmat.groups()
1227                         fname = string.capwords(fname)
1228                         if stz is None:
1229                                 stz = { ' lno': lno, ' tests': [] }
1230                         if not stz.has_key(fname): stz[fname] = [ ]
1231                         hcurrent = stz[fname]
1232                 elif regexp.compile('\s').match(l):
1233                         if not hcurrent: badctrl('unexpected continuation')
1234                 else:
1235                         badctrl('syntax error')
1236                 hcurrent.append((lno, l))
1237         end_stanza(stz)
1238
1239         def testbadctrl(stz, lno, m):
1240                 report_badctrl(lno, m)
1241                 stz[' errs'] += 1
1242
1243         for stz in stanzas:
1244                 try:
1245                         try: tnames = stz['Tests']
1246                         except KeyError:
1247                                 tnames = ['*']
1248                                 raise Unsupported(stz[' lno'],
1249                                         'no Tests field')
1250                         tnames = map((lambda lt: lt[1]), tnames)
1251                         tnames = string.join(tnames).split()
1252                         base = {
1253                                 'restriction_names': [],
1254                                 'restrictions': [],
1255                                 'feature_names': [],
1256                                 'features': [],
1257                                 'testsdir': 'debian/tests',
1258                                 'depends' : '@'
1259                         }
1260                         for fname in stz.keys():
1261                                 if fname.startswith(' '): continue
1262                                 vl = stz[fname]
1263                                 try: fclass = globals()['Field_'+
1264                                         fname.replace('-','_')]
1265                                 except KeyError: raise Unsupported(vl[0][0],
1266                                         'unknown metadata field %s' % fname)
1267                                 f = fclass(stz, fname, base, tnames, vl)
1268                                 f.parse()
1269                         for tname in tnames:
1270                                 t = Test(tname, base, act)
1271                                 stz[' tests'].append(t)
1272                 except Unsupported, u:
1273                         for tname in tnames: u.report(tname)
1274                         continue
1275
1276         return stanzas
1277
1278 def print_exception(ei, msgprefix=''):
1279         if msgprefix: pstderr(msgprefix)
1280         (et, q, tb) = ei
1281         if et is Quit:
1282                 pstderr('adt-run: ' + q.m)
1283                 psummary('quitting: '+q.m)
1284                 return q.ec
1285         else:
1286                 pstderr("adt-run: unexpected, exceptional, error:")
1287                 psummary('quitting: unexpected error, consult transcript')
1288                 traceback.print_exc(None, sys.stderr)
1289                 if trace_stream is not None:
1290                         traceback.print_exc(None, trace_stream)
1291                 return 20
1292
1293 def cleanup():
1294         global trace_stream
1295         try:
1296                 if testbed is not None:
1297                         testbed.reset_apt()
1298                         testbed.stop()
1299                 if opts.tmpdir is None and tmpdir is not None:
1300                         rmtree('tmpdir', tmpdir)
1301                 if trace_stream is not None:
1302                         trace_stream.close()
1303                         trace_stream = None
1304         except:
1305                 print_exception(sys.exc_info(),
1306                         '\nadt-run: error cleaning up:\n')
1307                 os._exit(20)
1308
1309 #---------- registration, installation etc. of .deb's: Binaries
1310
1311 def determine_package(act):
1312         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1313         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1314         if rc: badpkg('failed to parse binary package, code %d' % rc)
1315         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1316         act.pkg = None
1317         for l in output.split('\n'):
1318                 m = re.match(l)
1319                 if not m: continue
1320                 if act.pkg: badpkg('two Package: lines in control file')
1321                 act.pkg = m.groups()[0]
1322         if not act.pkg: badpkg('no good Package: line in control file')
1323
1324 class Binaries:
1325  def __init__(b, tb):
1326         b.dir = TemporaryDir('binaries')
1327         b.dir.write()
1328         ok = False
1329
1330         if opts.gnupghome is None:
1331                 opts.gnupghome = tmpdir+'/gnupg'
1332
1333         b._debug('initialising')
1334         try:
1335                 for x in ['pubring','secring']:
1336                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1337                 ok = True
1338         except (IOError,OSError), oe:
1339                 if oe.errno != errno.ENOENT: raise
1340
1341         if ok: b._debug('no key generation needed')
1342         else: b.genkey()
1343
1344  def _debug(b, s):
1345         debug('* '+s)
1346
1347  def genkey(b):
1348         b._debug('preparing for key generation')
1349
1350         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1351         mkdir_okexist(opts.gnupghome, 0700)
1352
1353         script = '''
1354   exec >&2
1355   cd "$1"
1356   cat <<"END" >key-gen-params
1357 Key-Type: DSA
1358 Key-Length: 1024
1359 Key-Usage: sign
1360 Name-Real: autopkgtest per-run key
1361 Name-Comment: do not trust this key
1362 Name-Email: autopkgtest@example.com
1363 END
1364   set -x
1365   gpg --homedir="$1" --batch --gen-key key-gen-params
1366                 '''
1367         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1368         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1369         if rc: bomb('key generation failed, code %d' % rc)
1370
1371  def apt_configs(b):
1372         return {
1373                 "Debug::pkgProblemResolver": "true",
1374                 "APT::Get::force-yes" : "true",
1375                 "APT::Get::Assume-Yes" : "true",
1376                 "quiet" : "true",
1377         }
1378
1379  def _configure_apt(b, tb):
1380         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1381                         True)
1382         f = open(config.write(),'w')
1383         for (k,v) in b.apt_configs().iteritems():
1384                 print >>f, '%s { "%s"; };' % (k, v)
1385         f.close()
1386         config.read(True)
1387
1388         prefs = OutputFile('apt-prefs','/etc/apt/preferences.d/90autopkgtest',
1389                         True)
1390         print >>open(prefs.write(),'w'), '''
1391 Package: *
1392 Pin: origin ""
1393 Pin-Priority: 1002
1394 '''
1395         prefs.read(True)
1396
1397  def _apt_get(b):
1398         ag = ['apt-get','-q']
1399         for kv in b.apt_configs().iteritems():
1400                 ag += ['-o', '%s=%s' % kv]
1401         return ' '.join(ag)
1402
1403  def reset(b):
1404         b._debug('reset')
1405         rmtree('binaries', b.dir.read())
1406         b.dir.invalidate()
1407         b.dir.write()
1408         b.install = []
1409         b.blamed = []
1410         b.registered = set()
1411
1412  def register(b, act, pkg, af, forwhat, blamed):
1413         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1414                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1415
1416         if act.ah['deb_'+forwhat] == 'ignore': return
1417
1418         b.blamed += testbed.blamed
1419
1420         leafname = pkg+'.deb'
1421         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1422
1423         try: os.remove(dest.write())
1424         except (IOError,OSError), oe:
1425                 if oe.errno != errno.ENOENT: raise e
1426
1427         try: os.link(af.read(), dest.write())
1428         except (IOError,OSError), oe:
1429                 if oe.errno != errno.EXDEV: raise e
1430                 shutil.copy(af.read(), dest)
1431
1432         if act.ah['deb_'+forwhat] == 'install':
1433                 b.install.append(pkg)
1434
1435         b.registered.add(pkg)
1436
1437  def publish(b):
1438         b._debug('publish')
1439
1440         b._configure_apt(testbed)
1441
1442         script = '''
1443   exec >&2
1444   cd "$1"
1445   apt-ftparchive packages . >Packages
1446   gzip <Packages >Packages.gz
1447   apt-ftparchive release . >Release
1448   rm -f Release.gpg
1449   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1450   gpg --homedir="$2" --batch --export >archive-key.pgp
1451                 '''
1452         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1453         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1454         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1455
1456         b.dir.invalidate(True)
1457         apt_source = b.dir.read(True)
1458
1459         so = TemporaryFile('vlds')
1460         script = '''
1461   exec 3>&1 >&2
1462   apt-key add archive-key.pgp
1463   echo "deb file://'''+apt_source+''' /" >/etc/apt/sources.list.d/autopkgtest.list
1464   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1465     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1466   fi
1467   '''+ b._apt_get() +''' update >&2
1468   cat /var/lib/dpkg/status >&3
1469                 '''
1470         testbed.mungeing_apt()
1471         rc = testbed.execute('apt-key', ['sh','-ec',script],
1472                                 so=so.write(True), cwd=b.dir.write(True),
1473                                 script=script, kind='install')
1474         if rc: bomb('apt setup failed with exit code %d' % rc)
1475
1476         testbed.blamed += b.blamed
1477
1478         b._debug('publish reinstall checking...')
1479         pkgs_reinstall = set()
1480         pkg = None
1481         for l in open(so.read()):
1482                 if l.startswith('Package: '):
1483                         pkg = l[9:].rstrip()
1484                 elif l.startswith('Status: install '):
1485                         if pkg in b.registered:
1486                                 pkgs_reinstall.add(pkg)
1487                                 b._debug(' publish reinstall needs '+pkg)
1488
1489         if pkgs_reinstall:
1490                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1491                 what = 'apt-get-reinstall'
1492                 cmdl = (b._apt_get() + ' --reinstall install '+
1493                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1494                 cmdl = ['sh','-c',cmdl]
1495                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1496                 if rc: badpkg("installation of basic binarries failed,"
1497                                 " exit code %d" % rc)
1498
1499         b._debug('publish install...')
1500         for pkg in b.install:
1501                 what = 'apt-get-install-%s' % pkg
1502                 testbed.blame(pkg)
1503                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1504                 cmdl = ['sh','-c',cmdl]
1505                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1506                 if rc: badpkg("installation of %s failed, exit code %d"
1507                                 % (pkg, rc))
1508
1509         b._debug('publish done')
1510
1511 #---------- processing of sources (building)
1512
1513 def source_rules_command(act,script,what,which,work,cwd,
1514                                 results_lines=0,xargs=[]):
1515         script = [      "exec 3>&1 >&2",
1516                         "set -x"        ] + script
1517         script = '\n'.join(script)
1518         so = TemporaryFile('%s-%s-results' % (what,which))
1519         rc = testbed.execute('%s-%s' % (what,which),
1520                         ['sh','-ec',script]+xargs, script=script,
1521                         so=so.write(True), cwd=cwd, kind='build')
1522         results = open(so.read()).read().rstrip('\n')
1523         if len(results): results = results.split("\n")
1524         else: results = []
1525         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1526         if results_lines is not None and len(results) != results_lines:
1527                 badpkg("got %d lines of results from %s where %d expected"
1528                         % (len(results), which, results_lines))
1529         if results_lines==1: return results[0]
1530         return results
1531
1532 def build_source(act, control_override):
1533         act.blame = 'arg:'+act.af.spec
1534         testbed.blame(act.blame)
1535         testbed.prepare1([])
1536         testbed.needs_reset()
1537
1538         what = act.what
1539         basename = act.af.spec
1540         debiancontrol = None
1541         act.binaries = []
1542
1543         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1544
1545         if act.kind == 'dsc':
1546                 dsc = act.af
1547                 dsc_file = open(dsc.read())
1548                 in_files = False
1549                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1550                 for l in dsc_file:
1551                         l = l.rstrip('\n')
1552                         if l.startswith('Files:'): in_files = True; continue
1553                         elif l.startswith('#'): pass
1554                         elif not l.startswith(' '):
1555                                 in_files = False
1556                                 if l.startswith('Source:'):
1557                                         act.blame = 'dsc:'+l[7:].strip()
1558                                         testbed.blame(act.blame)
1559                         if not in_files: continue
1560
1561                         m = fre.match(l)
1562                         if not m: badpkg(".dsc contains unparseable line"
1563                                         " in Files: `%s'" % l)
1564                         leaf = m.groups(0)[0]
1565                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1566                                         sibling=True)
1567                         subfile.read(True)
1568                 dsc.read(True)
1569
1570         if act.kind == 'ubtree':
1571                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1572                         act.af, 'debian/control')
1573                 dsc = TemporaryFile(what+'-fakedsc')
1574                 dsc_w = open(dsc.write(), 'w')
1575                 for l in open(debiancontrol.read()):
1576                         l = l.rstrip('\n')
1577                         if not len(l): break
1578                         print >>dsc_w, l
1579                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1580                 dsc_w.close()
1581
1582         if act.kind == 'dsc':
1583                 testbed.prepare2([])
1584                 testbed.satisfy_dependencies_string('dpkg-dev',
1585                                                 'install dpkg-dev')
1586
1587         work = TemporaryDir(what+'-build')
1588         act.work = work
1589
1590         tmpdir = work.write(True)+'/tmpdir'
1591         tmpdir_script = [
1592                         'TMPDIR="$1"',
1593                         'rm -rf -- "$TMPDIR"',
1594                         'export TMPDIR',
1595                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1596                 ]
1597
1598         if act.kind == 'ubtree':
1599                 spec = '%s/real-tree' % work.write(True)
1600                 create_command = '''
1601                         rm -rf "$spec"
1602                         mkdir "$spec"
1603                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1604                         '''
1605                 initcwd = act.af.read(True)
1606
1607         if act.kind == 'dsc':
1608                 spec = dsc.read(True)
1609                 create_command = '''
1610                         dpkg-source -x $spec
1611                         '''
1612                 initcwd = work.write(True)
1613
1614         script = [
1615                 'spec="$2"',
1616                 'origpwd=`pwd`',
1617                 'cd '+work.write(True)
1618         ]
1619
1620         if opts.user:
1621                 script += ([ 'chown '+opts.user+' .' ] +
1622                         tmpdir_script +
1623                         [ 'spec="$spec" origpwd="$origpwd" '
1624                                 +opts.user_wrap(create_command) ])
1625         else:
1626                 script += (tmpdir_script +
1627                         [ create_command ])
1628
1629         script += [
1630                         'cd [a-z0-9]*-*/.',
1631                         'pwd >&3',
1632                         'set +e; test -f debian/tests/control; echo $? >&3'
1633                 ]
1634         (result_pwd, control_test_rc) = source_rules_command(
1635                         act,script,what,'extract',work,
1636                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1637
1638         filter = act.ah['dsc_filter']
1639
1640         if control_test_rc == '1': act.missing_tests_control = True
1641
1642         # For optional builds:
1643         #
1644         # We might need to build the package because:
1645         #   - we want its binaries (filter isn't _ and at least one of the
1646         #       deb_... isn't ignore)
1647         #   - the test control file says so
1648         #       (assuming we have any tests)
1649
1650         class NeedBuildException: pass
1651         def build_needed(m):
1652                 debug_b('build needed for %s' % m)
1653                 raise NeedBuildException()
1654
1655         try:
1656                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1657                                       act.ah['deb_fortests'] != 'ignore'):
1658                         build_needed('binaries')
1659
1660                 result_pwd_af = InputDir(what+'-treeforcontrol',
1661                         result_pwd, True)
1662                 stanzas = read_control(act, result_pwd_af, control_override)
1663                 for stanza in stanzas:
1664                         for t in stanza[' tests']:
1665                                 if 'no-build-needed' not in t.feature_names:
1666                                         build_needed('test %s' % t.tname)
1667                                 for d in t.depends:
1668                                         if '@' in d:
1669                                                 build_needed('test %s '
1670                                                  'dependency %s' % (t.tname,d))
1671
1672                 debug_b('build not needed')
1673                 built = False
1674
1675         except NeedBuildException:
1676
1677                 if act.kind != 'dsc':
1678                         testbed.prepare2([])
1679
1680                 testbed.satisfy_dependencies_string('build-essential',
1681                                 'install build-essential')
1682                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1683
1684                 script = tmpdir_script + [
1685                         'cd "$2"',
1686                         'dpkg-checkbuilddeps',
1687                         opts.user_wrap('debian/rules build'),
1688                         ]
1689                 source_rules_command(act,script,what,'build',work,
1690                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1691
1692                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1693                         badpkg("results dir `%s' is not in expected parent"
1694                                " dir `%s'" % (result_pwd, work.read(True)))
1695
1696                 built = True
1697
1698         act.tests_tree = RelativeInputDir(what+'-tests-tree',
1699                                 work, os.path.basename(result_pwd),
1700                                 True)
1701
1702         if not built:
1703                 act.blamed = []
1704                 return
1705
1706         act.blamed = copy.copy(testbed.blamed)
1707
1708         debug_b('filter=%s' % filter)
1709         if filter != '_':
1710                 script = tmpdir_script + [
1711                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1712                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1713                         'cd ..',
1714                         'echo *.deb >&3',
1715                         ]
1716                 result_debs = source_rules_command(act,script,what,
1717                                 'binary',work,work.write(True),
1718                                 results_lines=1, xargs=['x',tmpdir])
1719                 if result_debs == '*.deb': debs = []
1720                 else: debs = result_debs.split(' ')
1721                 debug_b('debs='+`debs`)
1722                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1723                 for deb in debs:
1724                         m = re.match(deb)
1725                         if not m: badpkg("badly-named binary `%s'" % deb)
1726                         pkg = m.groups()[0]
1727                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1728                         for pat in filter.split(','):
1729                                 debug_b('  pat=%s' % pat)
1730                                 if not fnmatch.fnmatchcase(pkg,pat):
1731                                         debug_b('   no match')
1732                                         continue
1733                                 deb_what = pkg+'_'+what+'.deb'
1734                                 bin = RelativeInputFile(deb_what,work,deb,True)
1735                                 debug_b('  deb_what=%s, bin=%s' %
1736                                         (deb_what, str(bin)))
1737                                 binaries.register(act,pkg,bin,
1738                                         'forbuilds',testbed.blamed)
1739                                 act.binaries.append((pkg,bin))
1740                                 break
1741                 debug_b('all done.')
1742
1743 #---------- main processing loop and main program
1744
1745 def process_actions():
1746         global binaries
1747
1748         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1749         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1750         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1751
1752         debug_a1('starting')
1753         testbed.open()
1754         binaries = Binaries(testbed)
1755
1756         binaries.reset()
1757         control_override = None
1758
1759         debug_a1('builds ...')
1760         for act in opts.actions:
1761                 debug_a2('%s %s' %
1762                         (act.kind, act.what))
1763
1764                 if act.kind == 'control':
1765                         control_override = act.af
1766                 if act.kind == 'deb':
1767                         testbed.blame('arg:'+act.af.spec)
1768                         determine_package(act)
1769                         testbed.blame('deb:'+act.pkg)
1770                         binaries.register(act,act.pkg,act.af,
1771                                 'forbuilds',testbed.blamed)
1772                 if act.kind == 'dsc' or act.kind == 'ubtree':
1773                         build_source(act, control_override)
1774                 if act.kind == 'tree':
1775                         act.binaries = []
1776                 if act.kind.endswith('tree') or act.kind == 'dsc':
1777                         control_override = None
1778                 if act.kind == 'instantiate':
1779                         pass
1780
1781         debug_a1('builds done.')
1782
1783         binaries.reset()
1784         control_override = None
1785
1786         debug_a1('tests ...')
1787         for act in opts.actions:
1788                 debug_a2('test %s %s' % (act.kind, act.what))
1789
1790                 testbed.needs_reset()
1791                 if act.kind == 'control':
1792                         control_override = act.af
1793                 if act.kind == 'deb':
1794                         binaries.register(act,act.pkg,act.af,'fortests',
1795                                 ['deb:'+act.pkg])
1796                 if act.kind == 'dsc' or act.kind == 'ubtree':
1797                         for (pkg,bin) in act.binaries:
1798                                 binaries.register(act,pkg,bin,'fortests',
1799                                         act.blamed)
1800                 if act.kind == 'dsc':
1801                         if act.ah['dsc_tests']:
1802                                 debug_a3('read control ...')
1803                                 stanzas = read_control(act, act.tests_tree,
1804                                                 control_override)
1805                                 testbed.blamed += act.blamed
1806                                 debug_a3('run_tests ...')
1807                                 run_tests(stanzas, act.tests_tree)
1808                         control_override = None
1809                 if act.kind == 'tree' or act.kind == 'ubtree':
1810                         testbed.blame('arg:'+act.af.spec)
1811                         stanzas = read_control(act, act.af, control_override)
1812                         debug_a3('run_tests ...')
1813                         run_tests(stanzas, act.af)
1814                         control_override = None
1815                 if act.kind == 'instantiate':
1816                         testbed.prepare([])
1817         debug_a1('tests done.')
1818
1819 def main():
1820         global testbed
1821         global tmpdir
1822         try:
1823                 parse_args()
1824         except SystemExit, se:
1825                 os._exit(20)
1826         try:
1827                 setup_trace()
1828                 testbed = Testbed()
1829                 testbed.start()
1830                 finalise_options()
1831                 process_actions()
1832         except:
1833                 ec = print_exception(sys.exc_info(), '')
1834                 cleanup()
1835                 os._exit(ec)
1836         cleanup()
1837         os._exit(errorcode)
1838
1839 main()