runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':10, 'install':900, 'test':600, 'build':3000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         shutil.rmtree(pathname)
 192
 193 def debug_subprocess(what, cmdl=None, script=None):
 194         o = '$ '+what+':'
 195         if cmdl is not None:
 196                 ol = []
 197                 for x in cmdl:
 198                         if x is script: x = '<SCRIPT>'
 199                         ol.append(x.    replace('\\','\\\\').
 200                                         replace(' ','\\ ')      )
 201                 o += ' '+ ' '.join(ol)
 202         debug(o)
 203         if script is not None and opts.debuglevel >= 1:
 204                 o = ''
 205                 for l in script.rstrip('\n').split('\n'):
 206                         o += '$     '+l+'\n'
 207                 debug(o,1)
 208
 209 def flatten(l):
 210         return reduce((lambda a,b: a + b), l, [])
 211
 212 #---------- fancy automatic file-copying class
 213
 214 class AutoFile:
 215         # p.what
 216         # p.path[tb]    None or path    not None => path known
 217         # p.file[tb]    None or path    not None => file exists
 218         # p.spec        string or None
 219         # p.spec_tbp    True or False, or not set if spec is None
 220         # p.dir         '' or '/'
 221
 222  def __init__(p, what):
 223         p.what = what
 224         p.path = [None,None]
 225         p.file = [None,None]
 226         p.spec = None
 227         p.dir = ''
 228
 229  def __repr__(p):
 230         return "<AF@%s>" % p.__str__()
 231  def __str__(p):
 232         def ptbp(tbp):
 233                 if p.path[tbp] is None: return '-'+p.dir
 234                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 235                 else: return p.path[tbp]+p.dir+'!'
 236         out = p.what
 237         if p.spec is not None:
 238                 if not hasattr(p,'spec_tb'): out += '~'
 239                 elif p.spec_tbp: out += '#'
 240                 else: out += '='
 241                 out += p.spec
 242         out += ':'
 243         out += ptbp(False)
 244         out += '|'
 245         out += ptbp(True)
 246         return out
 247
 248  def _wrong(p, how):
 249         xtra = ''
 250         if p.spec is not None:
 251                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 252         raise ("internal error: %s (%s)" % (how, str(p)))
 253
 254  def _ensure_path(p, tbp):
 255         if p.path[tbp] is None:
 256                 if '/' in p.what:
 257                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 258                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 259                 if not tbp:
 260                         p.path[tbp] = tmpdir+'/'+p.what
 261                 else:
 262                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 263
 264  def write(p, tbp=False):
 265         p._debug('write %s...' % 'HT'[tbp])
 266         p._ensure_path(tbp)
 267
 268         if p.dir and not p.file[tbp]:
 269                 if not tbp:
 270                         p._debug('mkdir H')
 271                         mkdir_okexist(p.path[tbp])
 272                 else:
 273                         cmdl = ['sh','-ec',
 274                                 'test -d "$1" || mkdir -p "$1"',
 275                                 'x', p.path[tbp]]
 276                         tf_what = urllib.quote(p.what).replace('/','%2F')
 277                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 278                         if rc: bomb('failed to create directory %s' %
 279                                 p.path[tbp])
 280
 281         p.file[tbp] = p.path[tbp]
 282         return p.path[tbp]
 283
 284  def read(p, tbp=False):
 285         p._debug('read %s...' % 'HT'[tbp])
 286         p._ensure_path(tbp)
 287
 288         if p.file[tbp] is None:
 289                 if p.file[not tbp] is None:
 290                         p._wrong("requesting read but nonexistent")
 291                 cud = ['copyup','copydown'][tbp]
 292                 src = p.file[not tbp] + p.dir
 293                 dst = p.path[tbp] + p.dir
 294                 testbed.command(cud, (src, dst))
 295                 p.file[tbp] = p.path[tbp]
 296
 297         return p.file[tbp] + p.dir
 298
 299  def invalidate(p, tbp=False):
 300         p.file[tbp] = None
 301         p._debug('invalidated %s' % 'HT'[tbp])
 302
 303  def _debug(p, m):
 304         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 305
 306  def _constructed(p):
 307         p._debug('constructed: '+str(p))
 308         p._check()
 309
 310  def _check(p):
 311         for tbp in [False,True]:
 312          for pf in [p.path, p.file]:
 313                 if pf[tbp] is None: continue
 314                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 315                 if p.dir and pf[tbp].endswith('/'):
 316                         pf[tbp] = pf[tbp].rstrip('/')
 317                         if not pf[tbp]: pf[tbp] = '/'
 318                 if not p.dir and pf[tbp].endswith('/'):
 319                         bomb("directory `%s' specified for "
 320                              "non-directory %s" % (pf[tbp], p.what))
 321
 322  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 323         AutoFile.__init__(p,what)
 324         sh_on = ''; sh_sibl = ''
 325         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 326         if sibling: sh_sibl=' (sibling)'
 327         parent._debug('using as base: %s: %s%s%s' %
 328                         (str(parent), leaf, sh_on, sh_sibl))
 329         if not sibling and not parent.dir:
 330                 parent._wrong('asked for non-sibling relative path of non-dir')
 331         if sibling: trim = os.path.dirname
 332         else: trim = lambda x: x
 333         for tbp in [False,True]:
 334                 if parent.path[tbp] is None: continue
 335                 trimmed = trim(parent.path[tbp])
 336                 if trimmed: trimmed += '/'
 337                 p.path[tbp] = trimmed + leaf
 338                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 339                         p.file[tbp] = p.path[tbp]
 340
 341 class InputFile(AutoFile):
 342  def _init(p, what, spec, spec_tbp=False):
 343         AutoFile.__init__(p, what)
 344         p.spec = spec
 345         p.spec_tbp = spec_tbp
 346         p.path[spec_tbp] = spec
 347         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 348  def __init__(p, what, spec, spec_tbp=False):
 349         p._init(what,spec,spec_tbp)
 350         p._constructed()
 351
 352 class InputDir(InputFile):
 353  def __init__(p, what, spec, spec_tbp=False):
 354         InputFile._init(p,what,spec,spec_tbp)
 355         p.dir = '/'
 356         p._constructed()
 357
 358 class OutputFile(AutoFile):
 359  def _init(p, what, spec, spec_tbp=False):
 360         AutoFile.__init__(p, what)
 361         p.spec = spec
 362         p.spec_tbp = spec_tbp
 363         p.path[spec_tbp] = spec
 364  def __init__(p, what, spec, spec_tbp=False):
 365         p._init(what,spec,spec_tbp)
 366         p._constructed()
 367
 368 class OutputDir(OutputFile):
 369  def __init__(p, what, spec, spec_tbp=False):
 370         OutputFile._init(p,what,spec,spec_tbp)
 371         p.dir = '/'
 372         p._constructed()
 373
 374 class RelativeInputFile(AutoFile):
 375  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 376         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 377         p._constructed()
 378
 379 class RelativeOutputFile(AutoFile):
 380  def __init__(p, what, parent, leaf, sibling=False):
 381         p._relative_init(what, parent, leaf, None, False, sibling)
 382         p._constructed()
 383
 384 class TemporaryFile(AutoFile):
 385  def __init__(p, what):
 386         AutoFile.__init__(p, what)
 387         p._constructed()
 388
 389 class TemporaryDir(AutoFile):
 390  def __init__(p, what):
 391         AutoFile.__init__(p, what)
 392         p.dir = '/'
 393         p._constructed()
 394
 395 #---------- parsing and representation of the arguments
 396
 397 class Action:
 398  def __init__(a, kind, af, arghandling, what):
 399         # extra attributes get added during processing
 400         a.kind = kind
 401         a.af = af
 402         a.ah = arghandling
 403         a.what = what
 404         a.missing_tests_control = False
 405  def __repr__(a):
 406         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 407
 408 def parse_args():
 409         global opts
 410         global n_non_actions # argh, stupid python scoping rules
 411         usage = "%prog <options> --- <virt-server>..."
 412         parser = OptionParser(usage=usage)
 413         pa = parser.add_option
 414         pe = parser.add_option
 415
 416         arghandling = {
 417                 'dsc_tests': True,
 418                 'dsc_filter': '*',
 419                 'deb_forbuilds': 'auto',
 420                 'deb_fortests': 'auto',
 421                 'tb': False,
 422                 'override_control': None
 423         }
 424         initial_arghandling = arghandling.copy()
 425         n_non_actions = 0
 426
 427         #----------
 428         # actions (ie, test sets to run, sources to build, binaries to use):
 429
 430         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 431                 global n_non_actions
 432                 parser.largs.append((value,kindpath))
 433                 n_non_actions += not(is_act)
 434
 435         def pa_action(long, metavar, kindpath, help, is_act=True):
 436                 pa('','--'+long, action='callback', callback=cb_action,
 437                         nargs=1, type='string',
 438                         callback_args=(long,kindpath,is_act), help=help)
 439
 440         pa_action('built-tree',         'TREE', '@/',
 441                 help='run tests from build tree TREE')
 442
 443         pa_action('unbuilt-tree',       'TREE', '@//',
 444                 help='run tests from build tree TREE')
 445
 446         pa_action('source',             'DSC', '@.dsc',
 447                 help='build DSC and use its tests and/or'
 448                     ' generated binary packages')
 449
 450         pa_action('binary',             'DEB', '@.deb',
 451                help='use binary package DEB according'
 452                     ' to most recent --binaries-* settings')
 453
 454         def cb_actnoarg(op,optstr,value,parser, largsentry):
 455                 parser.largs.append(largsentry)
 456         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 457                 callback_args=((None, ('instantiate',)),),
 458                 help='instantiate testbed now (during testing phase)'
 459                      ' and install packages'
 460                      ' selected for automatic installation, even'
 461                      ' if this might apparently not be required otherwise')
 462
 463         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 464                help='run tests from control file CONTROL instead,'
 465                     ' (applies to next test suite only)')
 466
 467         #----------
 468         # argument handling settings (what ways to use action
 469         #  arguments, and pathname processing):
 470
 471         def cb_setah(option, opt_str, value, parser, toset,setval):
 472                 if type(setval) == list:
 473                         if not value in setval:
 474                                 parser.error('value for %s option (%s) is not '
 475                                  'one of the permitted values (%s)' %
 476                                  (value, opt_str, setval.join(' ')))
 477                 elif setval is not None:
 478                         value = setval
 479                 for v in toset:
 480                         arghandling[v] = value
 481                 parser.largs.append(arghandling.copy())
 482
 483         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 484                 type = metavar
 485                 if type is not None: type = 'string'
 486                 pa('',long, action='callback', callback=cb_setah,
 487                    callback_args=(affected,effect), **kwargs)
 488
 489         #---- paths: host or testbed:
 490         #
 491         pa_setah('--paths-testbed', ['tb'],True,
 492                 help='subsequent path specifications refer to the testbed')
 493         pa_setah('--paths-host', ['tb'],False,
 494                 help='subsequent path specifications refer to the host')
 495
 496         #---- source processing settings:
 497
 498         pa_setah('--sources-tests', ['dsc_tests'],True,
 499                 help='run tests from builds of subsequent sources')
 500         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 501                 help='do not run tests from builds of subsequent sources')
 502
 503         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 504                 type='string', metavar='PATTERN-LIST',
 505                 help='from subsequent sources, use binaries matching'
 506                      ' PATTERN-LIST (comma-separated glob patterns)'
 507                      ' according to most recent --binaries-* settings')
 508         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 509                 help='from subsequent sources, do not use any binaries')
 510
 511         #---- binary package processing settings:
 512
 513         def pa_setahbins(long,toset,how):
 514          pa_setah(long, toset,['ignore','auto','install'],
 515                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 516                 help=how+' ignore binaries, install them as needed'
 517                         ' for dependencies, or unconditionally install'
 518                         ' them, respectively')
 519         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 520         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 521         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 522
 523         #----------
 524         # general options:
 525
 526         def cb_vserv(op,optstr,value,parser):
 527                 parser.values.vserver = list(parser.rargs)
 528                 del parser.rargs[:]
 529
 530         def cb_path(op,optstr,value,parser, constructor,long,dir):
 531                 name = long.replace('-','_')
 532                 af = constructor(long, value,arghandling['tb'])
 533                 setattr(parser.values, name, af)
 534
 535         def pa_path(long, constructor, help, dir=False):
 536                 pa('','--'+long, action='callback', callback=cb_path,
 537                         callback_args=(constructor,long,dir),
 538                         nargs=1, type='string',
 539                         help=help, metavar='PATH')
 540
 541         pa_path('output-dir', OutputDir, dir=True,
 542                 help='write stderr/out files in PATH')
 543
 544         pa('','--tmp-dir',              type='string', dest='tmpdir',
 545                 help='write temporary files to TMPDIR, emptying it'
 546                      ' beforehand and leaving it behind at the end')
 547         pa('','--log-file',             type='string', dest='logfile',
 548                 help='write the log LOGFILE, emptying it beforehand,'
 549                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 550         pa('','--summary-file',         type='string', dest='summary',
 551                 help='write a summary report to SUMMARY,'
 552                      ' emptying it beforehand')
 553
 554         pa('','--user',                 type='string', dest='user',
 555                 help='run tests as USER (needs root on testbed)')
 556         pa('','--gain-root',            type='string', dest='gainroot',
 557                 help='prefix debian/rules binary with GAINROOT')
 558         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 559         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 560         pa('','--gnupg-home',           type='string', dest='gnupghome',
 561                 default='~/.autopkgtest/gpg',
 562                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 563                         " signing private apt archive);"
 564                         " `fresh' means generate new key each time.")
 565
 566         #----------
 567         # actual meat:
 568
 569         class SpecialOption(optparse.Option): pass
 570         vs_op = SpecialOption('','--VSERVER-DUMMY')
 571         vs_op.action = 'callback'
 572         vs_op.type = None
 573         vs_op.default = None
 574         vs_op.nargs = 0
 575         vs_op.callback = cb_vserv
 576         vs_op.callback_args = ( )
 577         vs_op.callback_kwargs = { }
 578         vs_op.help = 'introduces virtualisation server and args'
 579         vs_op._short_opts = []
 580         vs_op._long_opts = ['---']
 581
 582         pa(vs_op)
 583
 584         (opts,args) = parser.parse_args()
 585         if not hasattr(opts,'vserver'):
 586                 parser.error('you must specifiy --- <virt-server>...')
 587         if n_non_actions >= len(parser.largs):
 588                 parser.error('nothing to do specified')
 589
 590         arghandling = initial_arghandling
 591         opts.actions = []
 592         ix = 0
 593         for act in args:
 594                 if type(act) == dict:
 595                         arghandling = act
 596                         continue
 597                 elif type(act) == tuple:
 598                         pass
 599                 elif type(act) == str:
 600                         act = (act,act)
 601                 else:
 602                         raise ("unknown action in list `%s' having"
 603                               " type `%s'" % (act, type(act)))
 604                 (pathstr, kindpath) = act
 605
 606                 constructor = InputFile
 607                 if type(kindpath) is tuple:             kind = kindpath[0]
 608                 elif kindpath.endswith('.deb'):         kind = 'deb'
 609                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 610                 elif kindpath.endswith('//'):
 611                         kind = 'ubtree'
 612                         constructor = InputDir
 613                 elif kindpath.endswith('/'):
 614                         kind = 'tree'
 615                         constructor = InputDir
 616                 else: parser.error("do not know how to handle filename `%s';"
 617                         " specify --source --binary or --build-tree" %
 618                         kindpath)
 619
 620                 what = '%s%s' % (kind,ix); ix += 1
 621
 622                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 623                 else: fwhatx = '-'+kind
 624
 625                 if pathstr is None: af = None
 626                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 627                 opts.actions.append(Action(kind, af, arghandling, what))
 628
 629 def setup_trace():
 630         global trace_stream, tmpdir, summary_stream
 631
 632         if opts.tmpdir is not None:
 633                 rmtree('tmpdir(specified)',opts.tmpdir)
 634                 mkdir_okexist(opts.tmpdir, 0700)
 635                 tmpdir = opts.tmpdir
 636         else:
 637                 assert(tmpdir is None)
 638                 tmpdir = tempfile.mkdtemp()
 639
 640         if opts.logfile is None:
 641                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 642                         opts.logfile = opts.output_dir.spec + '/log'
 643                 elif opts.tmpdir is not None:
 644                         opts.logfile = opts.tmpdir + '/log'
 645         if opts.logfile is not None:
 646                 trace_stream = open(opts.logfile, 'w', 0)
 647         if opts.summary is not None:
 648                 summary_stream = open(opts.summary, 'w', 0)
 649
 650         debug('options: '+`opts`, 1)
 651
 652 def finalise_options():
 653         global opts, tb, build_essential
 654
 655         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 656                 opts.user = ''
 657
 658         if opts.user is None:
 659                 su = 'suggested-normal-user='
 660                 ul = [
 661                         e[len(su):]
 662                         for e in testbed.caps
 663                         if e.startswith(su)
 664                         ]
 665                 if ul:
 666                         opts.user = ul[0]
 667                 else:
 668                         opts.user = ''
 669
 670         if opts.user:
 671                 if 'root-on-testbed' not in testbed.caps:
 672                         pstderr("warning: virtualisation"
 673                                 " system does not offer root on testbed,"
 674                                 " but --user option specified: failure likely")
 675                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 676         else:
 677                 opts.user_wrap = lambda x: x
 678
 679         if opts.gainroot is None:
 680                 opts.gainroot = ''
 681                 if (opts.user or
 682                     'root-on-testbed' not in testbed.caps):
 683                         opts.gainroot = 'fakeroot'
 684                         build_essential += ['fakeroot']
 685
 686         if opts.gnupghome.startswith('~/'):
 687                 try: home = os.environ['HOME']
 688                 except KeyError:
 689                         parser.error("HOME environment variable"
 690                                 " not set, needed for --gnupghome=`%s"
 691                                 % opts.gnupghome)
 692                 opts.gnupghome = home + opts.gnupghome[1:]
 693         elif opts.gnupghome == 'fresh':
 694                 opts.gnupghome = None
 695
 696 #---------- testbed management - the Testbed class
 697
 698 class Testbed:
 699  def __init__(tb):
 700         tb.sp = None
 701         tb.lastsend = None
 702         tb.scratch = None
 703         tb.modified = False
 704         tb.blamed = []
 705         tb._ephemeral = []
 706         tb._debug('init')
 707         tb._need_reset_apt = False
 708  def _debug(tb, m, minlevel=0):
 709         debug('** '+m, minlevel)
 710  def start(tb):
 711         tb._debug('start')
 712         p = subprocess.PIPE
 713         debug_subprocess('vserver', opts.vserver)
 714         tb._errplumb = Errplumb(True)
 715         tb.sp = subprocess.Popen(opts.vserver,
 716                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 717         tb.expect('ok')
 718         tb.caps = tb.commandr('capabilities')
 719  def stop(tb):
 720         tb._debug('stop')
 721         tb.close()
 722         if tb.sp is None: return
 723         ec = tb.sp.returncode
 724         if ec is None:
 725                 tb.sp.stdout.close()
 726                 tb.send('quit')
 727                 tb.sp.stdin.close()
 728                 ec = tb.sp.wait()
 729         if ec:
 730                 tb.bomb('testbed gave exit status %d after quit' % ec)
 731         tb._errplumb.wait()
 732  def open(tb):
 733         tb._debug('open, scratch=%s' % tb.scratch)
 734         if tb.scratch is not None: return
 735         pl = tb.commandr('open')
 736         tb.scratch = InputDir('tb-scratch', pl[0], True)
 737         tb.deps_processed = []
 738  def mungeing_apt(tb):
 739         if not 'revert' in tb.caps:
 740                 tb._need_reset_apt = True
 741  def reset_apt(tb):
 742         if not tb._need_reset_apt: return
 743         what = 'aptget-update-reset'
 744         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 745         rc = tb.execute(what, cmdl, kind='install')
 746         if rc:
 747                 pstderr("\n" "warning: failed to restore"
 748                         " testbed apt cache, exit code %d" % rc)
 749         tb._need_reset_apt = False
 750  def close(tb):
 751         tb._debug('close, scratch=%s' % tb.scratch)
 752         if tb.scratch is None: return
 753         tb.scratch = None
 754         if tb.sp is None: return
 755         tb.command('close')
 756  def prepare1(tb, deps_new):
 757         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 758                 (tb.modified, tb.deps_processed, deps_new), 1)
 759         if 'revert' in tb.caps and (tb.modified or
 760             [d for d in tb.deps_processed if d not in deps_new]):
 761                 for af in tb._ephemeral: af.read(False)
 762                 tb._debug('reset **')
 763                 tb.command('revert')
 764                 tb.blamed = []
 765                 for af in tb._ephemeral: af.invalidate(True)
 766         tb.modified = False
 767  def prepare2(tb, deps_new):
 768         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 769         binaries.publish()
 770         tb._install_deps(deps_new)
 771  def prepare(tb, deps_new):
 772         tb.prepare1(deps_new)
 773         tb.prepare2(deps_new)
 774  def register_ephemeral(tb, af):
 775         tb._ephemeral.append(af)
 776  def _install_deps(tb, deps_new):
 777         tb._debug(' installing dependencies '+`deps_new`, 1)
 778         tb.deps_processed = deps_new
 779         if not deps_new: return
 780         dstr = ', '.join(deps_new)
 781         script = binaries.apt_pkg_gdebi_script(
 782                 dstr, [[
 783                 'from GDebi.DebPackage import DebPackage',
 784                 'd = DebPackage(cache)',
 785                 'res = d.satisfyDependsStr(arg)',
 786                 ]])
 787         cmdl = ['python','-c',script]
 788         what = 'install-deps'
 789         rc = testbed.execute(what+'-debinstall', cmdl, script=script,
 790                                 kind='install')
 791         if rc: badpkg('dependency install failed, exit code %d' % rc)
 792  def needs_reset(tb):
 793         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 794         tb.modified = True
 795  def blame(tb, m):
 796         tb._debug('blame += %s' % m, 1)
 797         tb.blamed.append(m)
 798  def bomb(tb, m):
 799         tb._debug('bomb %s' % m)
 800         if tb.sp is not None:
 801                 tb.sp.stdout.close()
 802                 tb.sp.stdin.close()
 803                 ec = tb.sp.wait()
 804                 if ec: pstderr('adt-run: testbed failing,'
 805                                 ' exit status %d' % ec)
 806         tb.sp = None
 807         raise Quit(16, 'testbed failed: %s' % m)
 808  def send(tb, string):
 809         tb.sp.stdin
 810         try:
 811                 debug('>> '+string, 2)
 812                 print >>tb.sp.stdin, string
 813                 tb.sp.stdin.flush()
 814                 tb.lastsend = string
 815         except:
 816                 (type, value, dummy) = sys.exc_info()
 817                 tb.bomb('cannot send to testbed: %s' % traceback.
 818                         format_exception_only(type, value))
 819  def expect(tb, keyword, nresults=None):
 820         l = tb.sp.stdout.readline()
 821         if not l: tb.bomb('unexpected eof from the testbed')
 822         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 823         l = l.rstrip('\n')
 824         debug('<< '+l, 2)
 825         ll = l.split()
 826         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 827         if ll[0] != keyword:
 828                 if tb.lastsend is None:
 829                         tb.bomb("got banner `%s', expected `%s...'" %
 830                                 (l, keyword))
 831                 else:
 832                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 833                                 (tb.lastsend, l, keyword))
 834         ll = ll[1:]
 835         if nresults is not None and len(ll) != nresults:
 836                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 837                         " expected %d result parameters" %
 838                         (string, l, len(ll), nresults))
 839         return ll
 840  def commandr(tb, cmd, args=(), nresults=None):
 841         # pass args=[None,...] or =(None,...) to avoid more url quoting
 842         if type(cmd) is str: cmd = [cmd]
 843         if len(args) and args[0] is None: args = args[1:]
 844         else: args = map(urllib.quote, args)
 845         al = cmd + args
 846         tb.send(string.join(al))
 847         ll = tb.expect('ok', nresults)
 848         rl = map(urllib.unquote, ll)
 849         return rl
 850  def command(tb, cmd, args=()):
 851         tb.commandr(cmd, args, 0)
 852  def commandr1(tb, cmd, args=()):
 853         rl = tb.commandr(cmd, args, 1)
 854         return rl[0]
 855  def execute(tb, what, cmdl,
 856                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 857                 script=False, tmpdir=None, kind='short'):
 858         # Options for script:
 859         #   False - do not call debug_subprocess, no synch. reporting required
 860         #   None or string - call debug_subprocess with that value,
 861         #                       plumb stderr through synchronously if possible
 862         # Options for se:
 863         #   None - usual Errplumb (output is of kind 2c)
 864         #   string - path on testbed (output is of kind 2d)
 865
 866         timeout = timeouts[kind]
 867
 868         if script is not False: debug_subprocess(what, cmdl, script=script)
 869         if cwd is None: cwd = tb.scratch.write(True)
 870
 871         xdump = None
 872         if se is None:
 873                 ep = Errplumb()
 874                 se_catch = TemporaryFile(what+'-xerr')
 875                 se_use = se_catch.write(True)
 876                 if not opts.quiet: xdump = 'debug=2-2'
 877                 elif trace_stream is not None:
 878                         xdump = 'debug=2-%d' % trace_stream.fileno()
 879         else:
 880                 ep = None
 881                 se_catch = None
 882                 se_use = se
 883
 884         cmdl = [None,
 885                 ','.join(map(urllib.quote, cmdl)),
 886                 si, so, se_use, cwd]
 887
 888         if timeout is not None and timeout > 0:
 889                 cmdl.append('timeout=%d' % timeout)
 890
 891         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 892         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 893         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 894
 895         rc = tb.commandr1('execute', cmdl)
 896         try: rc = int(rc)
 897         except ValueError: bomb("execute for %s gave invalid response `%s'"
 898                                         % (what,rc))
 899
 900         if se_catch is not None:
 901                 debug_file(se_catch.read())
 902         if ep is not None:
 903                 ep.wait()
 904
 905         return rc
 906
 907 #---------- representation of test control files: Field*, Test, etc.
 908
 909 class FieldBase:
 910  def __init__(f, fname, stz, base, tnames, vl):
 911         assert(vl)
 912         f.stz = stz
 913         f.base = base
 914         f.tnames = tnames
 915         f.vl = vl
 916  def words(f):
 917         def distribute(vle):
 918                 (lno, v) = vle
 919                 r = v.split()
 920                 r = map((lambda w: (lno, w)), r)
 921                 return r
 922         return flatten(map(distribute, f.vl))
 923  def atmostone(f):
 924         if len(vl) == 1:
 925                 (f.lno, f.v) = vl[0]
 926         else:
 927                 raise Unsupported(f.vl[1][0],
 928                         'only one %s field allowed' % fn)
 929         return f.v
 930
 931 class FieldIgnore(FieldBase):
 932  def parse(f): pass
 933
 934 class Restriction:
 935  def __init__(r,rname,base): pass
 936
 937 class Restriction_rw_build_tree(Restriction): pass
 938 class Restriction_breaks_testbed(Restriction):
 939  def __init__(r, rname, base):
 940         if 'revert' not in testbed.caps:
 941                 raise Unsupported(f.lno,
 942                         'Test breaks testbed but testbed cannot revert')
 943 class Restriction_needs_root(Restriction):
 944  def __init__(r, rname, base):
 945         if 'root-on-testbed' not in testbed.caps:
 946                 raise Unsupported(f.lno,
 947                         'Test needs root on testbed which is not available')
 948
 949 class Field_Restrictions(FieldBase):
 950  def parse(f):
 951         for wle in f.words():
 952                 (lno, rname) = wle
 953                 nrname = rname.replace('-','_')
 954                 try: rclass = globals()['Restriction_'+nrname]
 955                 except KeyError: raise Unsupported(lno,
 956                         'unknown restriction %s' % rname)
 957                 r = rclass(nrname, f.base)
 958                 f.base['restriction_names'].append(rname)
 959                 f.base['restrictions'].append(r)
 960
 961 class Field_Features(FieldIgnore):
 962  def parse(f):
 963         for wle in f.words():
 964                 (lno, fname) = wle
 965                 f.base['feature_names'].append(fname)
 966                 nfname = fname.replace('-','_')
 967                 try: fclass = globals()['Feature_'+nfname]
 968                 except KeyError: continue
 969                 ft = fclass(nfname, f.base)
 970                 f.base['features'].append(ft)
 971
 972 class Field_Tests(FieldIgnore): pass
 973
 974 class Field_Depends(FieldBase):
 975  def parse(f):
 976         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 977         dl = map(lambda x: x.strip(),
 978                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 979         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
 980         for d in dl:
 981                 if re.search(d):
 982                         badpkg("Test Depends field contains dependency"
 983                                " `%s' with invalid characters" % d)
 984         f.base['depends'] = dl
 985
 986 class Field_Tests_directory(FieldBase):
 987  def parse(f):
 988         td = atmostone(f)
 989         if td.startswith('/'): raise Unspported(f.lno,
 990                 'Tests-Directory may not be absolute')
 991         f.base['testsdir'] = td
 992
 993 def run_tests(stanzas, tree):
 994         global errorcode, testbed
 995         if stanzas == ():
 996                 report('*', 'SKIP no tests in this package')
 997                 errorcode |= 8
 998         for stanza in stanzas:
 999                 tests = stanza[' tests']
1000                 if not tests:
1001                         report('*', 'SKIP package has metadata but no tests')
1002                         errorcode |= 8
1003                 for t in tests:
1004                         t.prepare()
1005                         t.run(tree)
1006                         if 'breaks-testbed' in t.restriction_names:
1007                                 testbed.needs_reset()
1008                 testbed.needs_reset()
1009
1010 class Test:
1011  def __init__(t, tname, base, act):
1012         if '/' in tname: raise Unsupported(base[' lno'],
1013                 'test name may not contain / character')
1014         for k in base: setattr(t,k,base[k])
1015         t.tname = tname
1016         t.act = act
1017         t.what = act.what+'t-'+tname
1018         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1019         else: t.path = tname
1020         t._debug('constructed; path=%s' % t.path)
1021         t._debug(' .depends=%s' % t.depends)
1022  def _debug(t, m):
1023         debug('& %s: %s' % (t.what, m))
1024  def report(t, m):
1025         report(t.what, m)
1026  def reportfail(t, m):
1027         global errorcode
1028         errorcode |= 4
1029         report(t.what, 'FAIL ' + m)
1030  def prepare(t):
1031         t._debug('preparing')
1032         dn = []
1033         for d in t.depends:
1034                 t._debug(' processing dependency '+d)
1035                 if not '@' in d:
1036                         t._debug('  literal dependency '+d)
1037                         dn.append(d)
1038                 else:
1039                         for (pkg,bin) in t.act.binaries:
1040                                 d = d.replace('@',pkg)
1041                                 t._debug('  synthesised dependency '+d)
1042                                 dn.append(d)
1043         testbed.prepare(dn)
1044  def run(t, tree):
1045         t._debug('[----------------------------------------')
1046         def stdouterr(oe):
1047                 idstr = t.what + '-' + oe
1048                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1049                         use_dir = opts.output_dir
1050                 else:
1051                         use_dir = testbed.scratch
1052                 return RelativeOutputFile(idstr, use_dir, idstr)
1053
1054         if hasattr(t.act,'work'): t.act.work.write(True)
1055         tree.read(True)
1056
1057         af = RelativeInputFile(t.what, tree, t.path)
1058         so = stdouterr('stdout')
1059         se = stdouterr('stderr')
1060
1061         tf = af.read(True)
1062         tmpdir = None
1063
1064         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1065         if rc: bomb('failed to chmod +x %s' % tf)
1066
1067         if 'needs-root' not in t.restriction_names and opts.user is not None:
1068                 tfl = ['su',opts.user,'-c',tf]
1069                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1070                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1071                 if opts.user: script += '; chown %s "$1"' % opts.user
1072                 if 'rw-build-tree' in t.restriction_names:
1073                         script += '; chown -R %s "$2"' % opts.user
1074                 rc = testbed.execute('mktmpdir-'+t.what,
1075                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1076                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1077                                 % (tmpdir, rc))
1078         else:
1079                 tfl = [tf]
1080
1081         rc = testbed.execute('test-'+t.what, tfl,
1082                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1083                 tmpdir=tmpdir, kind='test')
1084
1085         so_read = so.read()
1086         se_read = se.read()
1087
1088         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1089         stab = os.stat(se_read)
1090         if stab.st_size != 0:
1091                 l = open(se_read).readline()
1092                 l = l.rstrip('\n \t\r')
1093                 if len(l) > 35: l = l[:35] + '...'
1094                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1095                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1096                 debug_file(se_read)
1097         elif rc != 0:
1098                 t.reportfail('non-zero exit status %d' % rc)
1099         else:
1100                 t.report('PASS')
1101
1102         stab = os.stat(so_read)
1103         if stab.st_size != 0:
1104                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1105                 debug_file(so_read)
1106
1107         t._debug('----------------------------------------]')
1108
1109 def read_control(act, tree, control_override):
1110         stanzas = [ ]
1111
1112         if control_override is not None:
1113                 control_af = control_override
1114                 testbed.blame('arg:'+control_override.spec)
1115         else:
1116                 if act.missing_tests_control:
1117                         return ()
1118                 control_af = RelativeInputFile(act.what+'-testcontrol',
1119                         tree, 'debian/tests/control')
1120         try:
1121                 control = open(control_af.read(), 'r')
1122         except (IOError,OSError), oe:
1123                 if oe[0] != errno.ENOENT: raise
1124                 return []
1125
1126         lno = 0
1127         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1128         stz = None      # stz[field_name][index] = (lno, value)
1129                         # special field names:
1130                         # stz[' lno'] = number
1131                         # stz[' tests'] = list of Test objects
1132         def end_stanza(stz):
1133                 if stz is None: return
1134                 stz[' errs'] = 0
1135                 stanzas.append(stz)
1136                 stz = None
1137                 hcurrent = None
1138
1139         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1140         while 1:
1141                 l = control.readline()
1142                 if not l: break
1143                 lno += 1
1144                 if not l.endswith('\n'): badctrl('unterminated line')
1145                 if regexp.compile('\s*\#').match(l): continue
1146                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1147                 initmat = initre.match(l)
1148                 if initmat:
1149                         (fname, l) = initmat.groups()
1150                         fname = string.capwords(fname)
1151                         if stz is None:
1152                                 stz = { ' lno': lno, ' tests': [] }
1153                         if not stz.has_key(fname): stz[fname] = [ ]
1154                         hcurrent = stz[fname]
1155                 elif regexp.compile('\s').match(l):
1156                         if not hcurrent: badctrl('unexpected continuation')
1157                 else:
1158                         badctrl('syntax error')
1159                 hcurrent.append((lno, l))
1160         end_stanza(stz)
1161
1162         def testbadctrl(stz, lno, m):
1163                 report_badctrl(lno, m)
1164                 stz[' errs'] += 1
1165
1166         for stz in stanzas:
1167                 try:
1168                         try: tnames = stz['Tests']
1169                         except KeyError:
1170                                 tnames = ['*']
1171                                 raise Unsupported(stz[' lno'],
1172                                         'no Tests field')
1173                         tnames = map((lambda lt: lt[1]), tnames)
1174                         tnames = string.join(tnames).split()
1175                         base = {
1176                                 'restriction_names': [],
1177                                 'restrictions': [],
1178                                 'feature_names': [],
1179                                 'features': [],
1180                                 'testsdir': 'debian/tests',
1181                                 'depends' : '@'
1182                         }
1183                         for fname in stz.keys():
1184                                 if fname.startswith(' '): continue
1185                                 vl = stz[fname]
1186                                 try: fclass = globals()['Field_'+
1187                                         fname.replace('-','_')]
1188                                 except KeyError: raise Unsupported(vl[0][0],
1189                                         'unknown metadata field %s' % fname)
1190                                 f = fclass(stz, fname, base, tnames, vl)
1191                                 f.parse()
1192                         for tname in tnames:
1193                                 t = Test(tname, base, act)
1194                                 stz[' tests'].append(t)
1195                 except Unsupported, u:
1196                         for tname in tnames: u.report(tname)
1197                         continue
1198
1199         return stanzas
1200
1201 def print_exception(ei, msgprefix=''):
1202         if msgprefix: pstderr(msgprefix)
1203         (et, q, tb) = ei
1204         if et is Quit:
1205                 pstderr('adt-run: ' + q.m)
1206                 psummary('quitting: '+q.m)
1207                 return q.ec
1208         else:
1209                 pstderr("adt-run: unexpected, exceptional, error:")
1210                 psummary('quitting: unexpected error, consult transcript')
1211                 traceback.print_exc(None, sys.stderr)
1212                 if trace_stream is not None:
1213                         traceback.print_exc(None, trace_stream)
1214                 return 20
1215
1216 def cleanup():
1217         global trace_stream
1218         try:
1219                 if testbed is not None:
1220                         testbed.reset_apt()
1221                         testbed.stop()
1222                 if opts.tmpdir is None and tmpdir is not None:
1223                         rmtree('tmpdir', tmpdir)
1224                 if trace_stream is not None:
1225                         trace_stream.close()
1226                         trace_stream = None
1227         except:
1228                 print_exception(sys.exc_info(),
1229                         '\nadt-run: error cleaning up:\n')
1230                 os._exit(20)
1231
1232 #---------- registration, installation etc. of .deb's: Binaries
1233
1234 def determine_package(act):
1235         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1236         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1237         if rc: badpkg('failed to parse binary package, code %d' % rc)
1238         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1239         act.pkg = None
1240         for l in output.split('\n'):
1241                 m = re.match(l)
1242                 if not m: continue
1243                 if act.pkg: badpkg('two Package: lines in control file')
1244                 act.pkg = m.groups()[0]
1245         if not act.pkg: badpkg('no good Package: line in control file')
1246
1247 class Binaries:
1248  def __init__(b):
1249         b.dir = TemporaryDir('binaries')
1250         b.dir.write()
1251         ok = False
1252
1253         if opts.gnupghome is None:
1254                 opts.gnupghome = tmpdir+'/gnupg'
1255
1256         b._debug('initialising')
1257         try:
1258                 for x in ['pubring','secring']:
1259                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1260                 ok = True
1261         except (IOError,OSError), oe:
1262                 if oe.errno != errno.ENOENT: raise
1263
1264         if ok: b._debug('no key generation needed')
1265         else: b.genkey()
1266
1267  def _debug(b, s):
1268         debug('* '+s)
1269
1270  def genkey(b):
1271         b._debug('preparing for key generation')
1272
1273         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1274         mkdir_okexist(opts.gnupghome, 0700)
1275
1276         script = '''
1277   exec >&2
1278   cd "$1"
1279   cat <<"END" >key-gen-params
1280 Key-Type: DSA
1281 Key-Length: 1024
1282 Key-Usage: sign
1283 Name-Real: autopkgtest per-run key
1284 Name-Comment: do not trust this key
1285 Name-Email: autopkgtest@example.com
1286 END
1287   set -x
1288   gpg --homedir="$1" --batch --gen-key key-gen-params
1289                 '''
1290         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1291         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1292         if rc: bomb('key generation failed, code %d' % rc)
1293
1294  def apt_configs(b):
1295         return {
1296                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1297         }
1298
1299  def apt_pkg_gdebi_script(b, arg, middle):
1300         script = [
1301                 'import apt_pkg',
1302                 'import urllib',
1303                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1304                 ]
1305         for (k,v) in b.apt_configs().iteritems():
1306                 v = urllib.quote(v)
1307                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1308                                 % (k, v))
1309         script += [
1310                 'from GDebi.Cache import Cache',
1311                 'cache = Cache()',
1312                 ]
1313         for m in middle:
1314                 script += m + [
1315                 'print res',
1316                 'print d.missingDeps',
1317                 'print d.requiredChanges',
1318                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1319                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1320                           'd._failureString)',
1321                 'cache.commit()',
1322                 ''
1323                 ]
1324         return '\n'.join(script)
1325  def apt_get(b):
1326         ag = ['apt-get','-qy']
1327         for kv in b.apt_configs().iteritems():
1328                 ag += ['-o', '%s=%s' % kv]
1329         return ' '.join(ag)
1330
1331  def reset(b):
1332         b._debug('reset')
1333         rmtree('binaries', b.dir.read())
1334         b.dir.invalidate()
1335         b.dir.write()
1336         b.install = []
1337         b.blamed = []
1338         b.registered = set()
1339
1340  def register(b, act, pkg, af, forwhat, blamed):
1341         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1342                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1343
1344         if act.ah['deb_'+forwhat] == 'ignore': return
1345
1346         b.blamed += testbed.blamed
1347
1348         leafname = pkg+'.deb'
1349         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1350
1351         try: os.remove(dest.write())
1352         except (IOError,OSError), oe:
1353                 if oe.errno != errno.ENOENT: raise e
1354
1355         try: os.link(af.read(), dest.write())
1356         except (IOError,OSError), oe:
1357                 if oe.errno != errno.EXDEV: raise e
1358                 shutil.copy(af.read(), dest)
1359
1360         if act.ah['deb_'+forwhat] == 'install':
1361                 b.install.append(pkg)
1362
1363         b.registered.add(pkg)
1364
1365  def publish(b):
1366         b._debug('publish')
1367
1368         script = '''
1369   exec >&2
1370   cd "$1"
1371   apt-ftparchive packages . >Packages
1372   gzip <Packages >Packages.gz
1373   apt-ftparchive release . >Release
1374   rm -f Release.gpg
1375   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1376   gpg --homedir="$2" --batch --export >archive-key.pgp
1377                 '''
1378         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1379         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1380         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1381
1382         b.dir.invalidate(True)
1383         apt_source = b.dir.read(True)
1384
1385         so = TemporaryFile('vlds')
1386         script = '''
1387   exec 3>&1 >&2
1388   apt-key add archive-key.pgp
1389   echo "deb file://'''+apt_source+''' /" >sources.list
1390   cat /etc/apt/sources.list >>sources.list
1391   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1392     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1393   fi
1394   '''+ b.apt_get() +''' update >&2
1395   cat /var/lib/dpkg/status >&3
1396                 '''
1397         testbed.mungeing_apt()
1398         rc = testbed.execute('apt-key', ['sh','-ec',script],
1399                                 so=so.write(True), cwd=b.dir.write(True),
1400                                 script=script, kind='install')
1401         if rc: bomb('apt setup failed with exit code %d' % rc)
1402
1403         testbed.blamed += b.blamed
1404
1405         b._debug('publish reinstall checking...')
1406         pkgs_reinstall = set()
1407         pkg = None
1408         for l in open(so.read()):
1409                 if l.startswith('Package: '):
1410                         pkg = l[9:].rstrip()
1411                 elif l.startswith('Status: install '):
1412                         if pkg in b.registered:
1413                                 pkgs_reinstall.add(pkg)
1414                                 b._debug(' publish reinstall needs '+pkg)
1415
1416         if pkgs_reinstall:
1417                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1418                 what = 'apt-get-reinstall'
1419                 cmdl = (b.apt_get() + ' --reinstall install '+
1420                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1421                 cmdl = ['sh','-c',cmdl]
1422                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1423                 if rc: badpkg("installation of basic binarries failed,"
1424                                 " exit code %d" % rc)
1425
1426         b._debug('publish install...')
1427         for pkg in b.install:
1428                 what = 'apt-get-install-%s' % pkg
1429                 testbed.blame(pkg)
1430                 cmdl = b.apt_get() + ' install ' + pkg + ' >&2'
1431                 cmdl = ['sh','-c',cmdl]
1432                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1433                 if rc: badpkg("installation of %s failed, exit code %d"
1434                                 % (pkg, rc))
1435
1436         b._debug('publish done')
1437
1438 #---------- processing of sources (building)
1439
1440 def source_rules_command(act,script,what,which,work,cwd,
1441                                 results_lines=0,xargs=[]):
1442         script = [      "exec 3>&1 >&2",
1443                         "set -x"        ] + script
1444         script = '\n'.join(script)
1445         so = TemporaryFile('%s-%s-results' % (what,which))
1446         rc = testbed.execute('%s-%s' % (what,which),
1447                         ['sh','-ec',script]+xargs, script=script,
1448                         so=so.write(True), cwd=cwd, kind='build')
1449         results = open(so.read()).read().rstrip('\n')
1450         if len(results): results = results.split("\n")
1451         else: results = []
1452         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1453         if results_lines is not None and len(results) != results_lines:
1454                 badpkg("got %d lines of results from %s where %d expected"
1455                         % (len(results), which, results_lines))
1456         if results_lines==1: return results[0]
1457         return results
1458
1459 def build_source(act, control_override):
1460         act.blame = 'arg:'+act.af.spec
1461         testbed.blame(act.blame)
1462         testbed.prepare1([])
1463         testbed.needs_reset()
1464
1465         what = act.what
1466         basename = act.af.spec
1467         debiancontrol = None
1468         act.binaries = []
1469
1470         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1471
1472         if act.kind == 'dsc':
1473                 dsc = act.af
1474                 dsc_file = open(dsc.read())
1475                 in_files = False
1476                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1477                 for l in dsc_file:
1478                         l = l.rstrip('\n')
1479                         if l.startswith('Files:'): in_files = True; continue
1480                         elif l.startswith('#'): pass
1481                         elif not l.startswith(' '):
1482                                 in_files = False
1483                                 if l.startswith('Source:'):
1484                                         act.blame = 'dsc:'+l[7:].strip()
1485                                         testbed.blame(act.blame)
1486                         if not in_files: continue
1487
1488                         m = fre.match(l)
1489                         if not m: badpkg(".dsc contains unparseable line"
1490                                         " in Files: `%s'" % l)
1491                         leaf = m.groups(0)[0]
1492                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1493                                         sibling=True)
1494                         subfile.read(True)
1495                 dsc.read(True)
1496
1497         if act.kind == 'ubtree':
1498                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1499                         act.af, 'debian/control')
1500                 dsc = TemporaryFile(what+'-fakedsc')
1501                 dsc_w = open(dsc.write(), 'w')
1502                 for l in open(debiancontrol.read()):
1503                         l = l.rstrip('\n')
1504                         if not len(l): break
1505                         print >>dsc_w, l
1506                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1507                 dsc_w.close()
1508
1509         if act.kind == 'dsc':
1510                 testbed.prepare2([])
1511                 script = binaries.apt_pkg_gdebi_script('', [[
1512                                 'from GDebi.DebPackage import DebPackage',
1513                                 'd = DebPackage(cache)',
1514                                 'res = d.satisfyDependsStr("dpkg-dev")',
1515                         ]])
1516                 cmdl = ['python','-c',script]
1517                 whatp = what+'-dpkgsource'
1518                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1519                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1520
1521         work = TemporaryDir(what+'-build')
1522         act.work = work
1523
1524         tmpdir = work.write(True)+'/tmpdir'
1525         tmpdir_script = [
1526                         'TMPDIR="$1"',
1527                         'rm -rf -- "$TMPDIR"',
1528                         'export TMPDIR',
1529                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1530                 ]
1531
1532         if act.kind == 'ubtree':
1533                 spec = '%s/real-tree' % work.write(True)
1534                 create_command = '''
1535                         rm -rf "$spec"
1536                         mkdir "$spec"
1537                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1538                         '''
1539                 initcwd = act.af.read(True)
1540
1541         if act.kind == 'dsc':
1542                 spec = dsc.read(True)
1543                 create_command = '''
1544                         dpkg-source -x $spec
1545                         '''
1546                 initcwd = work.write(True)
1547
1548         script = [
1549                 'spec="$2"',
1550                 'origpwd=`pwd`',
1551                 'cd '+work.write(True)
1552         ]
1553
1554         if opts.user:
1555                 script += ([ 'chown '+opts.user+' .' ] +
1556                         tmpdir_script +
1557                         [ 'spec="$spec" origpwd="$origpwd" '
1558                                 +opts.user_wrap(create_command) ])
1559         else:
1560                 script += (tmpdir_script +
1561                         [ create_command ])
1562
1563         script += [
1564                         'cd */.',
1565                         'pwd >&3',
1566                         'set +e; test -f debian/tests/control; echo $? >&3'
1567                 ]
1568         (result_pwd, control_test_rc) = source_rules_command(
1569                         act,script,what,'extract',work,
1570                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1571
1572         filter = act.ah['dsc_filter']
1573
1574         if control_test_rc == '1': act.missing_tests_control = True
1575
1576         # For optional builds:
1577         #
1578         # We might need to build the package because:
1579         #   - we want its binaries (filter isn't _ and at least one of the
1580         #       deb_... isn't ignore)
1581         #   - the test control file says so
1582         #       (assuming we have any tests)
1583
1584         class NeedBuildException: pass
1585         def build_needed(m):
1586                 debug_b('build needed for %s' % m)
1587                 raise NeedBuildException()
1588
1589         try:
1590                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1591                                       act.ah['deb_fortests'] != 'ignore'):
1592                         build_needed('binaries')
1593
1594                 result_pwd_af = InputDir(what+'-treeforcontrol',
1595                         result_pwd, True)
1596                 stanzas = read_control(act, result_pwd_af, control_override)
1597                 for stanza in stanzas:
1598                         for t in stanza[' tests']:
1599                                 if 'no-build-needed' not in t.feature_names:
1600                                         build_needed('test %s' % t.tname)
1601                                 for d in t.depends:
1602                                         if '@' in d:
1603                                                 build_needed('test %s '
1604                                                  'dependency %s' % (t.tname,d))
1605
1606                 debug_b('build not needed')
1607                 built = False
1608
1609         except NeedBuildException:
1610
1611                 if act.kind != 'dsc':
1612                         testbed.prepare2([])
1613
1614                 script = binaries.apt_pkg_gdebi_script(
1615                         dsc.read(True), [[
1616                         'from GDebi.DscSrcPackage import DscSrcPackage',
1617                         'd = DscSrcPackage(cache, arg)',
1618                         'res = d.checkDeb()',
1619                          ],[
1620                         'from GDebi.DebPackage import DebPackage',
1621                         'd = DebPackage(cache)',
1622                         'res = d.satisfyDependsStr("'+
1623                                         ','.join(build_essential)+
1624                                 '")',
1625                         ]])
1626
1627                 cmdl = ['python','-c',script]
1628                 whatp = what+'-builddeps'
1629                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1630                 if rc: badpkg('build-depends install failed,'
1631                               ' exit code %d' % rc)
1632
1633                 script = tmpdir_script + [
1634                         'cd "$2"',
1635                         'dpkg-checkbuilddeps',
1636                         opts.user_wrap('debian/rules build'),
1637                         ]
1638                 source_rules_command(act,script,what,'build',work,
1639                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1640
1641                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1642                         badpkg("results dir `%s' is not in expected parent"
1643                                " dir `%s'" % (result_pwd, work.read(True)))
1644
1645                 built = True
1646
1647         act.tests_tree = InputDir(what+'-tests-tree',
1648                                 work.read(True)+os.path.basename(result_pwd),
1649                                 True)
1650         if act.ah['dsc_tests']:
1651                 testbed.register_ephemeral(act.work)
1652                 testbed.register_ephemeral(act.tests_tree)
1653
1654         if not built:
1655                 act.blamed = []
1656                 return
1657
1658         act.blamed = copy.copy(testbed.blamed)
1659
1660         debug_b('filter=%s' % filter)
1661         if filter != '_':
1662                 script = tmpdir_script + [
1663                         'cd '+work.write(True)+'/*/.',
1664                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1665                         'cd ..',
1666                         'echo *.deb >&3',
1667                         ]
1668                 result_debs = source_rules_command(act,script,what,
1669                                 'binary',work,work.write(True),
1670                                 results_lines=1, xargs=['x',tmpdir])
1671                 if result_debs == '*': debs = []
1672                 else: debs = result_debs.split(' ')
1673                 debug_b('debs='+`debs`)
1674                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1675                 for deb in debs:
1676                         m = re.match(deb)
1677                         if not m: badpkg("badly-named binary `%s'" % deb)
1678                         pkg = m.groups()[0]
1679                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1680                         for pat in filter.split(','):
1681                                 debug_b('  pat=%s' % pat)
1682                                 if not fnmatch.fnmatchcase(pkg,pat):
1683                                         debug_b('   no match')
1684                                         continue
1685                                 deb_what = pkg+'_'+what+'.deb'
1686                                 bin = RelativeInputFile(deb_what,work,deb,True)
1687                                 debug_b('  deb_what=%s, bin=%s' %
1688                                         (deb_what, str(bin)))
1689                                 binaries.register(act,pkg,bin,
1690                                         'forbuilds',testbed.blamed)
1691                                 act.binaries.append((pkg,bin))
1692                                 break
1693                 debug_b('all done.')
1694
1695 #---------- main processing loop and main program
1696
1697 def process_actions():
1698         global binaries
1699
1700         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1701         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1702         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1703
1704         debug_a1('starting')
1705         testbed.open()
1706         binaries = Binaries()
1707
1708         for act in opts.actions:
1709                 if act.af is not None and not act.af.spec_tbp:
1710                         testbed.register_ephemeral(act.af)
1711
1712         binaries.reset()
1713         control_override = None
1714
1715         debug_a1('builds ...')
1716         for act in opts.actions:
1717                 debug_a2('%s %s' %
1718                         (act.kind, act.what))
1719
1720                 if act.kind == 'control':
1721                         control_override = act.af
1722                 if act.kind == 'deb':
1723                         testbed.blame('arg:'+act.af.spec)
1724                         determine_package(act)
1725                         testbed.blame('deb:'+act.pkg)
1726                         binaries.register(act,act.pkg,act.af,
1727                                 'forbuilds',testbed.blamed)
1728                 if act.kind == 'dsc' or act.kind == 'ubtree':
1729                         build_source(act, control_override)
1730                 if act.kind == 'tree':
1731                         act.binaries = []
1732                 if act.kind.endswith('tree') or act.kind == 'dsc':
1733                         control_override = None
1734                 if act.kind == 'instantiate':
1735                         pass
1736
1737         debug_a1('builds done.')
1738
1739         binaries.reset()
1740         control_override = None
1741
1742         debug_a1('tests ...')
1743         for act in opts.actions:
1744                 debug_a2('test %s %s' % (act.kind, act.what))
1745
1746                 testbed.needs_reset()
1747                 if act.kind == 'control':
1748                         control_override = act.af
1749                 if act.kind == 'deb':
1750                         binaries.register(act,act.pkg,act.af,'fortests',
1751                                 ['deb:'+act.pkg])
1752                 if act.kind == 'dsc' or act.kind == 'ubtree':
1753                         for (pkg,bin) in act.binaries:
1754                                 binaries.register(act,pkg,bin,'fortests',
1755                                         act.blamed)
1756                 if act.kind == 'dsc':
1757                         if act.ah['dsc_tests']:
1758                                 debug_a3('read control ...')
1759                                 stanzas = read_control(act, act.tests_tree,
1760                                                 control_override)
1761                                 testbed.blamed += act.blamed
1762                                 debug_a3('run_tests ...')
1763                                 run_tests(stanzas, act.tests_tree)
1764                         control_override = None
1765                 if act.kind == 'tree' or act.kind == 'ubtree':
1766                         testbed.blame('arg:'+act.af.spec)
1767                         stanzas = read_control(act, act.af, control_override)
1768                         debug_a3('run_tests ...')
1769                         run_tests(stanzas, act.af)
1770                         control_override = None
1771                 if act.kind == 'instantiate':
1772                         testbed.prepare([])
1773         debug_a1('tests done.')
1774
1775 def main():
1776         global testbed
1777         global tmpdir
1778         try:
1779                 parse_args()
1780         except SystemExit, se:
1781                 os._exit(20)
1782         try:
1783                 setup_trace()
1784                 testbed = Testbed()
1785                 testbed.start()
1786                 finalise_options()
1787                 process_actions()
1788         except:
1789                 ec = print_exception(sys.exc_info(), '')
1790                 cleanup()
1791                 os._exit(ec)
1792         cleanup()
1793         os._exit(errorcode)
1794
1795 main()