runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':10, 'install':900, 'test':600, 'build':3000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = sys.stderr
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         ep._sp.stdin.close()
 128         rc = ep._sp.wait()
 129         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 130         ep._sp = None
 131
 132 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 133         if dbg is not None:
 134                 if isinstance(dbg,tuple): (what,script) = dbg
 135                 else: (what,script) = (dbg,None)
 136                 debug_subprocess(what, cmdl, script=script)
 137         ep = Errplumb(critical)
 138         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 139         output = running.communicate()[0]
 140         rc = running.wait()
 141         ep.wait()
 142         return (rc, output)
 143
 144 def psummary(m):
 145         if summary_stream is not None: print >>summary_stream, m
 146
 147 def preport(m):
 148         print m
 149         sys.stdout.flush()
 150         if trace_stream is not None: print >>trace_stream, m
 151         psummary(m)
 152
 153 def report(tname, result):
 154         preport('%-20s %s' % (tname, result))
 155
 156 #---------- errors we define
 157
 158 class Quit:
 159         def __init__(q,ec,m): q.ec = ec; q.m = m
 160
 161 def bomb(m):
 162         raise Quit(20, "unexpected error: %s" % m)
 163
 164 def badpkg(m):
 165         preport('blame: ' + ' '.join(testbed.blamed))
 166         preport('badpkg: ' + m)
 167         raise Quit(12, "erroneous package: %s" % m)
 168
 169 class Unsupported:
 170  def __init__(u, lno, m):
 171         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 172         else: u.m = m
 173  def report(u, tname):
 174         global errorcode
 175         errorcode != 2
 176         report(tname, 'SKIP %s' % u.m)
 177
 178 #---------- convenience function
 179
 180 def mkdir_okexist(pathname, mode=02755):
 181         try:
 182                 os.mkdir(pathname, mode)
 183         except OSError, oe:
 184                 if oe.errno != errno.EEXIST: raise
 185
 186 def rmtree(what, pathname):
 187         debug('/ %s rmtree %s' % (what, pathname), 2)
 188         shutil.rmtree(pathname)
 189
 190 def debug_subprocess(what, cmdl=None, script=None):
 191         o = '$ '+what+':'
 192         if cmdl is not None:
 193                 ol = []
 194                 for x in cmdl:
 195                         if x is script: x = '<SCRIPT>'
 196                         ol.append(x.    replace('\\','\\\\').
 197                                         replace(' ','\\ ')      )
 198                 o += ' '+ ' '.join(ol)
 199         debug(o)
 200         if script is not None and opts.debuglevel >= 1:
 201                 o = ''
 202                 for l in script.rstrip('\n').split('\n'):
 203                         o += '$     '+l+'\n'
 204                 debug(o,1)
 205
 206 def flatten(l):
 207         return reduce((lambda a,b: a + b), l, [])
 208
 209 #---------- fancy automatic file-copying class
 210
 211 class AutoFile:
 212         # p.what
 213         # p.path[tb]    None or path    not None => path known
 214         # p.file[tb]    None or path    not None => file exists
 215         # p.spec        string or None
 216         # p.spec_tbp    True or False, or not set if spec is None
 217         # p.dir         '' or '/'
 218
 219  def __init__(p, what):
 220         p.what = what
 221         p.path = [None,None]
 222         p.file = [None,None]
 223         p.spec = None
 224         p.dir = ''
 225
 226  def __repr__(p):
 227         return "<AF@%s>" % p.__str__()
 228  def __str__(p):
 229         def ptbp(tbp):
 230                 if p.path[tbp] is None: return '-'+p.dir
 231                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 232                 else: return p.path[tbp]+p.dir+'!'
 233         out = p.what
 234         if p.spec is not None:
 235                 if not hasattr(p,'spec_tb'): out += '~'
 236                 elif p.spec_tbp: out += '#'
 237                 else: out += '='
 238                 out += p.spec
 239         out += ':'
 240         out += ptbp(False)
 241         out += '|'
 242         out += ptbp(True)
 243         return out
 244
 245  def _wrong(p, how):
 246         xtra = ''
 247         if p.spec is not None:
 248                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 249         raise ("internal error: %s (%s)" % (how, str(p)))
 250
 251  def _ensure_path(p, tbp):
 252         if p.path[tbp] is None:
 253                 if '/' in p.what:
 254                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 255                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 256                 if not tbp:
 257                         p.path[tbp] = tmpdir+'/'+p.what
 258                 else:
 259                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 260
 261  def write(p, tbp=False):
 262         p._debug('write %s...' % 'HT'[tbp])
 263         p._ensure_path(tbp)
 264
 265         if p.dir and not p.file[tbp]:
 266                 if not tbp:
 267                         p._debug('mkdir H')
 268                         mkdir_okexist(p.path[tbp])
 269                 else:
 270                         cmdl = ['sh','-ec',
 271                                 'test -d "$1" || mkdir -p "$1"',
 272                                 'x', p.path[tbp]]
 273                         tf_what = urllib.quote(p.what).replace('/','%2F')
 274                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 275                         if rc: bomb('failed to create directory %s' %
 276                                 p.path[tbp])
 277
 278         p.file[tbp] = p.path[tbp]
 279         return p.path[tbp]
 280
 281  def read(p, tbp=False):
 282         p._debug('read %s...' % 'HT'[tbp])
 283         p._ensure_path(tbp)
 284
 285         if p.file[tbp] is None:
 286                 if p.file[not tbp] is None:
 287                         p._wrong("requesting read but nonexistent")
 288                 cud = ['copyup','copydown'][tbp]
 289                 src = p.file[not tbp] + p.dir
 290                 dst = p.path[tbp] + p.dir
 291                 testbed.command(cud, (src, dst))
 292                 p.file[tbp] = p.path[tbp]
 293
 294         return p.file[tbp] + p.dir
 295
 296  def invalidate(p, tbp=False):
 297         p.file[tbp] = None
 298         p._debug('invalidated %s' % 'HT'[tbp])
 299
 300  def _debug(p, m):
 301         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 302
 303  def _constructed(p):
 304         p._debug('constructed: '+str(p))
 305         p._check()
 306
 307  def _check(p):
 308         for tbp in [False,True]:
 309          for pf in [p.path, p.file]:
 310                 if pf[tbp] is None: continue
 311                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 312                 if p.dir and pf[tbp].endswith('/'):
 313                         pf[tbp] = pf[tbp].rstrip('/')
 314                         if not pf[tbp]: pf[tbp] = '/'
 315                 if not p.dir and pf[tbp].endswith('/'):
 316                         bomb("directory `%s' specified for "
 317                              "non-directory %s" % (pf[tbp], p.what))
 318
 319  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 320         AutoFile.__init__(p,what)
 321         sh_on = ''; sh_sibl = ''
 322         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 323         if sibling: sh_sibl=' (sibling)'
 324         parent._debug('using as base: %s: %s%s%s' %
 325                         (str(parent), leaf, sh_on, sh_sibl))
 326         if not sibling and not parent.dir:
 327                 parent._wrong('asked for non-sibling relative path of non-dir')
 328         if sibling: trim = os.path.dirname
 329         else: trim = lambda x: x
 330         for tbp in [False,True]:
 331                 if parent.path[tbp] is None: continue
 332                 trimmed = trim(parent.path[tbp])
 333                 if trimmed: trimmed += '/'
 334                 p.path[tbp] = trimmed + leaf
 335                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 336                         p.file[tbp] = p.path[tbp]
 337
 338 class InputFile(AutoFile):
 339  def _init(p, what, spec, spec_tbp=False):
 340         AutoFile.__init__(p, what)
 341         p.spec = spec
 342         p.spec_tbp = spec_tbp
 343         p.path[spec_tbp] = spec
 344         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 345  def __init__(p, what, spec, spec_tbp=False):
 346         p._init(what,spec,spec_tbp)
 347         p._constructed()
 348
 349 class InputDir(InputFile):
 350  def __init__(p, what, spec, spec_tbp=False):
 351         InputFile._init(p,what,spec,spec_tbp)
 352         p.dir = '/'
 353         p._constructed()
 354
 355 class OutputFile(AutoFile):
 356  def _init(p, what, spec, spec_tbp=False):
 357         AutoFile.__init__(p, what)
 358         p.spec = spec
 359         p.spec_tbp = spec_tbp
 360         p.path[spec_tbp] = spec
 361  def __init__(p, what, spec, spec_tbp=False):
 362         p._init(what,spec,spec_tbp)
 363         p._constructed()
 364
 365 class OutputDir(OutputFile):
 366  def __init__(p, what, spec, spec_tbp=False):
 367         OutputFile._init(p,what,spec,spec_tbp)
 368         p.dir = '/'
 369         p._constructed()
 370
 371 class RelativeInputFile(AutoFile):
 372  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 373         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 374         p._constructed()
 375
 376 class RelativeOutputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, sibling=False):
 378         p._relative_init(what, parent, leaf, None, False, sibling)
 379         p._constructed()
 380
 381 class TemporaryFile(AutoFile):
 382  def __init__(p, what):
 383         AutoFile.__init__(p, what)
 384         p._constructed()
 385
 386 class TemporaryDir(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p.dir = '/'
 390         p._constructed()
 391
 392 #---------- parsing and representation of the arguments
 393
 394 class Action:
 395  def __init__(a, kind, af, arghandling, what):
 396         # extra attributes get added during processing
 397         a.kind = kind
 398         a.af = af
 399         a.ah = arghandling
 400         a.what = what
 401         a.missing_tests_control = False
 402  def __repr__(a):
 403         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 404
 405 def parse_args():
 406         global opts
 407         global n_non_actions # argh, stupid python scoping rules
 408         usage = "%prog <options> --- <virt-server>..."
 409         parser = OptionParser(usage=usage)
 410         pa = parser.add_option
 411         pe = parser.add_option
 412
 413         arghandling = {
 414                 'dsc_tests': True,
 415                 'dsc_filter': '*',
 416                 'deb_forbuilds': 'auto',
 417                 'deb_fortests': 'auto',
 418                 'tb': False,
 419                 'override_control': None
 420         }
 421         initial_arghandling = arghandling.copy()
 422         n_non_actions = 0
 423
 424         #----------
 425         # actions (ie, test sets to run, sources to build, binaries to use):
 426
 427         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 428                 global n_non_actions
 429                 parser.largs.append((value,kindpath))
 430                 n_non_actions += not(is_act)
 431
 432         def pa_action(long, metavar, kindpath, help, is_act=True):
 433                 pa('','--'+long, action='callback', callback=cb_action,
 434                         nargs=1, type='string',
 435                         callback_args=(long,kindpath,is_act), help=help)
 436
 437         pa_action('built-tree',         'TREE', '@/',
 438                 help='run tests from build tree TREE')
 439
 440         pa_action('unbuilt-tree',       'TREE', '@//',
 441                 help='run tests from build tree TREE')
 442
 443         pa_action('source',             'DSC', '@.dsc',
 444                 help='build DSC and use its tests and/or'
 445                     ' generated binary packages')
 446
 447         pa_action('binary',             'DEB', '@.deb',
 448                help='use binary package DEB according'
 449                     ' to most recent --binaries-* settings')
 450
 451         def cb_actnoarg(op,optstr,value,parser, largsentry):
 452                 parser.largs.append(largsentry)
 453         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 454                 callback_args=((None, ('instantiate',)),),
 455                 help='instantiate testbed now (during testing phase)'
 456                      ' and install packages'
 457                      ' selected for automatic installation, even'
 458                      ' if this might apparently not be required otherwise')
 459
 460         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 461                help='run tests from control file CONTROL instead,'
 462                     ' (applies to next test suite only)')
 463
 464         #----------
 465         # argument handling settings (what ways to use action
 466         #  arguments, and pathname processing):
 467
 468         def cb_setah(option, opt_str, value, parser, toset,setval):
 469                 if type(setval) == list:
 470                         if not value in setval:
 471                                 parser.error('value for %s option (%s) is not '
 472                                  'one of the permitted values (%s)' %
 473                                  (value, opt_str, setval.join(' ')))
 474                 elif setval is not None:
 475                         value = setval
 476                 for v in toset:
 477                         arghandling[v] = value
 478                 parser.largs.append(arghandling.copy())
 479
 480         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 481                 type = metavar
 482                 if type is not None: type = 'string'
 483                 pa('',long, action='callback', callback=cb_setah,
 484                    callback_args=(affected,effect), **kwargs)
 485
 486         #---- paths: host or testbed:
 487         #
 488         pa_setah('--paths-testbed', ['tb'],True,
 489                 help='subsequent path specifications refer to the testbed')
 490         pa_setah('--paths-host', ['tb'],False,
 491                 help='subsequent path specifications refer to the host')
 492
 493         #---- source processing settings:
 494
 495         pa_setah('--sources-tests', ['dsc_tests'],True,
 496                 help='run tests from builds of subsequent sources')
 497         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 498                 help='do not run tests from builds of subsequent sources')
 499
 500         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 501                 type='string', metavar='PATTERN-LIST',
 502                 help='from subsequent sources, use binaries matching'
 503                      ' PATTERN-LIST (comma-separated glob patterns)'
 504                      ' according to most recent --binaries-* settings')
 505         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 506                 help='from subsequent sources, do not use any binaries')
 507
 508         #---- binary package processing settings:
 509
 510         def pa_setahbins(long,toset,how):
 511          pa_setah(long, toset,['ignore','auto','install'],
 512                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 513                 help=how+' ignore binaries, install them as needed'
 514                         ' for dependencies, or unconditionally install'
 515                         ' them, respectively')
 516         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 517         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 518         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 519
 520         #----------
 521         # general options:
 522
 523         def cb_vserv(op,optstr,value,parser):
 524                 parser.values.vserver = list(parser.rargs)
 525                 del parser.rargs[:]
 526
 527         def cb_path(op,optstr,value,parser, constructor,long,dir):
 528                 name = long.replace('-','_')
 529                 af = constructor(long, value,arghandling['tb'])
 530                 setattr(parser.values, name, af)
 531
 532         def pa_path(long, constructor, help, dir=False):
 533                 pa('','--'+long, action='callback', callback=cb_path,
 534                         callback_args=(constructor,long,dir),
 535                         nargs=1, type='string',
 536                         help=help, metavar='PATH')
 537
 538         pa_path('output-dir', OutputDir, dir=True,
 539                 help='write stderr/out files in PATH')
 540
 541         pa('','--tmp-dir',              type='string', dest='tmpdir',
 542                 help='write temporary files to TMPDIR, emptying it'
 543                      ' beforehand and leaving it behind at the end')
 544         pa('','--log-file',             type='string', dest='logfile',
 545                 help='write the log LOGFILE, emptying it beforehand,'
 546                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 547         pa('','--summary-file',         type='string', dest='summary',
 548                 help='write a summary report to SUMMARY,'
 549                      ' emptying it beforehand')
 550
 551         pa('','--user',                 type='string', dest='user',
 552                 help='run tests as USER (needs root on testbed)')
 553         pa('','--gain-root',            type='string', dest='gainroot',
 554                 help='prefix debian/rules binary with GAINROOT')
 555         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 556         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 557         pa('','--gnupg-home',           type='string', dest='gnupghome',
 558                 default='~/.autopkgtest/gpg',
 559                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 560                         " signing private apt archive);"
 561                         " `fresh' means generate new key each time.")
 562
 563         #----------
 564         # actual meat:
 565
 566         class SpecialOption(optparse.Option): pass
 567         vs_op = SpecialOption('','--VSERVER-DUMMY')
 568         vs_op.action = 'callback'
 569         vs_op.type = None
 570         vs_op.default = None
 571         vs_op.nargs = 0
 572         vs_op.callback = cb_vserv
 573         vs_op.callback_args = ( )
 574         vs_op.callback_kwargs = { }
 575         vs_op.help = 'introduces virtualisation server and args'
 576         vs_op._short_opts = []
 577         vs_op._long_opts = ['---']
 578
 579         pa(vs_op)
 580
 581         (opts,args) = parser.parse_args()
 582         if not hasattr(opts,'vserver'):
 583                 parser.error('you must specifiy --- <virt-server>...')
 584         if n_non_actions >= len(parser.largs):
 585                 parser.error('nothing to do specified')
 586
 587         arghandling = initial_arghandling
 588         opts.actions = []
 589         ix = 0
 590         for act in args:
 591                 if type(act) == dict:
 592                         arghandling = act
 593                         continue
 594                 elif type(act) == tuple:
 595                         pass
 596                 elif type(act) == str:
 597                         act = (act,act)
 598                 else:
 599                         raise ("unknown action in list `%s' having"
 600                               " type `%s'" % (act, type(act)))
 601                 (pathstr, kindpath) = act
 602
 603                 constructor = InputFile
 604                 if type(kindpath) is tuple:             kind = kindpath[0]
 605                 elif kindpath.endswith('.deb'):         kind = 'deb'
 606                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 607                 elif kindpath.endswith('//'):
 608                         kind = 'ubtree'
 609                         constructor = InputDir
 610                 elif kindpath.endswith('/'):
 611                         kind = 'tree'
 612                         constructor = InputDir
 613                 else: parser.error("do not know how to handle filename \`%s';"
 614                         " specify --source --binary or --build-tree")
 615
 616                 what = '%s%s' % (kind,ix); ix += 1
 617
 618                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 619                 else: fwhatx = '-'+kind
 620
 621                 if pathstr is None: af = None
 622                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 623                 opts.actions.append(Action(kind, af, arghandling, what))
 624
 625 def setup_trace():
 626         global trace_stream, tmpdir, summary_stream
 627
 628         if opts.tmpdir is not None:
 629                 rmtree('tmpdir(specified)',opts.tmpdir)
 630                 mkdir_okexist(opts.tmpdir, 0700)
 631                 tmpdir = opts.tmpdir
 632         else:
 633                 assert(tmpdir is None)
 634                 tmpdir = tempfile.mkdtemp()
 635
 636         if opts.logfile is None:
 637                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 638                         opts.logfile = opts.output_dir.spec + '/log'
 639                 elif opts.tmpdir is not None:
 640                         opts.logfile = opts.tmpdir + '/log'
 641         if opts.logfile is not None:
 642                 trace_stream = open(opts.logfile, 'w', 0)
 643         if opts.summary is not None:
 644                 summary_stream = open(opts.summary, 'w', 0)
 645
 646         debug('options: '+`opts`, 1)
 647
 648 def finalise_options():
 649         global opts, tb, build_essential
 650
 651         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 652                 opts.user = ''
 653
 654         if opts.user is None:
 655                 su = 'suggested-normal-user='
 656                 ul = [
 657                         e[len(su):]
 658                         for e in testbed.caps
 659                         if e.startswith(su)
 660                         ]
 661                 if ul:
 662                         opts.user = ul[0]
 663                 else:
 664                         opts.user = ''
 665
 666         if opts.user:
 667                 if 'root-on-testbed' not in testbed.caps:
 668                         pstderr("warning: virtualisation"
 669                                 " system does not offer root on testbed,"
 670                                 " but --user option specified: failure likely")
 671                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 672         else:
 673                 opts.user_wrap = lambda x: x
 674
 675         if opts.gainroot is None:
 676                 opts.gainroot = ''
 677                 if (opts.user or
 678                     'root-on-testbed' not in testbed.caps):
 679                         opts.gainroot = 'fakeroot'
 680                         build_essential += ['fakeroot']
 681
 682         if opts.gnupghome.startswith('~/'):
 683                 try: home = os.environ['HOME']
 684                 except KeyError:
 685                         parser.error("HOME environment variable"
 686                                 " not set, needed for --gnupghome=`%s"
 687                                 % opts.gnupghome)
 688                 opts.gnupghome = home + opts.gnupghome[1:]
 689         elif opts.gnupghome == 'fresh':
 690                 opts.gnupghome = None
 691
 692 #---------- testbed management - the Testbed class
 693
 694 class Testbed:
 695  def __init__(tb):
 696         tb.sp = None
 697         tb.lastsend = None
 698         tb.scratch = None
 699         tb.modified = False
 700         tb.blamed = []
 701         tb._ephemeral = []
 702         tb._debug('init')
 703         tb._need_reset_apt = False
 704  def _debug(tb, m, minlevel=0):
 705         debug('** '+m, minlevel)
 706  def start(tb):
 707         tb._debug('start')
 708         p = subprocess.PIPE
 709         debug_subprocess('vserver', opts.vserver)
 710         tb._errplumb = Errplumb(True)
 711         tb.sp = subprocess.Popen(opts.vserver,
 712                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 713         tb.expect('ok')
 714         tb.caps = tb.commandr('capabilities')
 715  def stop(tb):
 716         tb._debug('stop')
 717         tb.close()
 718         if tb.sp is None: return
 719         ec = tb.sp.returncode
 720         if ec is None:
 721                 tb.sp.stdout.close()
 722                 tb.send('quit')
 723                 tb.sp.stdin.close()
 724                 ec = tb.sp.wait()
 725         if ec:
 726                 tb.bomb('testbed gave exit status %d after quit' % ec)
 727         tb._errplumb.wait()
 728  def open(tb):
 729         tb._debug('open, scratch=%s' % tb.scratch)
 730         if tb.scratch is not None: return
 731         pl = tb.commandr('open')
 732         tb.scratch = InputDir('tb-scratch', pl[0], True)
 733         tb.deps_processed = []
 734  def mungeing_apt(tb):
 735         if not 'revert' in tb.caps:
 736                 tb._need_reset_apt = True
 737  def reset_apt(tb):
 738         if not tb._need_reset_apt: return
 739         what = 'aptget-update-reset'
 740         cmdl = ['apt-get','-qy','update']
 741         rc = tb.execute(what, cmdl, kind='install')
 742         if rc:
 743                 pstderr("\n" "warning: failed to restore"
 744                         " testbed apt cache, exit code %d" % rc)
 745         tb._need_reset_apt = False
 746  def close(tb):
 747         tb._debug('close, scratch=%s' % tb.scratch)
 748         if tb.scratch is None: return
 749         tb.scratch = None
 750         if tb.sp is None: return
 751         tb.command('close')
 752  def prepare1(tb, deps_new):
 753         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 754                 (tb.modified, tb.deps_processed, deps_new), 1)
 755         if 'revert' in tb.caps and (tb.modified or
 756             [d for d in tb.deps_processed if d not in deps_new]):
 757                 for af in tb._ephemeral: af.read(False)
 758                 tb._debug('reset **')
 759                 tb.command('revert')
 760                 tb.blamed = []
 761                 for af in tb._ephemeral: af.invalidate(True)
 762         tb.modified = False
 763  def prepare2(tb, deps_new):
 764         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 765         binaries.publish()
 766         tb._install_deps(deps_new)
 767  def prepare(tb, deps_new):
 768         tb.prepare1(deps_new)
 769         tb.prepare2(deps_new)
 770  def register_ephemeral(tb, af):
 771         tb._ephemeral.append(af)
 772  def _install_deps(tb, deps_new):
 773         tb._debug(' installing dependencies '+`deps_new`, 1)
 774         tb.deps_processed = deps_new
 775         if not deps_new: return
 776         dstr = ', '.join(deps_new)
 777         script = binaries.apt_pkg_gdebi_script(
 778                 dstr, [[
 779                 'from GDebi.DebPackage import DebPackage',
 780                 'd = DebPackage(cache)',
 781                 'res = d.satisfyDependsStr(arg)',
 782                 ]])
 783         cmdl = ['python','-c',script]
 784         what = 'install-deps'
 785         rc = testbed.execute(what+'-debinstall', cmdl, script=script,
 786                                 kind='install')
 787         if rc: badpkg('dependency install failed, exit code %d' % rc)
 788  def needs_reset(tb):
 789         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 790         tb.modified = True
 791  def blame(tb, m):
 792         tb._debug('blame += %s' % m, 1)
 793         tb.blamed.append(m)
 794  def bomb(tb, m):
 795         tb._debug('bomb %s' % m)
 796         if tb.sp is not None:
 797                 tb.sp.stdout.close()
 798                 tb.sp.stdin.close()
 799                 ec = tb.sp.wait()
 800                 if ec: pstderr('adt-run: testbed failing,'
 801                                 ' exit status %d' % ec)
 802         tb.sp = None
 803         raise Quit(16, 'testbed failed: %s' % m)
 804  def send(tb, string):
 805         tb.sp.stdin
 806         try:
 807                 debug('>> '+string, 2)
 808                 print >>tb.sp.stdin, string
 809                 tb.sp.stdin.flush()
 810                 tb.lastsend = string
 811         except:
 812                 (type, value, dummy) = sys.exc_info()
 813                 tb.bomb('cannot send to testbed: %s' % traceback.
 814                         format_exception_only(type, value))
 815  def expect(tb, keyword, nresults=None):
 816         l = tb.sp.stdout.readline()
 817         if not l: tb.bomb('unexpected eof from the testbed')
 818         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 819         l = l.rstrip('\n')
 820         debug('<< '+l, 2)
 821         ll = l.split()
 822         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 823         if ll[0] != keyword:
 824                 if tb.lastsend is None:
 825                         tb.bomb("got banner `%s', expected `%s...'" %
 826                                 (l, keyword))
 827                 else:
 828                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 829                                 (tb.lastsend, l, keyword))
 830         ll = ll[1:]
 831         if nresults is not None and len(ll) != nresults:
 832                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 833                         " expected %d result parameters" %
 834                         (string, l, len(ll), nresults))
 835         return ll
 836  def commandr(tb, cmd, args=(), nresults=None):
 837         # pass args=[None,...] or =(None,...) to avoid more url quoting
 838         if type(cmd) is str: cmd = [cmd]
 839         if len(args) and args[0] is None: args = args[1:]
 840         else: args = map(urllib.quote, args)
 841         al = cmd + args
 842         tb.send(string.join(al))
 843         ll = tb.expect('ok', nresults)
 844         rl = map(urllib.unquote, ll)
 845         return rl
 846  def command(tb, cmd, args=()):
 847         tb.commandr(cmd, args, 0)
 848  def commandr1(tb, cmd, args=()):
 849         rl = tb.commandr(cmd, args, 1)
 850         return rl[0]
 851  def execute(tb, what, cmdl,
 852                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 853                 script=False, tmpdir=None, kind='short'):
 854         # Options for script:
 855         #   False - do not call debug_subprocess, no synch. reporting required
 856         #   None or string - call debug_subprocess with that value,
 857         #                       plumb stderr through synchronously if possible
 858         # Options for se:
 859         #   None - usual Errplumb (output is of kind 2c)
 860         #   string - path on testbed (output is of kind 2d)
 861
 862         timeout = timeouts[kind]
 863
 864         if script is not False: debug_subprocess(what, cmdl, script=script)
 865         if cwd is None: cwd = tb.scratch.write(True)
 866
 867         xdump = None
 868         if se is None:
 869                 ep = Errplumb()
 870                 se_catch = TemporaryFile(what+'-xerr')
 871                 se_use = se_catch.write(True)
 872                 if not opts.quiet: xdump = 'debug=2-2'
 873                 elif trace_stream is not None:
 874                         xdump = 'debug=2-%d' % trace_stream.fileno()
 875         else:
 876                 ep = None
 877                 se_catch = None
 878                 se_use = se
 879
 880         cmdl = [None,
 881                 ','.join(map(urllib.quote, cmdl)),
 882                 si, so, se_use, cwd]
 883
 884         if timeout is not None and timeout > 0:
 885                 cmdl.append('timeout=%d' % timeout)
 886
 887         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 888         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 889         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 890
 891         rc = tb.commandr1('execute', cmdl)
 892         try: rc = int(rc)
 893         except ValueError: bomb("execute for %s gave invalid response `%s'"
 894                                         % (what,rc))
 895
 896         if se_catch is not None:
 897                 debug_file(se_catch.read())
 898         if ep is not None:
 899                 ep.wait()
 900
 901         return rc
 902
 903 #---------- representation of test control files: Field*, Test, etc.
 904
 905 class FieldBase:
 906  def __init__(f, fname, stz, base, tnames, vl):
 907         assert(vl)
 908         f.stz = stz
 909         f.base = base
 910         f.tnames = tnames
 911         f.vl = vl
 912  def words(f):
 913         def distribute(vle):
 914                 (lno, v) = vle
 915                 r = v.split()
 916                 r = map((lambda w: (lno, w)), r)
 917                 return r
 918         return flatten(map(distribute, f.vl))
 919  def atmostone(f):
 920         if len(vl) == 1:
 921                 (f.lno, f.v) = vl[0]
 922         else:
 923                 raise Unsupported(f.vl[1][0],
 924                         'only one %s field allowed' % fn)
 925         return f.v
 926
 927 class FieldIgnore(FieldBase):
 928  def parse(f): pass
 929
 930 class Restriction:
 931  def __init__(r,rname,base): pass
 932
 933 class Restriction_rw_build_tree(Restriction): pass
 934 class Restriction_breaks_testbed(Restriction):
 935  def __init__(r, rname, base):
 936         if 'revert' not in testbed.caps:
 937                 raise Unsupported(f.lno,
 938                         'Test breaks testbed but testbed cannot revert')
 939 class Restriction_needs_root(Restriction):
 940  def __init__(r, rname, base):
 941         if 'root-on-testbed' not in testbed.caps:
 942                 raise Unsupported(f.lno,
 943                         'Test needs root on testbed which is not available')
 944
 945 class Field_Restrictions(FieldBase):
 946  def parse(f):
 947         for wle in f.words():
 948                 (lno, rname) = wle
 949                 nrname = rname.replace('-','_')
 950                 try: rclass = globals()['Restriction_'+nrname]
 951                 except KeyError: raise Unsupported(lno,
 952                         'unknown restriction %s' % rname)
 953                 r = rclass(nrname, f.base)
 954                 f.base['restriction_names'].append(rname)
 955                 f.base['restrictions'].append(r)
 956
 957 class Field_Features(FieldIgnore):
 958  def parse(f):
 959         for wle in f.words():
 960                 (lno, fname) = wle
 961                 f.base['feature_names'].append(fname)
 962                 nfname = fname.replace('-','_')
 963                 try: fclass = globals()['Feature_'+nfname]
 964                 except KeyError: continue
 965                 ft = fclass(nfname, f.base)
 966                 f.base['features'].append(ft)
 967
 968 class Field_Tests(FieldIgnore): pass
 969
 970 class Field_Depends(FieldBase):
 971  def parse(f):
 972         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 973         dl = map(lambda x: x.strip(),
 974                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 975         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
 976         for d in dl:
 977                 if re.search(d):
 978                         badpkg("Test Depends field contains dependency"
 979                                " `%s' with invalid characters" % d)
 980         f.base['depends'] = dl
 981
 982 class Field_Tests_directory(FieldBase):
 983  def parse(f):
 984         td = atmostone(f)
 985         if td.startswith('/'): raise Unspported(f.lno,
 986                 'Tests-Directory may not be absolute')
 987         f.base['testsdir'] = td
 988
 989 def run_tests(stanzas, tree):
 990         global errorcode, testbed
 991         if stanzas == ():
 992                 report('*', 'SKIP no tests in this package')
 993                 errorcode |= 8
 994         for stanza in stanzas:
 995                 tests = stanza[' tests']
 996                 if not tests:
 997                         report('*', 'SKIP package has metadata but no tests')
 998                         errorcode |= 8
 999                 for t in tests:
1000                         t.prepare()
1001                         t.run(tree)
1002                         if 'breaks-testbed' in t.restriction_names:
1003                                 testbed.needs_reset()
1004                 testbed.needs_reset()
1005
1006 class Test:
1007  def __init__(t, tname, base, act):
1008         if '/' in tname: raise Unsupported(base[' lno'],
1009                 'test name may not contain / character')
1010         for k in base: setattr(t,k,base[k])
1011         t.tname = tname
1012         t.act = act
1013         t.what = act.what+'t-'+tname
1014         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1015         else: t.path = tname
1016         t._debug('constructed; path=%s' % t.path)
1017         t._debug(' .depends=%s' % t.depends)
1018  def _debug(t, m):
1019         debug('& %s: %s' % (t.what, m))
1020  def report(t, m):
1021         report(t.what, m)
1022  def reportfail(t, m):
1023         global errorcode
1024         errorcode |= 4
1025         report(t.what, 'FAIL ' + m)
1026  def prepare(t):
1027         t._debug('preparing')
1028         dn = []
1029         for d in t.depends:
1030                 t._debug(' processing dependency '+d)
1031                 if not '@' in d:
1032                         t._debug('  literal dependency '+d)
1033                         dn.append(d)
1034                 else:
1035                         for (pkg,bin) in t.act.binaries:
1036                                 d = d.replace('@',pkg)
1037                                 t._debug('  synthesised dependency '+d)
1038                                 dn.append(d)
1039         testbed.prepare(dn)
1040  def run(t, tree):
1041         t._debug('[----------------------------------------')
1042         def stdouterr(oe):
1043                 idstr = t.what + '-' + oe
1044                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1045                         use_dir = opts.output_dir
1046                 else:
1047                         use_dir = testbed.scratch
1048                 return RelativeOutputFile(idstr, use_dir, idstr)
1049
1050         t.act.work.write(True)
1051
1052         af = RelativeInputFile(t.what, tree, t.path)
1053         so = stdouterr('stdout')
1054         se = stdouterr('stderr')
1055
1056         tf = af.read(True)
1057         tmpdir = None
1058         tree.read(True)
1059
1060         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1061         if rc: bomb('failed to chmod +x %s' % tf)
1062
1063         if 'needs-root' not in t.restriction_names and opts.user is not None:
1064                 tfl = ['su',opts.user,'-c',tf]
1065                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1066                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1067                 if opts.user: script += '; chown %s "$1"' % opts.user
1068                 if 'rw-build-tree' in t.restriction_names:
1069                         script += '; chown -R %s "$2"' % opts.user
1070                 rc = testbed.execute('mktmpdir-'+t.what,
1071                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1072                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1073                                 % (tmpdir, rc))
1074         else:
1075                 tfl = [tf]
1076
1077         rc = testbed.execute('test-'+t.what, tfl,
1078                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1079                 tmpdir=tmpdir, kind='test')
1080
1081         so_read = so.read()
1082         se_read = se.read()
1083
1084         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1085         stab = os.stat(se_read)
1086         if stab.st_size != 0:
1087                 l = open(se_read).readline()
1088                 l = l.rstrip('\n \t\r')
1089                 if len(l) > 35: l = l[:35] + '...'
1090                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1091                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1092                 debug_file(se_read)
1093         elif rc != 0:
1094                 t.reportfail('non-zero exit status %d' % rc)
1095         else:
1096                 t.report('PASS')
1097
1098         stab = os.stat(so_read)
1099         if stab.st_size != 0:
1100                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1101                 debug_file(so_read)
1102
1103         t._debug('----------------------------------------]')
1104
1105 def read_control(act, tree, control_override):
1106         stanzas = [ ]
1107
1108         if control_override is not None:
1109                 control_af = control_override
1110                 testbed.blame('arg:'+control_override.spec)
1111         else:
1112                 if act.missing_tests_control:
1113                         return ()
1114                 control_af = RelativeInputFile(act.what+'-testcontrol',
1115                         tree, 'debian/tests/control')
1116         try:
1117                 control = open(control_af.read(), 'r')
1118         except OSError, oe:
1119                 if oe[0] != errno.ENOENT: raise
1120                 return []
1121
1122         lno = 0
1123         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1124         stz = None      # stz[field_name][index] = (lno, value)
1125                         # special field names:
1126                         # stz[' lno'] = number
1127                         # stz[' tests'] = list of Test objects
1128         def end_stanza(stz):
1129                 if stz is None: return
1130                 stz[' errs'] = 0
1131                 stanzas.append(stz)
1132                 stz = None
1133                 hcurrent = None
1134
1135         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1136         while 1:
1137                 l = control.readline()
1138                 if not l: break
1139                 lno += 1
1140                 if not l.endswith('\n'): badctrl('unterminated line')
1141                 if regexp.compile('\s*\#').match(l): continue
1142                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1143                 initmat = initre.match(l)
1144                 if initmat:
1145                         (fname, l) = initmat.groups()
1146                         fname = string.capwords(fname)
1147                         if stz is None:
1148                                 stz = { ' lno': lno, ' tests': [] }
1149                         if not stz.has_key(fname): stz[fname] = [ ]
1150                         hcurrent = stz[fname]
1151                 elif regexp.compile('\s').match(l):
1152                         if not hcurrent: badctrl('unexpected continuation')
1153                 else:
1154                         badctrl('syntax error')
1155                 hcurrent.append((lno, l))
1156         end_stanza(stz)
1157
1158         def testbadctrl(stz, lno, m):
1159                 report_badctrl(lno, m)
1160                 stz[' errs'] += 1
1161
1162         for stz in stanzas:
1163                 try:
1164                         try: tnames = stz['Tests']
1165                         except KeyError:
1166                                 tnames = ['*']
1167                                 raise Unsupported(stz[' lno'],
1168                                         'no Tests field')
1169                         tnames = map((lambda lt: lt[1]), tnames)
1170                         tnames = string.join(tnames).split()
1171                         base = {
1172                                 'restriction_names': [],
1173                                 'restrictions': [],
1174                                 'feature_names': [],
1175                                 'features': [],
1176                                 'testsdir': 'debian/tests',
1177                                 'depends' : '@'
1178                         }
1179                         for fname in stz.keys():
1180                                 if fname.startswith(' '): continue
1181                                 vl = stz[fname]
1182                                 try: fclass = globals()['Field_'+
1183                                         fname.replace('-','_')]
1184                                 except KeyError: raise Unsupported(vl[0][0],
1185                                         'unknown metadata field %s' % fname)
1186                                 f = fclass(stz, fname, base, tnames, vl)
1187                                 f.parse()
1188                         for tname in tnames:
1189                                 t = Test(tname, base, act)
1190                                 stz[' tests'].append(t)
1191                 except Unsupported, u:
1192                         for tname in tnames: u.report(tname)
1193                         continue
1194
1195         return stanzas
1196
1197 def print_exception(ei, msgprefix=''):
1198         if msgprefix: pstderr(msgprefix)
1199         (et, q, tb) = ei
1200         if et is Quit:
1201                 pstderr('adt-run: ' + q.m)
1202                 psummary('quitting: '+q.m)
1203                 return q.ec
1204         else:
1205                 pstderr("adt-run: unexpected, exceptional, error:")
1206                 psummary('quitting: unexpected error, consult transcript')
1207                 traceback.print_exc(None, sys.stderr)
1208                 if trace_stream is not None:
1209                         traceback.print_exc(None, trace_stream)
1210                 return 20
1211
1212 def cleanup():
1213         global trace_stream
1214         try:
1215                 rm_ec = 0
1216                 if opts.tmpdir is None and tmpdir is not None:
1217                         rmtree('tmpdir', tmpdir)
1218                 if testbed is not None:
1219                         testbed.reset_apt()
1220                         testbed.stop()
1221                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1222                 if trace_stream is not None:
1223                         trace_stream.close()
1224                         trace_stream = None
1225         except:
1226                 print_exception(sys.exc_info(),
1227                         '\nadt-run: error cleaning up:\n')
1228                 os._exit(20)
1229
1230 #---------- registration, installation etc. of .deb's: Binaries
1231
1232 def determine_package(act):
1233         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1234         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1235         if rc: badpkg('failed to parse binary package, code %d' % rc)
1236         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1237         act.pkg = None
1238         for l in output.split('\n'):
1239                 m = re.match(l)
1240                 if not m: continue
1241                 if act.pkg: badpkg('two Package: lines in control file')
1242                 act.pkg = m.groups()[0]
1243         if not act.pkg: badpkg('no good Package: line in control file')
1244
1245 class Binaries:
1246  def __init__(b):
1247         b.dir = TemporaryDir('binaries')
1248         b.dir.write()
1249         ok = False
1250
1251         if opts.gnupghome is None:
1252                 opts.gnupghome = tmpdir+'/gnupg'
1253
1254         b._debug('initialising')
1255         try:
1256                 for x in ['pubring','secring']:
1257                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1258                 ok = True
1259         except OSError, oe:
1260                 if oe.errno != errno.ENOENT: raise
1261
1262         if ok: b._debug('no key generation needed')
1263         else: b.genkey()
1264
1265  def _debug(b, s):
1266         debug('* '+s)
1267
1268  def genkey(b):
1269         b._debug('preparing for key generation')
1270
1271         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1272         mkdir_okexist(opts.gnupghome, 0700)
1273
1274         script = '''
1275   exec >&2
1276   cd "$1"
1277   cat <<"END" >key-gen-params
1278 Key-Type: DSA
1279 Key-Length: 1024
1280 Key-Usage: sign
1281 Name-Real: autopkgtest per-run key
1282 Name-Comment: do not trust this key
1283 Name-Email: autopkgtest@example.com
1284 END
1285   set -x
1286   gpg --homedir="$1" --batch --gen-key key-gen-params
1287                 '''
1288         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1289         rc = subprocess_cooked(cmdl, dbg=(genkey,script))[0]
1290         if rc:
1291                 try:
1292                         f = open(opts.gnupghome+'/key-gen-log')
1293                         tp = file.read()
1294                 except OSError, e: tp = e
1295                 pstderr(tp)
1296                 bomb('key generation failed, code %d' % rc)
1297
1298  def apt_configs(b):
1299         return {
1300                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1301         }
1302
1303  def apt_pkg_gdebi_script(b, arg, middle):
1304         script = [
1305                 'import apt_pkg',
1306                 'import urllib',
1307                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1308                 ]
1309         for (k,v) in b.apt_configs().iteritems():
1310                 v = urllib.quote(v)
1311                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1312                                 % (k, v))
1313         script += [
1314                 'from GDebi.Cache import Cache',
1315                 'cache = Cache()',
1316                 ]
1317         for m in middle:
1318                 script += m + [
1319                 'print res',
1320                 'print d.missingDeps',
1321                 'print d.requiredChanges',
1322                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1323                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1324                           'd._failureString)',
1325                 'cache.commit()',
1326                 ''
1327                 ]
1328         return '\n'.join(script)
1329  def apt_get(b):
1330         ag = ['apt-get','-qy']
1331         for kv in b.apt_configs().iteritems():
1332                 ag += ['-o', '%s=%s' % kv]
1333         return ag
1334
1335  def reset(b):
1336         b._debug('reset')
1337         rmtree('binaries', b.dir.read())
1338         b.dir.invalidate()
1339         b.dir.write()
1340         b.install = []
1341         b.blamed = []
1342         b.registered = set()
1343
1344  def register(b, act, pkg, af, forwhat, blamed):
1345         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1346                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1347
1348         if act.ah['deb_'+forwhat] == 'ignore': return
1349
1350         b.blamed += testbed.blamed
1351
1352         leafname = pkg+'.deb'
1353         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1354
1355         try: os.remove(dest.write())
1356         except OSError, oe:
1357                 if oe.errno != errno.ENOENT: raise e
1358
1359         try: os.link(af.read(), dest.write())
1360         except OSError, oe:
1361                 if oe.errno != errno.EXDEV: raise e
1362                 shutil.copy(af.read(), dest)
1363
1364         if act.ah['deb_'+forwhat] == 'install':
1365                 b.install.append(pkg)
1366
1367         b.registered.add(pkg)
1368
1369  def publish(b):
1370         b._debug('publish')
1371
1372         script = '''
1373   exec >&2
1374   cd "$1"
1375   apt-ftparchive packages . >Packages
1376   gzip <Packages >Packages.gz
1377   apt-ftparchive release . >Release
1378   rm -f Release.gpg
1379   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1380   gpg --homedir="$2" --batch --export >archive-key.pgp
1381                 '''
1382         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1383         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1384         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1385
1386         b.dir.invalidate(True)
1387         apt_source = b.dir.read(True)
1388
1389         so = TemporaryFile('vlds')
1390         script = '''
1391   exec 3>&1 >&2
1392   apt-key add archive-key.pgp
1393   echo "deb file://'''+apt_source+''' /" >sources.list
1394   cat /etc/apt/sources.list >>sources.list
1395   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1396     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1397   fi
1398   '''+ ' '.join(b.apt_get()) +''' update >&2
1399   cat /var/lib/dpkg/status >&3
1400                 '''
1401         testbed.mungeing_apt()
1402         rc = testbed.execute('apt-key', ['sh','-ec',script],
1403                                 so=so.write(True), cwd=b.dir.write(True),
1404                                 script=script, kind='install')
1405         if rc: bomb('apt setup failed with exit code %d' % rc)
1406
1407         testbed.blamed += b.blamed
1408
1409         b._debug('publish reinstall checking...')
1410         pkgs_reinstall = set()
1411         pkg = None
1412         for l in open(so.read()):
1413                 if l.startswith('Package: '):
1414                         pkg = l[9:].rstrip()
1415                 elif l.startswith('Status: install '):
1416                         if pkg in b.registered:
1417                                 pkgs_reinstall.add(pkg)
1418                                 b._debug(' publish reinstall needs '+pkg)
1419
1420         if pkgs_reinstall:
1421                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1422                 what = 'apt-get-reinstall'
1423                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1424                         [pkg for pkg in pkgs_reinstall])
1425                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1426                 if rc: badpkg("installation of basic binarries failed,"
1427                                 " exit code %d" % rc)
1428
1429         b._debug('publish install...')
1430         for pkg in b.install:
1431                 what = 'apt-get-install-%s' % pkg
1432                 testbed.blame(pkg)
1433                 cmdl = b.apt_get() + ['install',pkg]
1434                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1435                 if rc: badpkg("installation of %s failed, exit code %d"
1436                                 % (pkg, rc))
1437
1438         b._debug('publish done')
1439
1440 #---------- processing of sources (building)
1441
1442 def source_rules_command(act,script,what,which,work,cwd,
1443                                 results_lines=0,xargs=[]):
1444         script = [      "exec 3>&1 >&2",
1445                         "set -x"        ] + script
1446         script = '\n'.join(script)
1447         so = TemporaryFile('%s-%s-results' % (what,which))
1448         rc = testbed.execute('%s-%s' % (what,which),
1449                         ['sh','-ec',script]+xargs, script=script,
1450                         so=so.write(True), cwd=cwd, kind='build')
1451         results = open(so.read()).read().rstrip('\n')
1452         if len(results): results = results.split("\n")
1453         else: results = []
1454         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1455         if results_lines is not None and len(results) != results_lines:
1456                 badpkg("got %d lines of results from %s where %d expected"
1457                         % (len(results), which, results_lines))
1458         if results_lines==1: return results[0]
1459         return results
1460
1461 def build_source(act, control_override):
1462         act.blame = 'arg:'+act.af.spec
1463         testbed.blame(act.blame)
1464         testbed.prepare1([])
1465         testbed.needs_reset()
1466
1467         what = act.what
1468         basename = act.af.spec
1469         debiancontrol = None
1470         act.binaries = []
1471
1472         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1473
1474         if act.kind == 'dsc':
1475                 dsc = act.af
1476                 dsc_file = open(dsc.read())
1477                 in_files = False
1478                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1479                 for l in dsc_file:
1480                         l = l.rstrip('\n')
1481                         if l.startswith('Files:'): in_files = True; continue
1482                         elif l.startswith('#'): pass
1483                         elif not l.startswith(' '):
1484                                 in_files = False
1485                                 if l.startswith('Source:'):
1486                                         act.blame = 'dsc:'+l[7:].strip()
1487                                         testbed.blame(act.blame)
1488                         if not in_files: continue
1489
1490                         m = fre.match(l)
1491                         if not m: badpkg(".dsc contains unparseable line"
1492                                         " in Files: `%s'" % l)
1493                         leaf = m.groups(0)[0]
1494                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1495                                         sibling=True)
1496                         subfile.read(True)
1497                 dsc.read(True)
1498
1499         if act.kind == 'ubtree':
1500                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1501                         act.af, 'debian/control')
1502                 dsc = TemporaryFile(what+'-fakedsc')
1503                 dsc_w = open(dsc.write(), 'w')
1504                 for l in open(debiancontrol.read()):
1505                         l = l.rstrip('\n')
1506                         if not len(l): break
1507                         print >>dsc_w, l
1508                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1509                 dsc_w.close()
1510
1511         if act.kind == 'dsc':
1512                 testbed.prepare2([])
1513                 script = binaries.apt_pkg_gdebi_script('', [[
1514                                 'from GDebi.DebPackage import DebPackage',
1515                                 'd = DebPackage(cache)',
1516                                 'res = d.satisfyDependsStr("dpkg-dev")',
1517                         ]])
1518                 cmdl = ['python','-c',script]
1519                 whatp = what+'-dpkgsource'
1520                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1521                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1522
1523         work = TemporaryDir(what+'-build')
1524         act.work = work
1525
1526         tmpdir = work.write(True)+'/tmpdir'
1527         tmpdir_script = [
1528                         'TMPDIR="$1"',
1529                         'rm -rf -- "$TMPDIR"',
1530                         'export TMPDIR',
1531                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1532                 ]
1533
1534         if act.kind == 'ubtree':
1535                 spec = '%s/real-tree' % work.write(True)
1536                 create_command = '''
1537                         rm -rf "$spec"
1538                         mkdir "$spec"
1539                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1540                         '''
1541                 initcwd = act.af.read(True)
1542
1543         if act.kind == 'dsc':
1544                 spec = dsc.read(True)
1545                 create_command = '''
1546                         dpkg-source -x $spec
1547                         '''
1548                 initcwd = work.write(True)
1549
1550         script = [
1551                 'spec="$2"',
1552                 'origpwd=`pwd`',
1553                 'cd '+work.write(True)
1554         ]
1555
1556         if opts.user:
1557                 script += ([ 'chown '+opts.user+' .' ] +
1558                         tmpdir_script +
1559                         [ 'spec="$spec" origpwd="$origpwd" '
1560                                 +opts.user_wrap(create_command) ])
1561         else:
1562                 script += (tmpdir_script +
1563                         [ create_command ])
1564
1565         script += [
1566                         'cd */.',
1567                         'pwd >&3',
1568                         'set +e; test -f debian/tests/control; echo $? >&3'
1569                 ]
1570         (result_pwd, control_test_rc) = source_rules_command(
1571                         act,script,what,'extract',work,
1572                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1573
1574         filter = act.ah['dsc_filter']
1575
1576         if control_test_rc == '1': act.missing_tests_control = True
1577
1578         # For optional builds:
1579         #
1580         # We might need to build the package because:
1581         #   - we want its binaries (filter isn't _ and at least one of the
1582         #       deb_... isn't ignore)
1583         #   - the test control file says so
1584         #       (assuming we have any tests)
1585
1586         class NeedBuildException: pass
1587         def build_needed(m):
1588                 debug_b('build needed for %s' % m)
1589                 raise NeedBuildException()
1590
1591         try:
1592                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1593                                       act.ah['deb_fortests'] != 'ignore'):
1594                         build_needed('binaries')
1595
1596                 result_pwd_af = InputDir(what+'-treeforcontrol',
1597                         result_pwd, True)
1598                 stanzas = read_control(act, result_pwd_af, control_override)
1599                 for stanza in stanzas:
1600                         for t in stanza[' tests']:
1601                                 if 'no-build-needed' not in t.feature_names:
1602                                         build_needed('test %s' % t.tname)
1603                                 for d in t.depends:
1604                                         if '@' in d:
1605                                                 build_needed('test %s '
1606                                                  'dependency %s' % (t.tname,d))
1607
1608                 debug_b('build not needed')
1609                 built = False
1610
1611         except NeedBuildException:
1612
1613                 if act.kind != 'dsc':
1614                         testbed.prepare2([])
1615
1616                 script = binaries.apt_pkg_gdebi_script(
1617                         dsc.read(True), [[
1618                         'from GDebi.DscSrcPackage import DscSrcPackage',
1619                         'd = DscSrcPackage(cache, arg)',
1620                         'res = d.checkDeb()',
1621                          ],[
1622                         'from GDebi.DebPackage import DebPackage',
1623                         'd = DebPackage(cache)',
1624                         'res = d.satisfyDependsStr("'+
1625                                         ','.join(build_essential)+
1626                                 '")',
1627                         ]])
1628
1629                 cmdl = ['python','-c',script]
1630                 whatp = what+'-builddeps'
1631                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1632                 if rc: badpkg('build-depends install failed,'
1633                               ' exit code %d' % rc)
1634
1635                 script = tmpdir_script + [
1636                         'cd "$2"',
1637                         'dpkg-checkbuilddeps',
1638                         opts.user_wrap('debian/rules build'),
1639                         ]
1640                 source_rules_command(act,script,what,'build',work,
1641                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1642
1643                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1644                         badpkg("results dir `%s' is not in expected parent"
1645                                " dir `%s'" % (result_pwd, work.read(True)))
1646
1647                 built = True
1648
1649         act.tests_tree = InputDir(what+'-tests-tree',
1650                                 work.read(True)+os.path.basename(result_pwd),
1651                                 True)
1652         if act.ah['dsc_tests']:
1653                 testbed.register_ephemeral(act.work)
1654                 testbed.register_ephemeral(act.tests_tree)
1655
1656         if not built:
1657                 act.blamed = []
1658                 return
1659
1660         act.blamed = copy.copy(testbed.blamed)
1661
1662         debug_b('filter=%s' % filter)
1663         if filter != '_':
1664                 script = tmpdir_script + [
1665                         'cd '+work.write(True)+'/*/.',
1666                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1667                         'cd ..',
1668                         'echo *.deb >&3',
1669                         ]
1670                 result_debs = source_rules_command(act,script,what,
1671                                 'binary',work,work.write(True),
1672                                 results_lines=1, xargs=['x',tmpdir])
1673                 if result_debs == '*': debs = []
1674                 else: debs = result_debs.split(' ')
1675                 debug_b('debs='+`debs`)
1676                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1677                 for deb in debs:
1678                         m = re.match(deb)
1679                         if not m: badpkg("badly-named binary `%s'" % deb)
1680                         pkg = m.groups()[0]
1681                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1682                         for pat in filter.split(','):
1683                                 debug_b('  pat=%s' % pat)
1684                                 if not fnmatch.fnmatchcase(pkg,pat):
1685                                         debug_b('   no match')
1686                                         continue
1687                                 deb_what = pkg+'_'+what+'.deb'
1688                                 bin = RelativeInputFile(deb_what,work,deb,True)
1689                                 debug_b('  deb_what=%s, bin=%s' %
1690                                         (deb_what, str(bin)))
1691                                 binaries.register(act,pkg,bin,
1692                                         'forbuilds',testbed.blamed)
1693                                 act.binaries.append((pkg,bin))
1694                                 break
1695                 debug_b('all done.')
1696
1697 #---------- main processing loop and main program
1698
1699 def process_actions():
1700         global binaries
1701
1702         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1703         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1704         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1705
1706         debug_a1('starting')
1707         testbed.open()
1708         binaries = Binaries()
1709
1710         for act in opts.actions:
1711                 if act.af is not None and not act.af.spec_tbp:
1712                         testbed.register_ephemeral(act.af)
1713
1714         binaries.reset()
1715         control_override = None
1716
1717         debug_a1('builds ...')
1718         for act in opts.actions:
1719                 debug_a2('%s %s' %
1720                         (act.kind, act.what))
1721
1722                 if act.kind == 'control':
1723                         control_override = act.af
1724                 if act.kind == 'deb':
1725                         testbed.blame('arg:'+act.af.spec)
1726                         determine_package(act)
1727                         testbed.blame('deb:'+act.pkg)
1728                         binaries.register(act,act.pkg,act.af,
1729                                 'forbuilds',testbed.blamed)
1730                 if act.kind == 'dsc' or act.kind == 'ubtree':
1731                         build_source(act, control_override)
1732                 if act.kind == 'tree':
1733                         act.binaries = []
1734                 if act.kind.endswith('tree') or act.kind == 'dsc':
1735                         control_override = None
1736                 if act.kind == 'instantiate':
1737                         pass
1738
1739         debug_a1('builds done.')
1740
1741         binaries.reset()
1742         control_override = None
1743
1744         debug_a1('tests ...')
1745         for act in opts.actions:
1746                 debug_a2('test %s %s' % (act.kind, act.what))
1747
1748                 testbed.needs_reset()
1749                 if act.kind == 'control':
1750                         control_override = act.af
1751                 if act.kind == 'deb':
1752                         binaries.register(act,act.pkg,act.af,'fortests',
1753                                 ['deb:'+act.pkg])
1754                 if act.kind == 'dsc' or act.kind == 'ubtree':
1755                         for (pkg,bin) in act.binaries:
1756                                 binaries.register(act,pkg,bin,'fortests',
1757                                         act.blamed)
1758                 if act.kind == 'dsc':
1759                         if act.ah['dsc_tests']:
1760                                 debug_a3('read control ...')
1761                                 stanzas = read_control(act, act.tests_tree,
1762                                                 control_override)
1763                                 testbed.blamed += act.blamed
1764                                 debug_a3('run_tests ...')
1765                                 run_tests(stanzas, act.tests_tree)
1766                         control_override = None
1767                 if act.kind == 'tree' or act.kind == 'ubtree':
1768                         testbed.blame('arg:'+act.af.spec)
1769                         stanzas = read_control(act, act.af, control_override)
1770                         debug_a3('run_tests ...')
1771                         run_tests(stanzas, act.af)
1772                         control_override = None
1773                 if act.kind == 'instantiate':
1774                         testbed.prepare([])
1775         debug_a1('tests done.')
1776
1777 def main():
1778         global testbed
1779         global tmpdir
1780         try:
1781                 parse_args()
1782         except SystemExit, se:
1783                 os._exit(20)
1784         try:
1785                 setup_trace()
1786                 testbed = Testbed()
1787                 testbed.start()
1788                 finalise_options()
1789                 process_actions()
1790         except:
1791                 ec = print_exception(sys.exc_info(), '')
1792                 cleanup()
1793                 os._exit(ec)
1794         cleanup()
1795         os._exit(errorcode)
1796
1797 main()