runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 binaries = None         # Binaries (.debs we have registered)
  49
  50 #---------- output handling
  51 #
  52 #  There are at least the following kinds of output:
  53 #
  54 #   1. stderr output which consists of
  55 #   1a. our errors
  56 #   1b. stuff printed to stderr by the virtualisation server
  57 #   1c. stuff printed to stderr by our short-lived subprocesseses
  58 #           which we don't expect to fail
  59 #
  60 #   2. trace information, which consists of
  61 #   2a. our trace information from calls to debug()
  62 #   2b. progress information and stderr output from
  63 #        general scripts we run on the host
  64 #   2c. progress information and stderr output from things
  65 #        we run on the testbed including builds
  66 #   2d. stderr and stdout output from actual tests
  67 #
  68 #   xxxx
  69 #   3. actual test results (printed to our stdout)
  70 #
  71 # Cloning of 1a and 2a, where necessary, is done by us writing
  72 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  73 # is done by forking off a copy of ourselves to do plumbing,
  74 # which copy we wait for at the appropriate point.
  75
  76 trace_stream = None
  77
  78 def pstderr(m):
  79         print >>sys.stderr, m
  80         if trace_stream is not None: print >>trace_stream, m
  81
  82 def debug(m):
  83         if not opts.debug and trace_stream is None: return
  84         for l in m.rstrip('\n').split('\n'):
  85                 s = 'atd-run: debug: ' + l
  86                 if opts.debug: print >>sys.stderr, s
  87                 if trace_stream is not None: print >>trace_stream, s
  88
  89 def debug_file(hp):
  90         def do_copy(stream, what):
  91                 rc = subprocess.call(['cat',hp], stdout=stream)
  92                 if rc: bomb('cat failed copying data from %s'
  93                             ' to %s, exit code %d' % (hp, what, rc))
  94         if opts.debug: do_copy(sys.stderr, 'stderr')
  95         if trace_stream is not None: do_copy(trace_stream, 'trace log')
  96
  97 class Errplumb:
  98  def __init__(ep, critical=False):
  99         to_stderr = critical or opts.debug
 100         count = to_stderr + (trace_stream is not None)
 101         if count == 0:
 102                 ep.stream = open('/dev/null','w')
 103                 ep._sp = None
 104         elif count == 1:
 105                 if to_stderr: ep.stream = sys.stderr
 106                 else: ep.stream = trace_stream
 107                 ep._sp = None
 108         else:
 109                 ep._sp = subprocess.Popen(['tee','/dev/stderr'],
 110                         stdin=subprocess.PIPE, stdout=trace_stream,
 111                         close_fds=True)
 112                 ep.stream = ep._sp.stdin
 113  def wait(ep):
 114         if ep._sp is None: return
 115         ep._sp.stdin.close()
 116         rc = ep._sp.wait()
 117         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 118         ep._sp = None
 119
 120 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 121         if dbg is not None:
 122                 if isinstance(dbg,tuple): (what,script) = dbg
 123                 else: (what,script) = (dbg,None)
 124                 debug_subprocess(what, cmdl, script=script)
 125         ep = Errplumb(critical)
 126         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 127         output = running.communicate()[0]
 128         rc = running.wait()
 129         ep.wait()
 130         return (rc, output)
 131
 132 def preport(m):
 133         print m
 134         if trace_stream is not None: print >>trace_stream, m
 135
 136 def report(tname, result):
 137         preport('%-20s %s' % (tname, result))
 138
 139 #---------- errors we define
 140
 141 class Quit:
 142         def __init__(q,ec,m): q.ec = ec; q.m = m
 143
 144 def bomb(m):
 145         raise Quit(20, "unexpected error: %s" % m)
 146
 147 def badpkg(m):
 148         preport('blame: ' + ' '.join(testbed.blamed))
 149         preport('badpkg: ' + m)
 150         raise Quit(12, "erroneous package: %s" % m)
 151
 152 class Unsupported:
 153  def __init__(u, lno, m):
 154         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 155         else: u.m = m
 156  def report(u, tname):
 157         global errorcode
 158         errorcode != 2
 159         report(tname, 'SKIP %s' % u.m)
 160
 161 #---------- convenience function
 162
 163 def mkdir_okexist(pathname, mode=02755):
 164         try:
 165                 os.mkdir(pathname, mode)
 166         except OSError, oe:
 167                 if oe.errno != errno.EEXIST: raise
 168
 169 def rmtree(what, pathname):
 170         debug('/ %s rmtree %s' % (what, pathname))
 171         shutil.rmtree(pathname)
 172
 173 def debug_subprocess(what, cmdl=None, script=None):
 174         o = '$ '+what+':'
 175         if cmdl is not None:
 176                 ol = []
 177                 for x in cmdl:
 178                         if x is script: x = '<SCRIPT>'
 179                         ol.append(x.    replace('\\','\\\\').
 180                                         replace(' ','\\ ')      )
 181                 o += ' '+ ' '.join(ol)
 182         if script is not None:
 183                 o += '\n'
 184                 for l in script.rstrip('\n').split('\n'):
 185                         o += '$     '+l+'\n'
 186         debug(o)
 187
 188 def flatten(l):
 189         return reduce((lambda a,b: a + b), l, [])
 190
 191 #---------- fancy automatic file-copying class
 192
 193 class AutoFile:
 194         # p.what
 195         # p.path[tb]    None or path    not None => path known
 196         # p.file[tb]    None or path    not None => file exists
 197         # p.spec        string or None
 198         # p.spec_tbp    True or False, or not set if spec is None
 199         # p.dir         '' or '/'
 200
 201  def __init__(p, what):
 202         p.what = what
 203         p.path = [None,None]
 204         p.file = [None,None]
 205         p.spec = None
 206         p.dir = ''
 207
 208  def __repr__(p):
 209         return "<AF@%s>" % p.__str__()
 210  def __str__(p):
 211         def ptbp(tbp):
 212                 if p.path[tbp] is None: return '-'+p.dir
 213                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 214                 else: return p.path[tbp]+p.dir+'!'
 215         out = p.what
 216         if p.spec is not None:
 217                 if p.spec_tbp: out += '#'
 218                 else: out += '='
 219                 out += p.spec
 220         out += ':'
 221         out += ptbp(False)
 222         out += '|'
 223         out += ptbp(True)
 224         return out
 225
 226  def _wrong(p, how):
 227         xtra = ''
 228         if p.spec is not None: xtra = ' spec[%s]=%s' % (p.spec, p.spec_tb)
 229         raise ("internal error: %s (%s)" % (how, str(p)))
 230
 231  def _ensure_path(p, tbp):
 232         if p.path[tbp] is None:
 233                 if '/' in p.what:
 234                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 235                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 236                 if not tbp:
 237                         p.path[tbp] = tmpdir+'/'+p.what
 238                 else:
 239                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 240
 241  def write(p, tbp=False):
 242         p._debug('write %s...' % 'HT'[tbp])
 243         p._ensure_path(tbp)
 244
 245         if p.dir and not p.file[tbp]:
 246                 if not tbp:
 247                         p._debug('mkdir H')
 248                         mkdir_okexist(p.path[tbp])
 249                 else:
 250                         cmdl = ['sh','-ec',
 251                                 'test -d "$1" || mkdir -p "$1"',
 252                                 'x', p.path[tbp]]
 253                         tf_what = urllib.quote(p.what).replace('/','%2F')
 254                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 255                         if rc: bomb('failed to create directory %s' %
 256                                 p.path[tbp])
 257
 258         p.file[tbp] = p.path[tbp]
 259         return p.path[tbp]
 260
 261  def read(p, tbp=False):
 262         p._debug('read %s...' % 'HT'[tbp])
 263         p._ensure_path(tbp)
 264
 265         if p.file[tbp] is None:
 266                 if p.file[not tbp] is None:
 267                         p._wrong("requesting read but nonexistent")
 268                 cud = ['copyup','copydown'][tbp]
 269                 src = p.file[not tbp] + p.dir
 270                 dst = p.path[tbp] + p.dir
 271                 testbed.command(cud, (src, dst))
 272                 p.file[tbp] = p.path[tbp]
 273
 274         return p.file[tbp] + p.dir
 275
 276  def invalidate(p, tbp=False):
 277         p.file[tbp] = None
 278         p._debug('invalidated %s' % 'HT'[tbp])
 279
 280  def _debug(p, m):
 281         debug('/ %s#%x: %s' % (p.what, id(p), m))
 282
 283  def _constructed(p):
 284         p._debug('constructed: '+str(p))
 285         p._check()
 286
 287  def _check(p):
 288         for tbp in [False,True]:
 289          for pf in [p.path, p.file]:
 290                 if pf[tbp] is None: continue
 291                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 292                 if p.dir and pf[tbp].endswith('/'):
 293                         pf[tbp] = pf[tbp].rstrip('/')
 294                         if not pf[tbp]: pf[tbp] = '/'
 295                 if not p.dir and pf[tbp].endswith('/'):
 296                         bomb("directory `%s' specified for "
 297                              "non-directory %s" % (pf[tbp], p.what))
 298
 299  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 300         AutoFile.__init__(p,what)
 301         sh_on = ''; sh_sibl = ''
 302         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 303         if sibling: sh_sibl=' (sibling)'
 304         parent._debug('using as base: %s: %s%s%s' %
 305                         (str(parent), leaf, sh_on, sh_sibl))
 306         if not sibling and not parent.dir:
 307                 parent._wrong('asked for non-sibling relative path of non-dir')
 308         if sibling: trim = os.path.dirname
 309         else: trim = lambda x: x
 310         for tbp in [False,True]:
 311                 if parent.path[tbp] is None: continue
 312                 trimmed = trim(parent.path[tbp])
 313                 if trimmed: trimmed += '/'
 314                 p.path[tbp] = trimmed + leaf
 315                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 316                         p.file[tbp] = p.path[tbp]
 317
 318 class InputFile(AutoFile):
 319  def _init(p, what, spec, spec_tbp=False):
 320         AutoFile.__init__(p, what)
 321         p.spec = spec
 322         p.spec_tbp = spec_tbp
 323         p.path[spec_tbp] = spec
 324         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 325  def __init__(p, what, spec, spec_tbp=False):
 326         p._init(what,spec,spec_tbp)
 327         p._constructed()
 328
 329 class InputDir(InputFile):
 330  def __init__(p, what, spec, spec_tbp=False):
 331         InputFile._init(p,what,spec,spec_tbp)
 332         p.dir = '/'
 333         p._constructed()
 334
 335 class OutputFile(AutoFile):
 336  def _init(p, what, spec, spec_tbp=False):
 337         AutoFile.__init__(p, what)
 338         p.spec = spec
 339         p.spec_tbp = spec_tbp
 340         p.path[spec_tbp] = spec
 341  def __init__(p, what, spec, spec_tbp=False):
 342         p._init(what,spec,spec_tbp)
 343         p._constructed()
 344
 345 class OutputDir(OutputFile):
 346  def __init__(p, what, spec, spec_tbp=False):
 347         OutputFile._init(p,what,spec,spec_tbp)
 348         p.dir = '/'
 349         p._constructed()
 350
 351 class RelativeInputFile(AutoFile):
 352  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 353         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 354         p._constructed()
 355
 356 class RelativeOutputFile(AutoFile):
 357  def __init__(p, what, parent, leaf, sibling=False):
 358         p._relative_init(what, parent, leaf, None, False, sibling)
 359         p._constructed()
 360
 361 class TemporaryFile(AutoFile):
 362  def __init__(p, what):
 363         AutoFile.__init__(p, what)
 364         p._constructed()
 365
 366 class TemporaryDir(AutoFile):
 367  def __init__(p, what):
 368         AutoFile.__init__(p, what)
 369         p.dir = '/'
 370         p._constructed()
 371
 372 #---------- parsing and representation of the arguments
 373
 374 class Action:
 375  def __init__(a, kind, af, arghandling, what):
 376         # extra attributes get added during processing
 377         a.kind = kind
 378         a.af = af
 379         a.ah = arghandling
 380         a.what = what
 381  def __repr__(a):
 382         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 383
 384 def parse_args():
 385         global opts
 386         usage = "%prog <options> --- <virt-server>..."
 387         parser = OptionParser(usage=usage)
 388         pa = parser.add_option
 389         pe = parser.add_option
 390
 391         arghandling = {
 392                 'dsc_tests': True,
 393                 'dsc_filter': '*',
 394                 'deb_forbuilds': 'auto',
 395                 'deb_fortests': 'auto',
 396                 'tb': False,
 397                 'override_control': None
 398         }
 399         initial_arghandling = arghandling.copy()
 400         n_non_actions = 0
 401
 402         #----------
 403         # actions (ie, test sets to run, sources to build, binaries to use):
 404
 405         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 406                 parser.largs.append((value,kindpath))
 407                 n_non_actions += not(is_act)
 408
 409         def pa_action(long, metavar, kindpath, help, is_act=True):
 410                 pa('','--'+long, action='callback', callback=cb_action,
 411                         nargs=1, type='string',
 412                         callback_args=(long,kindpath,is_act), help=help)
 413
 414         pa_action('build-tree',         'TREE', '@/',
 415                 help='run tests from build tree TREE')
 416
 417         pa_action('source',             'DSC', '@.dsc',
 418                 help='build DSC and use its tests and/or'
 419                     ' generated binary packages')
 420
 421         pa_action('binary',             'DEB', '@.deb',
 422                help='use binary package DEB according'
 423                     ' to most recent --binaries-* settings')
 424
 425         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 426                help='run tests from control file CONTROL instead,'
 427                     ' (applies to next test suite only)')
 428
 429         #----------
 430         # argument handling settings (what ways to use action
 431         #  arguments, and pathname processing):
 432
 433         def cb_setah(option, opt_str, value, parser, toset,setval):
 434                 if type(setval) == list:
 435                         if not value in setval:
 436                                 parser.error('value for %s option (%s) is not '
 437                                  'one of the permitted values (%s)' %
 438                                  (value, opt_str, setval.join(' ')))
 439                 elif setval is not None:
 440                         value = setval
 441                 for v in toset:
 442                         arghandling[v] = value
 443                 parser.largs.append(arghandling.copy())
 444
 445         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 446                 type = metavar
 447                 if type is not None: type = 'string'
 448                 pa('',long, action='callback', callback=cb_setah,
 449                    callback_args=(affected,effect), **kwargs)
 450
 451         #---- paths: host or testbed:
 452         #
 453         pa_setah('--paths-testbed', ['tb'],True,
 454                 help='subsequent path specifications refer to the testbed')
 455         pa_setah('--paths-host', ['tb'],False,
 456                 help='subsequent path specifications refer to the host')
 457
 458         #---- source processing settings:
 459
 460         pa_setah('--sources-tests', ['dsc_tests'],True,
 461                 help='run tests from builds of subsequent sources')
 462         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 463                 help='do not run tests from builds of subsequent sources')
 464
 465         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 466                 type='string', metavar='PATTERN-LIST',
 467                 help='from subsequent sources, use binaries matching'
 468                      ' PATTERN-LIST (comma-separated glob patterns)'
 469                      ' according to most recent --binaries-* settings')
 470         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 471                 help='from subsequent sources, do not use any binaries')
 472
 473         #---- binary package processing settings:
 474
 475         def pa_setahbins(long,toset,how):
 476          pa_setah(long, toset,['ignore','auto','install'],
 477                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 478                 help=how+' ignore binaries, install them as needed'
 479                         ' for dependencies, or unconditionally install'
 480                         ' them, respectively')
 481         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 482         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 483         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 484
 485         #----------
 486         # general options:
 487
 488         def cb_vserv(op,optstr,value,parser):
 489                 parser.values.vserver = list(parser.rargs)
 490                 del parser.rargs[:]
 491
 492         def cb_path(op,optstr,value,parser, constructor,long,dir):
 493                 name = long.replace('-','_')
 494                 af = constructor(arghandling['tb'], value, long, dir)
 495                 setattr(parser.values, name, af)
 496
 497         def pa_path(long, constructor, help, dir=False):
 498                 pa('','--'+long, action='callback', callback=cb_path,
 499                         callback_args=(constructor,long,dir),
 500                         nargs=1, type='string',
 501                         help=help, metavar='PATH')
 502
 503         pa_path('output-dir', OutputDir, dir=True,
 504                 help='write stderr/out files in PATH')
 505
 506         pa('','--tmp-dir',              type='string', dest='tmpdir',
 507                 help='write temporary files to TMPDIR, emptying it'
 508                      ' beforehand and leaving it behind at the end')
 509         pa('','--log-file',             type='string', dest='logfile',
 510                 help='write the log LOGFILE, emptying it beforehand,'
 511                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 512
 513         pa('','--user',                 type='string', dest='user',
 514                 help='run tests as USER (needs root on testbed)')
 515         pa('','--gain-root',            type='string', dest='gainroot',
 516                 help='prefix debian/rules binary with GAINROOT')
 517         pa('-q', '--quiet', action='store_false', dest='debug', default=True);
 518         pa('','--gnupg-home',           type='string', dest='gnupghome',
 519                 default='~/.autopkgtest/gpg',
 520                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 521                         " signing private apt archive);"
 522                         " `fresh' means generate new key each time.")
 523
 524         #----------
 525         # actual meat:
 526
 527         class SpecialOption(optparse.Option): pass
 528         vs_op = SpecialOption('','--VSERVER-DUMMY')
 529         vs_op.action = 'callback'
 530         vs_op.type = None
 531         vs_op.default = None
 532         vs_op.nargs = 0
 533         vs_op.callback = cb_vserv
 534         vs_op.callback_args = ( )
 535         vs_op.callback_kwargs = { }
 536         vs_op.help = 'introduces virtualisation server and args'
 537         vs_op._short_opts = []
 538         vs_op._long_opts = ['---']
 539
 540         pa(vs_op)
 541
 542         (opts,args) = parser.parse_args()
 543         if not hasattr(opts,'vserver'):
 544                 parser.error('you must specifiy --- <virt-server>...')
 545         if n_non_actions >= len(parser.largs):
 546                 parser.error('nothing to do specified')
 547
 548         arghandling = initial_arghandling
 549         opts.actions = []
 550         ix = 0
 551         for act in args:
 552                 if type(act) == dict:
 553                         arghandling = act
 554                         continue
 555                 elif type(act) == tuple:
 556                         pass
 557                 elif type(act) == str:
 558                         act = (act,act)
 559                 else:
 560                         raise ("unknown action in list `%s' having"
 561                               " type `%s'" % (act, type(act)))
 562                 (pathstr, kindpath) = act
 563
 564                 constructor = InputFile
 565                 if type(kindpath) is tuple:             kind = kindpath[0]
 566                 elif kindpath.endswith('.deb'):         kind = 'deb'
 567                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 568                 elif kindpath.endswith('/'):
 569                         kind = 'tree'
 570                         constructor = InputDir
 571                 else: parser.error("do not know how to handle filename \`%s';"
 572                         " specify --source --binary or --build-tree")
 573
 574                 what = '%s%s' % (kind,ix); ix += 1
 575
 576                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 577                 else: fwhatx = '-'+kind
 578
 579                 af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 580                 opts.actions.append(Action(kind, af, arghandling, what))
 581
 582 def setup_trace():
 583         global trace_stream, tmpdir
 584
 585         if opts.tmpdir is not None:
 586                 rmtree('tmpdir(specified)',opts.tmpdir)
 587                 mkdir_okexist(opts.tmpdir, 0700)
 588                 tmpdir = opts.tmpdir
 589         else:
 590                 assert(tmpdir is None)
 591                 tmpdir = tempfile.mkdtemp()
 592
 593         if opts.logfile is None:
 594                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 595                         opts.logfile = opts.output_dir.spec + '/log'
 596                 elif opts.tmpdir is not None:
 597                         opts.logfile = opts.tmpdir + '/log'
 598         if opts.logfile is not None:
 599                 trace_stream = open(opts.logfile, 'w', 0)
 600
 601         debug('#options: '+`opts`)
 602
 603 def finalise_options():
 604         global opts, tb
 605
 606         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 607                 opts.user = ''
 608
 609         if opts.user is None:
 610                 su = 'suggested-normal-user='
 611                 ul = [
 612                         e[len(su):]
 613                         for e in testbed.caps
 614                         if e.startswith(su)
 615                         ]
 616                 if ul:
 617                         opts.user = ul[0]
 618                 else:
 619                         opts.user = ''
 620
 621         if opts.user:
 622                 if 'root-on-testbed' not in testbed.caps:
 623                         pstderr("warning: virtualisation"
 624                                 " system does not offer root on testbed,"
 625                                 " but --user option specified: failure likely")
 626                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 627         else:
 628                 opts.user_wrap = lambda x: x
 629
 630         if opts.gainroot is None:
 631                 opts.gainroot = ''
 632                 if (opts.user or
 633                     'root-on-testbed' not in testbed.caps):
 634                         opts.gainroot = 'fakeroot'
 635
 636         if opts.gnupghome.startswith('~/'):
 637                 try: home = os.environ['HOME']
 638                 except KeyError:
 639                         parser.error("HOME environment variable"
 640                                 " not set, needed for --gnupghome=`%s"
 641                                 % opts.gnupghome)
 642                 opts.gnupghome = home + opts.gnupghome[1:]
 643         elif opts.gnupghome == 'fresh':
 644                 opts.gnupghome = None
 645
 646 #---------- testbed management - the Testbed class
 647
 648 class Testbed:
 649  def __init__(tb):
 650         tb.sp = None
 651         tb.lastsend = None
 652         tb.scratch = None
 653         tb.modified = False
 654         tb.blamed = []
 655         tb._ephemeral = []
 656         tb._debug('init')
 657         tb._need_reset_apt = False
 658  def _debug(tb, m):
 659         debug('** '+m)
 660  def start(tb):
 661         tb._debug('start')
 662         p = subprocess.PIPE
 663         debug_subprocess('vserver', opts.vserver)
 664         tb._errplumb = Errplumb(True)
 665         tb.sp = subprocess.Popen(opts.vserver,
 666                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 667         tb.expect('ok')
 668         tb.caps = tb.commandr('capabilities')
 669  def stop(tb):
 670         tb._debug('stop')
 671         tb.close()
 672         if tb.sp is None: return
 673         ec = tb.sp.returncode
 674         if ec is None:
 675                 tb.sp.stdout.close()
 676                 tb.send('quit')
 677                 tb.sp.stdin.close()
 678                 ec = tb.sp.wait()
 679         if ec:
 680                 tb.bomb('testbed gave exit status %d after quit' % ec)
 681         tb._errplumb.wait()
 682  def open(tb):
 683         tb._debug('open, scratch=%s' % tb.scratch)
 684         if tb.scratch is not None: return
 685         pl = tb.commandr('open')
 686         tb.scratch = InputDir('tb-scratch', pl[0], True)
 687         tb.deps_processed = []
 688  def mungeing_apt(tb):
 689         if not 'revert' in tb.caps:
 690                 tb._need_reset_apt = True
 691  def reset_apt(tb):
 692         if not tb._need_reset_apt: return
 693         what = 'aptget-update-reset'
 694         cmdl = ['apt-get','-qy','update']
 695         rc = tb.execute(what, cmdl)
 696         if rc:
 697                 pstderr("\n" "warning: failed to restore"
 698                         " testbed apt cache, exit code %d" % rc)
 699         tb._need_reset_apt = False
 700  def close(tb):
 701         tb._debug('close, scratch=%s' % tb.scratch)
 702         if tb.scratch is None: return
 703         tb.scratch = None
 704         if tb.sp is None: return
 705         tb.command('close')
 706  def prepare(tb, deps_new):
 707         tb._debug('prepare, modified=%s, deps_processed=%s, deps_new=%s' %
 708                 (tb.modified, tb.deps_processed, deps_new))
 709         if 'revert' in tb.caps and (tb.modified or
 710             [d for d in tb.deps_processed if d not in deps_new]):
 711                 tb._debug('reset **')
 712                 tb.command('revert')
 713                 tb.blamed = []
 714                 for af in tb._ephemeral: af.invalidate(True)
 715         binaries.publish()
 716         tb.modified = False
 717         tb._install_deps(deps_new)
 718  def register_ephemeral(tb, af):
 719         tb._ephemeral.append(af)
 720  def _install_deps(tb, deps_new):
 721         tb._debug(' installing dependencies '+`deps_new`)
 722         tb.deps_processed = deps_new
 723         if not deps_new: return
 724         dstr = ', '.join(deps_new)
 725         script = binaries.apt_pkg_gdebi_script(
 726                 dstr, [[
 727                 'from GDebi.DebPackage import DebPackage',
 728                 'd = DebPackage(cache)',
 729                 'res = d.satisfyDependsStr(arg)',
 730                 ]])
 731         cmdl = ['python','-c',script]
 732         what = 'install-deps'
 733         rc = testbed.execute(what, cmdl, script=script)
 734         if rc: badpkg('dependency install failed, exit code %d' % rc)
 735  def needs_reset(tb):
 736         tb._debug('needs_reset, previously=%s' % tb.modified)
 737         tb.modified = True
 738  def blame(tb, m):
 739         tb._debug('blame += %s' % m)
 740         tb.blamed.append(m)
 741  def bomb(tb, m):
 742         tb._debug('bomb %s' % m)
 743         if tb.sp is not None:
 744                 tb.sp.stdout.close()
 745                 tb.sp.stdin.close()
 746                 ec = tb.sp.wait()
 747                 if ec: pstderr('adt-run: testbed failing,'
 748                                 ' exit status %d' % ec)
 749         tb.sp = None
 750         raise Quit(16, 'testbed failed: %s' % m)
 751  def send(tb, string):
 752         tb.sp.stdin
 753         try:
 754                 debug('>> '+string)
 755                 print >>tb.sp.stdin, string
 756                 tb.sp.stdin.flush()
 757                 tb.lastsend = string
 758         except:
 759                 (type, value, dummy) = sys.exc_info()
 760                 tb.bomb('cannot send to testbed: %s' % traceback.
 761                         format_exception_only(type, value))
 762  def expect(tb, keyword, nresults=None):
 763         l = tb.sp.stdout.readline()
 764         if not l: tb.bomb('unexpected eof from the testbed')
 765         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 766         l = l.rstrip('\n')
 767         debug('<< '+l)
 768         ll = l.split()
 769         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 770         if ll[0] != keyword:
 771                 if tb.lastsend is None:
 772                         tb.bomb("got banner `%s', expected `%s...'" %
 773                                 (l, keyword))
 774                 else:
 775                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 776                                 (tb.lastsend, l, keyword))
 777         ll = ll[1:]
 778         if nresults is not None and len(ll) != nresults:
 779                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 780                         " expected %d result parameters" %
 781                         (string, l, len(ll), nresults))
 782         return ll
 783  def commandr(tb, cmd, args=(), nresults=None):
 784         # pass args=[None,...] or =(None,...) to avoid more url quoting
 785         if type(cmd) is str: cmd = [cmd]
 786         if len(args) and args[0] is None: args = args[1:]
 787         else: args = map(urllib.quote, args)
 788         al = cmd + args
 789         tb.send(string.join(al))
 790         ll = tb.expect('ok', nresults)
 791         rl = map(urllib.unquote, ll)
 792         return rl
 793  def command(tb, cmd, args=()):
 794         tb.commandr(cmd, args, 0)
 795  def commandr1(tb, cmd, args=()):
 796         rl = tb.commandr(cmd, args, 1)
 797         return rl[0]
 798  def execute(tb, what, cmdl,
 799                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 800                 script=False):
 801         # Options for script:
 802         #   False - do not call debug_subprocess, no synch. reporting required
 803         #   None or string - call debug_subprocess with that value,
 804         #                       plumb stderr through synchronously if possible
 805         # Options for se:
 806         #   None - usual Errplumb (output is of kind 2c)
 807         #   string - path on testbed (output is of kind 2d)
 808
 809         if script is not False: debug_subprocess(what, cmdl, script=script)
 810         if cwd is None: cwd = tb.scratch.write(True)
 811
 812         xdump = None
 813         if se is None:
 814                 ep = Errplumb()
 815                 se_catch = TemporaryFile(what+'-xerr')
 816                 se_use = se_catch.write(True)
 817                 if opts.debug: xdump = 'debug=2-2'
 818                 elif trace_stream is not None:
 819                         xdump = 'debug=2-%d' % trace_stream.fileno()
 820         else:
 821                 ep = None
 822                 se_catch = None
 823                 se_use = se
 824
 825         cmdl = [None,
 826                 ','.join(map(urllib.quote, cmdl)),
 827                 si, so, se_use, cwd]
 828         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 829         rc = tb.commandr1('execute', cmdl)
 830         try: rc = int(rc)
 831         except ValueError: bomb("execute for %s gave invalid response `%s'"
 832                                         % (what,rc))
 833
 834         if se_catch is not None:
 835                 debug_file(se_catch.read())
 836         if ep is not None:
 837                 ep.wait()
 838
 839         return rc
 840
 841 #---------- representation of test control files: Field*, Test, etc.
 842
 843 class FieldBase:
 844  def __init__(f, fname, stz, base, tnames, vl):
 845         assert(vl)
 846         f.stz = stz
 847         f.base = base
 848         f.tnames = tnames
 849         f.vl = vl
 850  def words(f):
 851         def distribute(vle):
 852                 (lno, v) = vle
 853                 r = v.split()
 854                 r = map((lambda w: (lno, w)), r)
 855                 return r
 856         return flatten(map(distribute, f.vl))
 857  def atmostone(f):
 858         if len(vl) == 1:
 859                 (f.lno, f.v) = vl[0]
 860         else:
 861                 raise Unsupported(f.vl[1][0],
 862                         'only one %s field allowed' % fn)
 863         return f.v
 864
 865 class FieldIgnore(FieldBase):
 866  def parse(f): pass
 867
 868 class Restriction:
 869  def __init__(r,rname,base): pass
 870
 871 class Restriction_rw_build_tree(Restriction): pass
 872 class Restriction_rw_tests_tree(Restriction): pass
 873 class Restriction_breaks_testbed(Restriction):
 874  def __init__(r, rname, base):
 875         if 'revert' not in testbed.caps:
 876                 raise Unsupported(f.lno,
 877                         'Test breaks testbed but testbed cannot revert')
 878
 879 class Field_Restrictions(FieldBase):
 880  def parse(f):
 881         for wle in f.words():
 882                 (lno, rname) = wle
 883                 rname = rname.replace('-','_')
 884                 try: rclass = globals()['Restriction_'+rname]
 885                 except KeyError: raise Unsupported(lno,
 886                         'unknown restriction %s' % rname)
 887                 r = rclass(rname, f.base)
 888                 f.base['restrictions'].append(r)
 889
 890 class Field_Tests(FieldIgnore): pass
 891
 892 class Field_Depends(FieldBase):
 893  def parse(f):
 894         dl = map(lambda x: x.strip(),
 895                 flatten(map(lambda x: x.split(','), f.vl)))
 896         re = regexp.compile('[^-.+:~]')
 897         for d in dl:
 898                 if re.search(d):
 899                         badpkg("Test Depends field contains dependency"
 900                                " `%s' with invalid characters" % d)
 901         f.base['depends'] = dl
 902
 903 class Field_Tests_directory(FieldBase):
 904  def parse(f):
 905         td = atmostone(f)
 906         if td.startswith('/'): raise Unspported(f.lno,
 907                 'Tests-Directory may not be absolute')
 908         f.base['testsdir'] = td
 909
 910 def run_tests(stanzas, tree):
 911         global errorcode, testbed
 912         for stanza in stanzas:
 913                 tests = stanza[' tests']
 914                 if not tests:
 915                         report('*', 'SKIP no tests in this package')
 916                         errorcode |= 8
 917                 for t in tests:
 918                         t.prepare()
 919                         t.run(tree)
 920                         if 'breaks-testbed' in t.restrictions:
 921                                 testbed.needs_reset()
 922                 testbed.needs_reset()
 923
 924 class Test:
 925  def __init__(t, tname, base, act):
 926         if '/' in tname: raise Unsupported(base[' lno'],
 927                 'test name may not contain / character')
 928         for k in base: setattr(t,k,base[k])
 929         t.tname = tname
 930         t.act = act
 931         t.what = act.what+'t-'+tname
 932         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
 933         else: t.path = tname
 934         t._debug('constructed; path=%s' % t.path)
 935         t._debug(' .depends=%s' % t.depends)
 936  def _debug(t, m):
 937         debug('& %s: %s' % (t.what, m))
 938  def report(t, m):
 939         report(t.what, m)
 940  def reportfail(t, m):
 941         global errorcode
 942         errorcode |= 4
 943         report(t.what, 'FAIL ' + m)
 944  def prepare(t):
 945         t._debug('preparing')
 946         dn = []
 947         for d in t.depends:
 948                 t._debug(' processing dependency '+d)
 949                 if not '_' in d:
 950                         t._debug('  literal dependency '+d)
 951                         dn.append(d)
 952                 else:
 953                         for (pkg,bin) in t.act.binaries:
 954                                 d = d.replace('_',pkg)
 955                                 t._debug('  synthesised dependency '+d)
 956                                 dn.append(d)
 957         testbed.prepare(dn)
 958  def run(t, tree):
 959         t._debug('[----------------------------------------')
 960         def stdouterr(oe):
 961                 idstr = t.what + '-' + oe
 962                 if opts.output_dir is not None and opts.output_dir.tb:
 963                         use_dir = opts.output_dir
 964                 else:
 965                         use_dir = testbed.scratch
 966                 return RelativeOutputFile(idstr, use_dir, idstr)
 967
 968         t.act.work.write(True)
 969
 970         af = RelativeInputFile(t.what, tree, t.path)
 971         so = stdouterr('stdout')
 972         se = stdouterr('stderr')
 973         rc = testbed.execute('test-'+t.what,
 974                 [opts.user_wrap(af.read(True))],
 975                 so=so.write(True), se=se.write(True), cwd=tree.read(True))
 976
 977         so_read = so.read()
 978         se_read = se.read()
 979
 980         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
 981         stab = os.stat(se_read)
 982         if stab.st_size != 0:
 983                 l = open(se_read).readline()
 984                 l = l.rstrip('\n \t\r')
 985                 if len(l) > 40: l = l[:40] + '...'
 986                 t.reportfail('status: %d, stderr: %s' % (rc, l))
 987                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
 988                 debug_file(se_read)
 989         elif rc != 0:
 990                 t.reportfail('non-zero exit status %d' % rc)
 991         else:
 992                 t.report('PASS')
 993
 994         stab = os.stat(so_read)
 995         if stab.st_size != 0:
 996                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
 997                 debug_file(so_read)
 998
 999         t._debug('----------------------------------------]')
1000
1001 def read_control(act, tree, control_override):
1002         stanzas = [ ]
1003
1004         if control_override is not None:
1005                 control_af = control_override
1006                 testbed.blame('arg:'+control_override.spec)
1007         else:
1008                 control_af = RelativeInputFile(act.what+'-testcontrol',
1009                         tree, 'debian/tests/control')
1010         try:
1011                 control = open(control_af.read(), 'r')
1012         except OSError, oe:
1013                 if oe[0] != errno.ENOENT: raise
1014                 return []
1015
1016         lno = 0
1017         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1018         stz = None      # stz[field_name][index] = (lno, value)
1019                         # special field names:
1020                         # stz[' lno'] = number
1021                         # stz[' tests'] = list of Test objects
1022         def end_stanza(stz):
1023                 if stz is None: return
1024                 stz[' errs'] = 0
1025                 stanzas.append(stz)
1026                 stz = None
1027                 hcurrent = None
1028
1029         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1030         while 1:
1031                 l = control.readline()
1032                 if not l: break
1033                 lno += 1
1034                 if not l.endswith('\n'): badctrl('unterminated line')
1035                 if regexp.compile('\s*\#').match(l): continue
1036                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1037                 initmat = initre.match(l)
1038                 if initmat:
1039                         (fname, l) = initmat.groups()
1040                         fname = string.capwords(fname)
1041                         if stz is None:
1042                                 stz = { ' lno': lno, ' tests': [] }
1043                         if not stz.has_key(fname): stz[fname] = [ ]
1044                         hcurrent = stz[fname]
1045                 elif regexp.compile('\s').match(l):
1046                         if not hcurrent: badctrl('unexpected continuation')
1047                 else:
1048                         badctrl('syntax error')
1049                 hcurrent.append((lno, l))
1050         end_stanza(stz)
1051
1052         def testbadctrl(stz, lno, m):
1053                 report_badctrl(lno, m)
1054                 stz[' errs'] += 1
1055
1056         for stz in stanzas:
1057                 try:
1058                         try: tnames = stz['Tests']
1059                         except KeyError:
1060                                 tnames = ['*']
1061                                 raise Unsupported(stz[' lno'],
1062                                         'no Tests field')
1063                         tnames = map((lambda lt: lt[1]), tnames)
1064                         tnames = string.join(tnames).split()
1065                         base = {
1066                                 'restrictions': [],
1067                                 'testsdir': 'debian/tests',
1068                                 'depends' : '_'
1069                         }
1070                         for fname in stz.keys():
1071                                 if fname.startswith(' '): continue
1072                                 vl = stz[fname]
1073                                 try: fclass = globals()['Field_'+
1074                                         fname.replace('-','_')]
1075                                 except KeyError: raise Unsupported(vl[0][0],
1076                                         'unknown metadata field %s' % fname)
1077                                 f = fclass(stz, fname, base, tnames, vl)
1078                                 f.parse()
1079                         for tname in tnames:
1080                                 t = Test(tname, base, act)
1081                                 stz[' tests'].append(t)
1082                 except Unsupported, u:
1083                         for tname in tnames: u.report(tname)
1084                         continue
1085
1086         return stanzas
1087
1088 def print_exception(ei, msgprefix=''):
1089         if msgprefix: pstderr(msgprefix)
1090         (et, q, tb) = ei
1091         if et is Quit:
1092                 pstderr('adt-run: ' + q.m)
1093                 return q.ec
1094         else:
1095                 pstderr("adt-run: unexpected, exceptional, error:")
1096                 traceback.print_exc(None, sys.stderr)
1097                 if trace_stream is not None:
1098                         traceback.print_exc(None, trace_stream)
1099                 return 20
1100
1101 def cleanup():
1102         global trace_stream
1103         try:
1104                 rm_ec = 0
1105                 if opts.tmpdir is None and tmpdir is not None:
1106                         rmtree('tmpdir', tmpdir)
1107                 if testbed is not None:
1108                         testbed.reset_apt()
1109                         testbed.stop()
1110                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1111                 if trace_stream is not None:
1112                         trace_stream.close()
1113                         trace_stream = None
1114         except:
1115                 print_exception(sys.exc_info(),
1116                         '\nadt-run: error cleaning up:\n')
1117                 os._exit(20)
1118
1119 #---------- registration, installation etc. of .deb's: Binaries
1120
1121 def determine_package(act):
1122         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1123         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1124         if rc: badpkg('failed to parse binary package, code %d' % rc)
1125         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1126         act.pkg = None
1127         for l in output.split('\n'):
1128                 m = re.match(l)
1129                 if not m: continue
1130                 if act.pkg: badpkg('two Package: lines in control file')
1131                 act.pkg = m.groups()[0]
1132         if not act.pkg: badpkg('no good Package: line in control file')
1133
1134 class Binaries:
1135  def __init__(b):
1136         b.dir = TemporaryDir('binaries')
1137         b.dir.write()
1138         ok = False
1139
1140         if opts.gnupghome is None:
1141                 opts.gnupghome = tmpdir+'/gnupg'
1142
1143         b._debug('initialising')
1144         try:
1145                 for x in ['pubring','secring']:
1146                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1147                 ok = True
1148         except OSError, oe:
1149                 if oe.errno != errno.ENOENT: raise
1150
1151         if ok: b._debug('no key generation needed')
1152         else: b.genkey()
1153
1154  def _debug(b, s):
1155         debug('* '+s)
1156
1157  def genkey(b):
1158         b._debug('preparing for key generation')
1159
1160         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1161         mkdir_okexist(opts.gnupghome, 0700)
1162
1163         script = '''
1164   exec >&2
1165   cd "$1"
1166   cat <<"END" >key-gen-params
1167 Key-Type: DSA
1168 Key-Length: 1024
1169 Key-Usage: sign
1170 Name-Real: autopkgtest per-run key
1171 Name-Comment: do not trust this key
1172 Name-Email: autopkgtest@example.com
1173 END
1174   set -x
1175   gpg --homedir="$1" --batch --gen-key key-gen-params
1176                 '''
1177         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1178         rc = subprocess_cooked(cmdl, dbg=(genkey,script))[0]
1179         if rc:
1180                 try:
1181                         f = open(opts.gnupghome+'/key-gen-log')
1182                         tp = file.read()
1183                 except OSError, e: tp = e
1184                 pstderr(tp)
1185                 bomb('key generation failed, code %d' % rc)
1186
1187  def apt_configs(b):
1188         return {
1189                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1190         }
1191
1192  def apt_pkg_gdebi_script(b, arg, middle):
1193         script = [
1194                 'import apt_pkg',
1195                 'import urllib',
1196                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1197                 ]
1198         for (k,v) in b.apt_configs().iteritems():
1199                 v = urllib.quote(v)
1200                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1201                                 % (k, v))
1202         script += [
1203                 'from GDebi.Cache import Cache',
1204                 'cache = Cache()',
1205                 ]
1206         for m in middle:
1207                 script += m + [
1208                 'print res',
1209                 'print d.missingDeps',
1210                 'print d.requiredChanges',
1211                 'assert(res)',
1212                 'cache.commit()',
1213                 ''
1214                 ]
1215         return '\n'.join(script)
1216  def apt_get(b):
1217         ag = ['apt-get','-qy']
1218         for kv in b.apt_configs().iteritems():
1219                 ag += ['-o', '%s=%s' % kv]
1220         return ag
1221
1222  def reset(b):
1223         b._debug('reset')
1224         rmtree('binaries', b.dir.read())
1225         b.dir.invalidate()
1226         b.dir.write()
1227         b.install = []
1228         b.blamed = []
1229         b.registered = set()
1230
1231  def register(b, act, pkg, af, forwhat, blamed):
1232         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1233                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1234
1235         if act.ah['deb_'+forwhat] == 'ignore': return
1236
1237         b.blamed += testbed.blamed
1238
1239         leafname = pkg+'.deb'
1240         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1241
1242         try: os.remove(dest.write())
1243         except OSError, oe:
1244                 if oe.errno != errno.ENOENT: raise e
1245
1246         try: os.link(af.read(), dest.write())
1247         except OSError, oe:
1248                 if oe.errno != errno.EXDEV: raise e
1249                 shutil.copy(af.read(), dest)
1250
1251         if act.ah['deb_'+forwhat] == 'install':
1252                 b.install.append(pkg)
1253
1254         b.registered.add(pkg)
1255
1256  def publish(b):
1257         b._debug('publish')
1258
1259         script = '''
1260   exec >&2
1261   cd "$1"
1262   apt-ftparchive packages . >Packages
1263   gzip <Packages >Packages.gz
1264   apt-ftparchive release . >Release
1265   rm -f Release.gpg
1266   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1267   gpg --homedir="$2" --batch --export >archive-key.pgp
1268                 '''
1269         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1270         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1271         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1272
1273         b.dir.invalidate(True)
1274         apt_source = b.dir.read(True)
1275
1276         so = TemporaryFile('vlds')
1277         script = '''
1278   exec 3>&1 >&2
1279   apt-key add archive-key.pgp
1280   echo "deb file://'''+apt_source+''' /" >sources.list
1281   cat /etc/apt/sources.list >>sources.list
1282   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1283     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1284   fi
1285   '''+ ' '.join(b.apt_get()) +''' update >&2
1286   cat /var/lib/dpkg/status >&3
1287                 '''
1288         testbed.mungeing_apt()
1289         rc = testbed.execute('apt-key', ['sh','-ec',script],
1290                                 so=so.write(True), cwd=b.dir.write(True),
1291                                 script=script)
1292         if rc: bomb('apt setup failed with exit code %d' % rc)
1293
1294         testbed.blamed += b.blamed
1295
1296         b._debug('publish reinstall checking...')
1297         pkgs_reinstall = set()
1298         pkg = None
1299         for l in open(so.read()):
1300                 if l.startswith('Package: '):
1301                         pkg = l[9:].rstrip()
1302                 elif l.startswith('Status: install '):
1303                         if pkg in b.registered:
1304                                 pkgs_reinstall.add(pkg)
1305                                 b._debug(' publish reinstall needs '+pkg)
1306
1307         if pkgs_reinstall:
1308                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1309                 what = 'apt-get-reinstall'
1310                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1311                         [pkg for pkg in pkgs_reinstall])
1312                 rc = testbed.execute(what, cmdl, script=None)
1313                 if rc: badpkg("installation of basic binarries failed,"
1314                                 " exit code %d" % rc)
1315
1316         b._debug('publish install...')
1317         for pkg in b.install:
1318                 what = 'apt-get-install-%s' % pkg
1319                 testbed.blame(pkg)
1320                 cmdl = b.apt_get() + ['install',pkg]
1321                 rc = testbed.execute(what, cmdl, script=None)
1322                 if rc: badpkg("installation of %s failed, exit code %d"
1323                                 % (pkg, rc))
1324
1325         b._debug('publish done')
1326
1327 #---------- processing of sources (building)
1328
1329 def source_rules_command(act,script,what,which,work,results_lines=0):
1330         script = [      "exec 3>&1 >&2",
1331                         "set -x"        ] + script
1332         script = '\n'.join(script)
1333         so = TemporaryFile('%s-%s-results' % (what,which))
1334         rc = testbed.execute('%s-%s' % (what,which),
1335                         ['sh','-xec',script], script=script,
1336                         so=so.write(True), cwd= work.write(True))
1337         results = open(so.read()).read().rstrip('\n').split("\n")
1338         if rc: badpkg("%s failed with exit code %d" % (which,rc))
1339         if results_lines is not None and len(results) != results_lines:
1340                 badpkg("got %d lines of results from %s where %d expected"
1341                         % (len(results), which, results_lines))
1342         if results_lines==1: return results[0]
1343         return results
1344
1345 def build_source(act):
1346         act.blame = 'arg:'+act.af.spec
1347         testbed.blame(act.blame)
1348         testbed.needs_reset()
1349
1350         what = act.what
1351         dsc = act.af
1352         basename = dsc.spec
1353
1354         dsc_file = open(dsc.read())
1355         in_files = False
1356         fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1357         for l in dsc_file:
1358                 l = l.rstrip('\n')
1359                 if l.startswith('Files:'): in_files = True; continue
1360                 elif l.startswith('#'): pass
1361                 elif not l.startswith(' '):
1362                         in_files = False
1363                         if l.startswith('Source:'):
1364                                 act.blame = 'dsc:'+l[7:].strip()
1365                                 testbed.blame(act.blame)
1366                 if not in_files: continue
1367
1368                 m = fre.match(l)
1369                 if not m: badpkg(".dsc contains unparseable line"
1370                                 " in Files: `%s'" % l)
1371                 leaf = m.groups(0)[0]
1372                 subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1373                                 sibling=True)
1374                 subfile.read(True)
1375         dsc.read(True)
1376
1377         script = binaries.apt_pkg_gdebi_script(
1378                 dsc.read(True), [[
1379                 'from GDebi.DscSrcPackage import DscSrcPackage',
1380                 'd = DscSrcPackage(cache, arg)',
1381                 'res = d.checkDeb()',
1382                  ],[
1383                 'from GDebi.DebPackage import DebPackage',
1384                 'd = DebPackage(cache)',
1385                 'res = d.satisfyDependsStr("build-essential")',
1386                 ]])
1387         cmdl = ['python','-c',script]
1388         whatp = what+'-builddeps'
1389         rc = testbed.execute(what, cmdl, script=script)
1390         if rc: badpkg('build-depends install failed, exit code %d' % rc)
1391
1392         work = TemporaryDir(what+'-build')
1393
1394         script = [
1395                         'cd '+work.write(True),
1396                 ]
1397         if opts.user: script += [
1398                         'chown '+opts.user+' .',
1399                         'dsc=dsc.read(True) '+
1400                                 opts.user_wrap('dpkg-source -x $dsc')
1401                 ]
1402         else: script += [
1403                         'dpkg-source -x '+dsc.read(True),
1404                 ]
1405         script += [
1406                         'cd */.',
1407                         'dpkg-checkbuilddeps',
1408                         'pwd >&3',
1409                         opts.user_wrap('debian/rules build'),
1410                 ]
1411         result_pwd = source_rules_command(act,script,what,'build',work,1)
1412
1413         if os.path.dirname(result_pwd)+'/' != work.read(True):
1414                 badpkg("results dir `%s' is not in expected parent dir `%s'"
1415                         % (result_pwd, work.read(True)))
1416
1417         act.work = work
1418         act.tests_tree = InputDir(what+'-tests-tree',
1419                                 work.read(True)+os.path.basename(result_pwd),
1420                                 True)
1421         if act.ah['dsc_tests']:
1422                 act.tests_tree.read()
1423                 testbed.register_ephemeral(act.work)
1424                 testbed.register_ephemeral(act.tests_tree)
1425
1426         act.blamed = copy.copy(testbed.blamed)
1427
1428         def debug_b(m): debug('* <dsc:%s> %s' % (act.what, m))
1429         act.binaries = []
1430         filter = act.ah['dsc_filter']
1431         debug_b('filter=%s' % filter)
1432         if filter != '_':
1433                 script = [
1434                         'cd '+work.write(True)+'/*/.',
1435                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1436                         'cd ..',
1437                         'echo *.deb >&3',
1438                         ]
1439                 result_debs = source_rules_command(act,script,what,
1440                                 'binary',work,1)
1441                 if result_debs == '*': debs = []
1442                 else: debs = result_debs.split(' ')
1443                 debug_b('debs='+`debs`)
1444                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1445                 for deb in debs:
1446                         m = re.match(deb)
1447                         if not m: badpkg("badly-named binary `%s'" % deb)
1448                         pkg = m.groups()[0]
1449                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1450                         for pat in filter.split(','):
1451                                 debug_b('  pat=%s' % pat)
1452                                 if not fnmatch.fnmatchcase(pkg,pat):
1453                                         debug_b('   no match')
1454                                         continue
1455                                 deb_what = pkg+'_'+what+'.deb'
1456                                 bin = RelativeInputFile(deb_what,work,deb,True)
1457                                 debug_b('  deb_what=%s, bin=%s' %
1458                                         (deb_what, str(bin)))
1459                                 binaries.register(act,pkg,bin,
1460                                         'forbuilds',testbed.blamed)
1461                                 act.binaries.append((pkg,bin))
1462                                 break
1463                 debug_b('all done.')
1464
1465 #---------- main processing loop and main program
1466
1467 def process_actions():
1468         global binaries
1469
1470         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1471         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1472         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1473
1474         debug_a1('starting')
1475         testbed.open()
1476         binaries = Binaries()
1477
1478         binaries.reset()
1479
1480         debug_a1('builds ...')
1481         for act in opts.actions:
1482                 debug_a2('%s %s' %
1483                         (act.kind, act.what))
1484
1485                 testbed.prepare([])
1486                 if act.kind == 'deb':
1487                         testbed.blame('arg:'+act.af.spec)
1488                         determine_package(act)
1489                         testbed.blame('deb:'+act.pkg)
1490                         binaries.register(act,act.pkg,act.af,
1491                                 'forbuilds',testbed.blamed)
1492                 if act.kind == 'dsc':
1493                         build_source(act)
1494                 if act.kind == 'tree':
1495                         act.binaries = []
1496
1497         debug_a1('builds done.')
1498
1499         binaries.reset()
1500         control_override = None
1501
1502         debug_a1('tests ...')
1503         for act in opts.actions:
1504                 debug_a2('test %s %s' % (act.kind, act.what))
1505
1506                 testbed.needs_reset()
1507                 if act.kind == 'control':
1508                         control_override = act.af
1509                 if act.kind == 'deb':
1510                         binaries.register(act,act.pkg,act.af,'fortests',
1511                                 ['deb:'+act.pkg])
1512                 if act.kind == 'dsc':
1513                         for (pkg,bin) in act.binaries:
1514                                 binaries.register(act,pkg,bin,'fortests',
1515                                         act.blamed)
1516                         if act.ah['dsc_tests']:
1517                                 debug_a3('read control ...')
1518                                 stanzas = read_control(act, act.tests_tree,
1519                                                 control_override)
1520                                 testbed.blamed += act.blamed
1521                                 debug_a3('run_tests ...')
1522                                 run_tests(stanzas, act.tests_tree)
1523                         control_override = None
1524                 if act.kind == 'tree':
1525                         testbed.blame('arg:'+act.af.spec)
1526                         stanzas = read_control(act, act.af, control_override)
1527                         debug_a3('run_tests ...')
1528                         run_tests(stanzas, act.af)
1529                         control_override = None
1530         debug_a1('tests done.')
1531
1532 def main():
1533         global testbed
1534         global tmpdir
1535         try:
1536                 parse_args()
1537         except SystemExit, se:
1538                 os._exit(20)
1539         try:
1540                 setup_trace()
1541                 testbed = Testbed()
1542                 testbed.start()
1543                 finalise_options()
1544                 process_actions()
1545         except:
1546                 ec = print_exception(sys.exc_info(), '')
1547                 cleanup()
1548                 os._exit(ec)
1549         cleanup()
1550         os._exit(errorcode)
1551
1552 main()