runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 binaries = None         # Binaries (.debs we have registered)
  49
  50 #---------- output handling
  51 #
  52 #  There are at least the following kinds of output:
  53 #
  54 #   1. stderr output which consists of
  55 #   1a. our errors
  56 #   1b. stuff printed to stderr by the virtualisation server
  57 #   1c. stuff printed to stderr by our short-lived subprocesseses
  58 #           which we don't expect to fail
  59 #
  60 #   2. trace information, which consists of
  61 #   2a. our trace information from calls to debug()
  62 #   2b. progress information and stderr output from
  63 #        general scripts we run on the host
  64 #   2c. progress information and stderr output from things
  65 #        we run on the testbed including builds
  66 #   2d. stderr and stdout output from actual tests
  67 #
  68 #   xxxx
  69 #   3. actual test results (printed to our stdout)
  70 #
  71 # Cloning of 1a and 2a, where necessary, is done by us writing
  72 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  73 # is done by forking off a copy of ourselves to do plumbing,
  74 # which copy we wait for at the appropriate point.
  75
  76 trace_stream = None
  77
  78 def pstderr(m):
  79         print >>sys.stderr, m
  80         if trace_stream is not None: print >>trace_stream, m
  81
  82 def debug(m, minlevel=0):
  83         if opts.debuglevel < minlevel: return
  84         if opts.quiet and trace_stream is None: return
  85         p = 'adt-run: debug'
  86         if minlevel: p += `minlevel`
  87         p += ': '
  88         for l in m.rstrip('\n').split('\n'):
  89                 s = p + l
  90                 if not opts.quiet: print >>sys.stderr, s
  91                 if trace_stream is not None: print >>trace_stream, s
  92
  93 def debug_file(hp, minlevel=0):
  94         if opts.debuglevel < minlevel: return
  95         def do_copy(stream, what):
  96                 rc = subprocess.call(['cat',hp], stdout=stream)
  97                 if rc: bomb('cat failed copying data from %s'
  98                             ' to %s, exit code %d' % (hp, what, rc))
  99         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 100         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 101
 102 class Errplumb:
 103  def __init__(ep, critical=False):
 104         to_stderr = critical or not opts.quiet
 105         count = to_stderr + (trace_stream is not None)
 106         if count == 0:
 107                 ep.stream = open('/dev/null','w')
 108                 ep._sp = None
 109         elif count == 1:
 110                 if to_stderr: ep.stream = sys.stderr
 111                 else: ep.stream = trace_stream
 112                 ep._sp = None
 113         else:
 114                 ep._sp = subprocess.Popen(['tee','/dev/stderr'],
 115                         stdin=subprocess.PIPE, stdout=trace_stream,
 116                         close_fds=True)
 117                 ep.stream = ep._sp.stdin
 118  def wait(ep):
 119         if ep._sp is None: return
 120         ep._sp.stdin.close()
 121         rc = ep._sp.wait()
 122         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 123         ep._sp = None
 124
 125 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 126         if dbg is not None:
 127                 if isinstance(dbg,tuple): (what,script) = dbg
 128                 else: (what,script) = (dbg,None)
 129                 debug_subprocess(what, cmdl, script=script)
 130         ep = Errplumb(critical)
 131         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 132         output = running.communicate()[0]
 133         rc = running.wait()
 134         ep.wait()
 135         return (rc, output)
 136
 137 def preport(m):
 138         print m
 139         if trace_stream is not None: print >>trace_stream, m
 140
 141 def report(tname, result):
 142         preport('%-20s %s' % (tname, result))
 143
 144 #---------- errors we define
 145
 146 class Quit:
 147         def __init__(q,ec,m): q.ec = ec; q.m = m
 148
 149 def bomb(m):
 150         raise Quit(20, "unexpected error: %s" % m)
 151
 152 def badpkg(m):
 153         preport('blame: ' + ' '.join(testbed.blamed))
 154         preport('badpkg: ' + m)
 155         raise Quit(12, "erroneous package: %s" % m)
 156
 157 class Unsupported:
 158  def __init__(u, lno, m):
 159         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 160         else: u.m = m
 161  def report(u, tname):
 162         global errorcode
 163         errorcode != 2
 164         report(tname, 'SKIP %s' % u.m)
 165
 166 #---------- convenience function
 167
 168 def mkdir_okexist(pathname, mode=02755):
 169         try:
 170                 os.mkdir(pathname, mode)
 171         except OSError, oe:
 172                 if oe.errno != errno.EEXIST: raise
 173
 174 def rmtree(what, pathname):
 175         debug('/ %s rmtree %s' % (what, pathname), 2)
 176         shutil.rmtree(pathname)
 177
 178 def debug_subprocess(what, cmdl=None, script=None):
 179         o = '$ '+what+':'
 180         if cmdl is not None:
 181                 ol = []
 182                 for x in cmdl:
 183                         if x is script: x = '<SCRIPT>'
 184                         ol.append(x.    replace('\\','\\\\').
 185                                         replace(' ','\\ ')      )
 186                 o += ' '+ ' '.join(ol)
 187         debug(o)
 188         if script is not None and opts.debuglevel >= 1:
 189                 o = ''
 190                 for l in script.rstrip('\n').split('\n'):
 191                         o += '$     '+l+'\n'
 192                 debug(o,1)
 193
 194 def flatten(l):
 195         return reduce((lambda a,b: a + b), l, [])
 196
 197 #---------- fancy automatic file-copying class
 198
 199 class AutoFile:
 200         # p.what
 201         # p.path[tb]    None or path    not None => path known
 202         # p.file[tb]    None or path    not None => file exists
 203         # p.spec        string or None
 204         # p.spec_tbp    True or False, or not set if spec is None
 205         # p.dir         '' or '/'
 206
 207  def __init__(p, what):
 208         p.what = what
 209         p.path = [None,None]
 210         p.file = [None,None]
 211         p.spec = None
 212         p.dir = ''
 213
 214  def __repr__(p):
 215         return "<AF@%s>" % p.__str__()
 216  def __str__(p):
 217         def ptbp(tbp):
 218                 if p.path[tbp] is None: return '-'+p.dir
 219                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 220                 else: return p.path[tbp]+p.dir+'!'
 221         out = p.what
 222         if p.spec is not None:
 223                 if p.spec_tbp: out += '#'
 224                 else: out += '='
 225                 out += p.spec
 226         out += ':'
 227         out += ptbp(False)
 228         out += '|'
 229         out += ptbp(True)
 230         return out
 231
 232  def _wrong(p, how):
 233         xtra = ''
 234         if p.spec is not None: xtra = ' spec[%s]=%s' % (p.spec, p.spec_tb)
 235         raise ("internal error: %s (%s)" % (how, str(p)))
 236
 237  def _ensure_path(p, tbp):
 238         if p.path[tbp] is None:
 239                 if '/' in p.what:
 240                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 241                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 242                 if not tbp:
 243                         p.path[tbp] = tmpdir+'/'+p.what
 244                 else:
 245                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 246
 247  def write(p, tbp=False):
 248         p._debug('write %s...' % 'HT'[tbp])
 249         p._ensure_path(tbp)
 250
 251         if p.dir and not p.file[tbp]:
 252                 if not tbp:
 253                         p._debug('mkdir H')
 254                         mkdir_okexist(p.path[tbp])
 255                 else:
 256                         cmdl = ['sh','-ec',
 257                                 'test -d "$1" || mkdir -p "$1"',
 258                                 'x', p.path[tbp]]
 259                         tf_what = urllib.quote(p.what).replace('/','%2F')
 260                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 261                         if rc: bomb('failed to create directory %s' %
 262                                 p.path[tbp])
 263
 264         p.file[tbp] = p.path[tbp]
 265         return p.path[tbp]
 266
 267  def read(p, tbp=False):
 268         p._debug('read %s...' % 'HT'[tbp])
 269         p._ensure_path(tbp)
 270
 271         if p.file[tbp] is None:
 272                 if p.file[not tbp] is None:
 273                         p._wrong("requesting read but nonexistent")
 274                 cud = ['copyup','copydown'][tbp]
 275                 src = p.file[not tbp] + p.dir
 276                 dst = p.path[tbp] + p.dir
 277                 testbed.command(cud, (src, dst))
 278                 p.file[tbp] = p.path[tbp]
 279
 280         return p.file[tbp] + p.dir
 281
 282  def invalidate(p, tbp=False):
 283         p.file[tbp] = None
 284         p._debug('invalidated %s' % 'HT'[tbp])
 285
 286  def _debug(p, m):
 287         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 288
 289  def _constructed(p):
 290         p._debug('constructed: '+str(p))
 291         p._check()
 292
 293  def _check(p):
 294         for tbp in [False,True]:
 295          for pf in [p.path, p.file]:
 296                 if pf[tbp] is None: continue
 297                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 298                 if p.dir and pf[tbp].endswith('/'):
 299                         pf[tbp] = pf[tbp].rstrip('/')
 300                         if not pf[tbp]: pf[tbp] = '/'
 301                 if not p.dir and pf[tbp].endswith('/'):
 302                         bomb("directory `%s' specified for "
 303                              "non-directory %s" % (pf[tbp], p.what))
 304
 305  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 306         AutoFile.__init__(p,what)
 307         sh_on = ''; sh_sibl = ''
 308         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 309         if sibling: sh_sibl=' (sibling)'
 310         parent._debug('using as base: %s: %s%s%s' %
 311                         (str(parent), leaf, sh_on, sh_sibl))
 312         if not sibling and not parent.dir:
 313                 parent._wrong('asked for non-sibling relative path of non-dir')
 314         if sibling: trim = os.path.dirname
 315         else: trim = lambda x: x
 316         for tbp in [False,True]:
 317                 if parent.path[tbp] is None: continue
 318                 trimmed = trim(parent.path[tbp])
 319                 if trimmed: trimmed += '/'
 320                 p.path[tbp] = trimmed + leaf
 321                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 322                         p.file[tbp] = p.path[tbp]
 323
 324 class InputFile(AutoFile):
 325  def _init(p, what, spec, spec_tbp=False):
 326         AutoFile.__init__(p, what)
 327         p.spec = spec
 328         p.spec_tbp = spec_tbp
 329         p.path[spec_tbp] = spec
 330         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 331  def __init__(p, what, spec, spec_tbp=False):
 332         p._init(what,spec,spec_tbp)
 333         p._constructed()
 334
 335 class InputDir(InputFile):
 336  def __init__(p, what, spec, spec_tbp=False):
 337         InputFile._init(p,what,spec,spec_tbp)
 338         p.dir = '/'
 339         p._constructed()
 340
 341 class OutputFile(AutoFile):
 342  def _init(p, what, spec, spec_tbp=False):
 343         AutoFile.__init__(p, what)
 344         p.spec = spec
 345         p.spec_tbp = spec_tbp
 346         p.path[spec_tbp] = spec
 347  def __init__(p, what, spec, spec_tbp=False):
 348         p._init(what,spec,spec_tbp)
 349         p._constructed()
 350
 351 class OutputDir(OutputFile):
 352  def __init__(p, what, spec, spec_tbp=False):
 353         OutputFile._init(p,what,spec,spec_tbp)
 354         p.dir = '/'
 355         p._constructed()
 356
 357 class RelativeInputFile(AutoFile):
 358  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 359         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 360         p._constructed()
 361
 362 class RelativeOutputFile(AutoFile):
 363  def __init__(p, what, parent, leaf, sibling=False):
 364         p._relative_init(what, parent, leaf, None, False, sibling)
 365         p._constructed()
 366
 367 class TemporaryFile(AutoFile):
 368  def __init__(p, what):
 369         AutoFile.__init__(p, what)
 370         p._constructed()
 371
 372 class TemporaryDir(AutoFile):
 373  def __init__(p, what):
 374         AutoFile.__init__(p, what)
 375         p.dir = '/'
 376         p._constructed()
 377
 378 #---------- parsing and representation of the arguments
 379
 380 class Action:
 381  def __init__(a, kind, af, arghandling, what):
 382         # extra attributes get added during processing
 383         a.kind = kind
 384         a.af = af
 385         a.ah = arghandling
 386         a.what = what
 387  def __repr__(a):
 388         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 389
 390 def parse_args():
 391         global opts
 392         usage = "%prog <options> --- <virt-server>..."
 393         parser = OptionParser(usage=usage)
 394         pa = parser.add_option
 395         pe = parser.add_option
 396
 397         arghandling = {
 398                 'dsc_tests': True,
 399                 'dsc_filter': '*',
 400                 'deb_forbuilds': 'auto',
 401                 'deb_fortests': 'auto',
 402                 'tb': False,
 403                 'override_control': None
 404         }
 405         initial_arghandling = arghandling.copy()
 406         n_non_actions = 0
 407
 408         #----------
 409         # actions (ie, test sets to run, sources to build, binaries to use):
 410
 411         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 412                 parser.largs.append((value,kindpath))
 413                 n_non_actions += not(is_act)
 414
 415         def pa_action(long, metavar, kindpath, help, is_act=True):
 416                 pa('','--'+long, action='callback', callback=cb_action,
 417                         nargs=1, type='string',
 418                         callback_args=(long,kindpath,is_act), help=help)
 419
 420         pa_action('built-tree',         'TREE', '@/',
 421                 help='run tests from build tree TREE')
 422
 423         pa_action('unbuilt-tree',       'TREE', '@//',
 424                 help='run tests from build tree TREE')
 425
 426         pa_action('source',             'DSC', '@.dsc',
 427                 help='build DSC and use its tests and/or'
 428                     ' generated binary packages')
 429
 430         pa_action('binary',             'DEB', '@.deb',
 431                help='use binary package DEB according'
 432                     ' to most recent --binaries-* settings')
 433
 434         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 435                help='run tests from control file CONTROL instead,'
 436                     ' (applies to next test suite only)')
 437
 438         #----------
 439         # argument handling settings (what ways to use action
 440         #  arguments, and pathname processing):
 441
 442         def cb_setah(option, opt_str, value, parser, toset,setval):
 443                 if type(setval) == list:
 444                         if not value in setval:
 445                                 parser.error('value for %s option (%s) is not '
 446                                  'one of the permitted values (%s)' %
 447                                  (value, opt_str, setval.join(' ')))
 448                 elif setval is not None:
 449                         value = setval
 450                 for v in toset:
 451                         arghandling[v] = value
 452                 parser.largs.append(arghandling.copy())
 453
 454         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 455                 type = metavar
 456                 if type is not None: type = 'string'
 457                 pa('',long, action='callback', callback=cb_setah,
 458                    callback_args=(affected,effect), **kwargs)
 459
 460         #---- paths: host or testbed:
 461         #
 462         pa_setah('--paths-testbed', ['tb'],True,
 463                 help='subsequent path specifications refer to the testbed')
 464         pa_setah('--paths-host', ['tb'],False,
 465                 help='subsequent path specifications refer to the host')
 466
 467         #---- source processing settings:
 468
 469         pa_setah('--sources-tests', ['dsc_tests'],True,
 470                 help='run tests from builds of subsequent sources')
 471         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 472                 help='do not run tests from builds of subsequent sources')
 473
 474         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 475                 type='string', metavar='PATTERN-LIST',
 476                 help='from subsequent sources, use binaries matching'
 477                      ' PATTERN-LIST (comma-separated glob patterns)'
 478                      ' according to most recent --binaries-* settings')
 479         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 480                 help='from subsequent sources, do not use any binaries')
 481
 482         #---- binary package processing settings:
 483
 484         def pa_setahbins(long,toset,how):
 485          pa_setah(long, toset,['ignore','auto','install'],
 486                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 487                 help=how+' ignore binaries, install them as needed'
 488                         ' for dependencies, or unconditionally install'
 489                         ' them, respectively')
 490         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 491         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 492         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 493
 494         #----------
 495         # general options:
 496
 497         def cb_vserv(op,optstr,value,parser):
 498                 parser.values.vserver = list(parser.rargs)
 499                 del parser.rargs[:]
 500
 501         def cb_path(op,optstr,value,parser, constructor,long,dir):
 502                 name = long.replace('-','_')
 503                 af = constructor(arghandling['tb'], value, long, dir)
 504                 setattr(parser.values, name, af)
 505
 506         def pa_path(long, constructor, help, dir=False):
 507                 pa('','--'+long, action='callback', callback=cb_path,
 508                         callback_args=(constructor,long,dir),
 509                         nargs=1, type='string',
 510                         help=help, metavar='PATH')
 511
 512         pa_path('output-dir', OutputDir, dir=True,
 513                 help='write stderr/out files in PATH')
 514
 515         pa('','--tmp-dir',              type='string', dest='tmpdir',
 516                 help='write temporary files to TMPDIR, emptying it'
 517                      ' beforehand and leaving it behind at the end')
 518         pa('','--log-file',             type='string', dest='logfile',
 519                 help='write the log LOGFILE, emptying it beforehand,'
 520                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 521
 522         pa('','--user',                 type='string', dest='user',
 523                 help='run tests as USER (needs root on testbed)')
 524         pa('','--gain-root',            type='string', dest='gainroot',
 525                 help='prefix debian/rules binary with GAINROOT')
 526         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 527         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 528         pa('','--gnupg-home',           type='string', dest='gnupghome',
 529                 default='~/.autopkgtest/gpg',
 530                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 531                         " signing private apt archive);"
 532                         " `fresh' means generate new key each time.")
 533
 534         #----------
 535         # actual meat:
 536
 537         class SpecialOption(optparse.Option): pass
 538         vs_op = SpecialOption('','--VSERVER-DUMMY')
 539         vs_op.action = 'callback'
 540         vs_op.type = None
 541         vs_op.default = None
 542         vs_op.nargs = 0
 543         vs_op.callback = cb_vserv
 544         vs_op.callback_args = ( )
 545         vs_op.callback_kwargs = { }
 546         vs_op.help = 'introduces virtualisation server and args'
 547         vs_op._short_opts = []
 548         vs_op._long_opts = ['---']
 549
 550         pa(vs_op)
 551
 552         (opts,args) = parser.parse_args()
 553         if not hasattr(opts,'vserver'):
 554                 parser.error('you must specifiy --- <virt-server>...')
 555         if n_non_actions >= len(parser.largs):
 556                 parser.error('nothing to do specified')
 557
 558         arghandling = initial_arghandling
 559         opts.actions = []
 560         ix = 0
 561         for act in args:
 562                 if type(act) == dict:
 563                         arghandling = act
 564                         continue
 565                 elif type(act) == tuple:
 566                         pass
 567                 elif type(act) == str:
 568                         act = (act,act)
 569                 else:
 570                         raise ("unknown action in list `%s' having"
 571                               " type `%s'" % (act, type(act)))
 572                 (pathstr, kindpath) = act
 573
 574                 constructor = InputFile
 575                 if type(kindpath) is tuple:             kind = kindpath[0]
 576                 elif kindpath.endswith('.deb'):         kind = 'deb'
 577                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 578                 elif kindpath.endswith('//'):
 579                         kind = 'ubtree'
 580                         constructor = InputDir
 581                 elif kindpath.endswith('/'):
 582                         kind = 'tree'
 583                         constructor = InputDir
 584                 else: parser.error("do not know how to handle filename \`%s';"
 585                         " specify --source --binary or --build-tree")
 586
 587                 what = '%s%s' % (kind,ix); ix += 1
 588
 589                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 590                 else: fwhatx = '-'+kind
 591
 592                 af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 593                 opts.actions.append(Action(kind, af, arghandling, what))
 594
 595 def setup_trace():
 596         global trace_stream, tmpdir
 597
 598         if opts.tmpdir is not None:
 599                 rmtree('tmpdir(specified)',opts.tmpdir)
 600                 mkdir_okexist(opts.tmpdir, 0700)
 601                 tmpdir = opts.tmpdir
 602         else:
 603                 assert(tmpdir is None)
 604                 tmpdir = tempfile.mkdtemp()
 605
 606         if opts.logfile is None:
 607                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 608                         opts.logfile = opts.output_dir.spec + '/log'
 609                 elif opts.tmpdir is not None:
 610                         opts.logfile = opts.tmpdir + '/log'
 611         if opts.logfile is not None:
 612                 trace_stream = open(opts.logfile, 'w', 0)
 613
 614         debug('options: '+`opts`, 1)
 615
 616 def finalise_options():
 617         global opts, tb
 618
 619         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 620                 opts.user = ''
 621
 622         if opts.user is None:
 623                 su = 'suggested-normal-user='
 624                 ul = [
 625                         e[len(su):]
 626                         for e in testbed.caps
 627                         if e.startswith(su)
 628                         ]
 629                 if ul:
 630                         opts.user = ul[0]
 631                 else:
 632                         opts.user = ''
 633
 634         if opts.user:
 635                 if 'root-on-testbed' not in testbed.caps:
 636                         pstderr("warning: virtualisation"
 637                                 " system does not offer root on testbed,"
 638                                 " but --user option specified: failure likely")
 639                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 640         else:
 641                 opts.user_wrap = lambda x: x
 642
 643         if opts.gainroot is None:
 644                 opts.gainroot = ''
 645                 if (opts.user or
 646                     'root-on-testbed' not in testbed.caps):
 647                         opts.gainroot = 'fakeroot'
 648
 649         if opts.gnupghome.startswith('~/'):
 650                 try: home = os.environ['HOME']
 651                 except KeyError:
 652                         parser.error("HOME environment variable"
 653                                 " not set, needed for --gnupghome=`%s"
 654                                 % opts.gnupghome)
 655                 opts.gnupghome = home + opts.gnupghome[1:]
 656         elif opts.gnupghome == 'fresh':
 657                 opts.gnupghome = None
 658
 659 #---------- testbed management - the Testbed class
 660
 661 class Testbed:
 662  def __init__(tb):
 663         tb.sp = None
 664         tb.lastsend = None
 665         tb.scratch = None
 666         tb.modified = False
 667         tb.blamed = []
 668         tb._ephemeral = []
 669         tb._debug('init')
 670         tb._need_reset_apt = False
 671  def _debug(tb, m, minlevel=0):
 672         debug('** '+m, minlevel)
 673  def start(tb):
 674         tb._debug('start')
 675         p = subprocess.PIPE
 676         debug_subprocess('vserver', opts.vserver)
 677         tb._errplumb = Errplumb(True)
 678         tb.sp = subprocess.Popen(opts.vserver,
 679                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 680         tb.expect('ok')
 681         tb.caps = tb.commandr('capabilities')
 682  def stop(tb):
 683         tb._debug('stop')
 684         tb.close()
 685         if tb.sp is None: return
 686         ec = tb.sp.returncode
 687         if ec is None:
 688                 tb.sp.stdout.close()
 689                 tb.send('quit')
 690                 tb.sp.stdin.close()
 691                 ec = tb.sp.wait()
 692         if ec:
 693                 tb.bomb('testbed gave exit status %d after quit' % ec)
 694         tb._errplumb.wait()
 695  def open(tb):
 696         tb._debug('open, scratch=%s' % tb.scratch)
 697         if tb.scratch is not None: return
 698         pl = tb.commandr('open')
 699         tb.scratch = InputDir('tb-scratch', pl[0], True)
 700         tb.deps_processed = []
 701  def mungeing_apt(tb):
 702         if not 'revert' in tb.caps:
 703                 tb._need_reset_apt = True
 704  def reset_apt(tb):
 705         if not tb._need_reset_apt: return
 706         what = 'aptget-update-reset'
 707         cmdl = ['apt-get','-qy','update']
 708         rc = tb.execute(what, cmdl)
 709         if rc:
 710                 pstderr("\n" "warning: failed to restore"
 711                         " testbed apt cache, exit code %d" % rc)
 712         tb._need_reset_apt = False
 713  def close(tb):
 714         tb._debug('close, scratch=%s' % tb.scratch)
 715         if tb.scratch is None: return
 716         tb.scratch = None
 717         if tb.sp is None: return
 718         tb.command('close')
 719  def prepare(tb, deps_new):
 720         tb._debug('prepare, modified=%s, deps_processed=%s, deps_new=%s' %
 721                 (tb.modified, tb.deps_processed, deps_new), 1)
 722         if 'revert' in tb.caps and (tb.modified or
 723             [d for d in tb.deps_processed if d not in deps_new]):
 724                 tb._debug('reset **')
 725                 tb.command('revert')
 726                 tb.blamed = []
 727                 for af in tb._ephemeral: af.invalidate(True)
 728         binaries.publish()
 729         tb.modified = False
 730         tb._install_deps(deps_new)
 731  def register_ephemeral(tb, af):
 732         if not getattr(af,'spec_tbp',False): tb._ephemeral.append(af)
 733  def _install_deps(tb, deps_new):
 734         tb._debug(' installing dependencies '+`deps_new`, 1)
 735         tb.deps_processed = deps_new
 736         if not deps_new: return
 737         dstr = ', '.join(deps_new)
 738         script = binaries.apt_pkg_gdebi_script(
 739                 dstr, [[
 740                 'from GDebi.DebPackage import DebPackage',
 741                 'd = DebPackage(cache)',
 742                 'res = d.satisfyDependsStr(arg)',
 743                 ]])
 744         cmdl = ['python','-c',script]
 745         what = 'install-deps'
 746         rc = testbed.execute(what+'-debinstall', cmdl, script=script)
 747         if rc: badpkg('dependency install failed, exit code %d' % rc)
 748  def needs_reset(tb):
 749         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 750         tb.modified = True
 751  def blame(tb, m):
 752         tb._debug('blame += %s' % m, 1)
 753         tb.blamed.append(m)
 754  def bomb(tb, m):
 755         tb._debug('bomb %s' % m)
 756         if tb.sp is not None:
 757                 tb.sp.stdout.close()
 758                 tb.sp.stdin.close()
 759                 ec = tb.sp.wait()
 760                 if ec: pstderr('adt-run: testbed failing,'
 761                                 ' exit status %d' % ec)
 762         tb.sp = None
 763         raise Quit(16, 'testbed failed: %s' % m)
 764  def send(tb, string):
 765         tb.sp.stdin
 766         try:
 767                 debug('>> '+string, 2)
 768                 print >>tb.sp.stdin, string
 769                 tb.sp.stdin.flush()
 770                 tb.lastsend = string
 771         except:
 772                 (type, value, dummy) = sys.exc_info()
 773                 tb.bomb('cannot send to testbed: %s' % traceback.
 774                         format_exception_only(type, value))
 775  def expect(tb, keyword, nresults=None):
 776         l = tb.sp.stdout.readline()
 777         if not l: tb.bomb('unexpected eof from the testbed')
 778         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 779         l = l.rstrip('\n')
 780         debug('<< '+l, 2)
 781         ll = l.split()
 782         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 783         if ll[0] != keyword:
 784                 if tb.lastsend is None:
 785                         tb.bomb("got banner `%s', expected `%s...'" %
 786                                 (l, keyword))
 787                 else:
 788                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 789                                 (tb.lastsend, l, keyword))
 790         ll = ll[1:]
 791         if nresults is not None and len(ll) != nresults:
 792                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 793                         " expected %d result parameters" %
 794                         (string, l, len(ll), nresults))
 795         return ll
 796  def commandr(tb, cmd, args=(), nresults=None):
 797         # pass args=[None,...] or =(None,...) to avoid more url quoting
 798         if type(cmd) is str: cmd = [cmd]
 799         if len(args) and args[0] is None: args = args[1:]
 800         else: args = map(urllib.quote, args)
 801         al = cmd + args
 802         tb.send(string.join(al))
 803         ll = tb.expect('ok', nresults)
 804         rl = map(urllib.unquote, ll)
 805         return rl
 806  def command(tb, cmd, args=()):
 807         tb.commandr(cmd, args, 0)
 808  def commandr1(tb, cmd, args=()):
 809         rl = tb.commandr(cmd, args, 1)
 810         return rl[0]
 811  def execute(tb, what, cmdl,
 812                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 813                 script=False):
 814         # Options for script:
 815         #   False - do not call debug_subprocess, no synch. reporting required
 816         #   None or string - call debug_subprocess with that value,
 817         #                       plumb stderr through synchronously if possible
 818         # Options for se:
 819         #   None - usual Errplumb (output is of kind 2c)
 820         #   string - path on testbed (output is of kind 2d)
 821
 822         if script is not False: debug_subprocess(what, cmdl, script=script)
 823         if cwd is None: cwd = tb.scratch.write(True)
 824
 825         xdump = None
 826         if se is None:
 827                 ep = Errplumb()
 828                 se_catch = TemporaryFile(what+'-xerr')
 829                 se_use = se_catch.write(True)
 830                 if not opts.quiet: xdump = 'debug=2-2'
 831                 elif trace_stream is not None:
 832                         xdump = 'debug=2-%d' % trace_stream.fileno()
 833         else:
 834                 ep = None
 835                 se_catch = None
 836                 se_use = se
 837
 838         cmdl = [None,
 839                 ','.join(map(urllib.quote, cmdl)),
 840                 si, so, se_use, cwd]
 841         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 842         rc = tb.commandr1('execute', cmdl)
 843         try: rc = int(rc)
 844         except ValueError: bomb("execute for %s gave invalid response `%s'"
 845                                         % (what,rc))
 846
 847         if se_catch is not None:
 848                 debug_file(se_catch.read())
 849         if ep is not None:
 850                 ep.wait()
 851
 852         return rc
 853
 854 #---------- representation of test control files: Field*, Test, etc.
 855
 856 class FieldBase:
 857  def __init__(f, fname, stz, base, tnames, vl):
 858         assert(vl)
 859         f.stz = stz
 860         f.base = base
 861         f.tnames = tnames
 862         f.vl = vl
 863  def words(f):
 864         def distribute(vle):
 865                 (lno, v) = vle
 866                 r = v.split()
 867                 r = map((lambda w: (lno, w)), r)
 868                 return r
 869         return flatten(map(distribute, f.vl))
 870  def atmostone(f):
 871         if len(vl) == 1:
 872                 (f.lno, f.v) = vl[0]
 873         else:
 874                 raise Unsupported(f.vl[1][0],
 875                         'only one %s field allowed' % fn)
 876         return f.v
 877
 878 class FieldIgnore(FieldBase):
 879  def parse(f): pass
 880
 881 class Restriction:
 882  def __init__(r,rname,base): pass
 883
 884 class Restriction_rw_build_tree(Restriction): pass
 885 class Restriction_breaks_testbed(Restriction):
 886  def __init__(r, rname, base):
 887         if 'revert' not in testbed.caps:
 888                 raise Unsupported(f.lno,
 889                         'Test breaks testbed but testbed cannot revert')
 890 class Restriction_needs_root(Restriction):
 891  def __init__(r, rname, base):
 892         if 'root-on-testbed' not in testbed.caps:
 893                 raise Unsupported(f.lno,
 894                         'Test needs root on testbed which is not available')
 895
 896 class Field_Restrictions(FieldBase):
 897  def parse(f):
 898         for wle in f.words():
 899                 (lno, rname) = wle
 900                 nrname = rname.replace('-','_')
 901                 try: rclass = globals()['Restriction_'+nrname]
 902                 except KeyError: raise Unsupported(lno,
 903                         'unknown restriction %s' % rname)
 904                 r = rclass(nrname, f.base)
 905                 f.base['restrictions'].append(r)
 906
 907 class Field_Tests(FieldIgnore): pass
 908 class Field_Features(FieldIgnore): pass
 909
 910 class Field_Depends(FieldBase):
 911  def parse(f):
 912         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 913         dl = map(lambda x: x.strip(),
 914                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 915         re = regexp.compile('[^-.+:~0-9a-z()<>=*]')
 916         for d in dl:
 917                 if re.search(d):
 918                         badpkg("Test Depends field contains dependency"
 919                                " `%s' with invalid characters" % d)
 920         f.base['depends'] = dl
 921
 922 class Field_Tests_directory(FieldBase):
 923  def parse(f):
 924         td = atmostone(f)
 925         if td.startswith('/'): raise Unspported(f.lno,
 926                 'Tests-Directory may not be absolute')
 927         f.base['testsdir'] = td
 928
 929 def run_tests(stanzas, tree):
 930         global errorcode, testbed
 931         for stanza in stanzas:
 932                 tests = stanza[' tests']
 933                 if not tests:
 934                         report('*', 'SKIP no tests in this package')
 935                         errorcode |= 8
 936                 for t in tests:
 937                         t.prepare()
 938                         t.run(tree)
 939                         if 'breaks-testbed' in t.restrictions:
 940                                 testbed.needs_reset()
 941                 testbed.needs_reset()
 942
 943 class Test:
 944  def __init__(t, tname, base, act):
 945         if '/' in tname: raise Unsupported(base[' lno'],
 946                 'test name may not contain / character')
 947         for k in base: setattr(t,k,base[k])
 948         t.tname = tname
 949         t.act = act
 950         t.what = act.what+'t-'+tname
 951         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
 952         else: t.path = tname
 953         t._debug('constructed; path=%s' % t.path)
 954         t._debug(' .depends=%s' % t.depends)
 955  def _debug(t, m):
 956         debug('& %s: %s' % (t.what, m))
 957  def report(t, m):
 958         report(t.what, m)
 959  def reportfail(t, m):
 960         global errorcode
 961         errorcode |= 4
 962         report(t.what, 'FAIL ' + m)
 963  def prepare(t):
 964         t._debug('preparing')
 965         dn = []
 966         for d in t.depends:
 967                 t._debug(' processing dependency '+d)
 968                 if not '@' in d:
 969                         t._debug('  literal dependency '+d)
 970                         dn.append(d)
 971                 else:
 972                         for (pkg,bin) in t.act.binaries:
 973                                 d = d.replace('@',pkg)
 974                                 t._debug('  synthesised dependency '+d)
 975                                 dn.append(d)
 976         testbed.prepare(dn)
 977  def run(t, tree):
 978         t._debug('[----------------------------------------')
 979         def stdouterr(oe):
 980                 idstr = t.what + '-' + oe
 981                 if opts.output_dir is not None and opts.output_dir.tb:
 982                         use_dir = opts.output_dir
 983                 else:
 984                         use_dir = testbed.scratch
 985                 return RelativeOutputFile(idstr, use_dir, idstr)
 986
 987         t.act.work.write(True)
 988
 989         af = RelativeInputFile(t.what, tree, t.path)
 990         so = stdouterr('stdout')
 991         se = stdouterr('stderr')
 992         tf = af.read(True)
 993         if 'needs-root' not in t.restrictions:
 994                 tf = opts.user_wrap(tf)
 995         rc = testbed.execute('test-'+t.what, [tf],
 996                 so=so.write(True), se=se.write(True), cwd=tree.read(True))
 997
 998         so_read = so.read()
 999         se_read = se.read()
1000
1001         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1002         stab = os.stat(se_read)
1003         if stab.st_size != 0:
1004                 l = open(se_read).readline()
1005                 l = l.rstrip('\n \t\r')
1006                 if len(l) > 40: l = l[:40] + '...'
1007                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1008                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1009                 debug_file(se_read)
1010         elif rc != 0:
1011                 t.reportfail('non-zero exit status %d' % rc)
1012         else:
1013                 t.report('PASS')
1014
1015         stab = os.stat(so_read)
1016         if stab.st_size != 0:
1017                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1018                 debug_file(so_read)
1019
1020         t._debug('----------------------------------------]')
1021
1022 def read_control(act, tree, control_override):
1023         stanzas = [ ]
1024
1025         if control_override is not None:
1026                 control_af = control_override
1027                 testbed.blame('arg:'+control_override.spec)
1028         else:
1029                 control_af = RelativeInputFile(act.what+'-testcontrol',
1030                         tree, 'debian/tests/control')
1031         try:
1032                 control = open(control_af.read(), 'r')
1033         except OSError, oe:
1034                 if oe[0] != errno.ENOENT: raise
1035                 return []
1036
1037         lno = 0
1038         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1039         stz = None      # stz[field_name][index] = (lno, value)
1040                         # special field names:
1041                         # stz[' lno'] = number
1042                         # stz[' tests'] = list of Test objects
1043         def end_stanza(stz):
1044                 if stz is None: return
1045                 stz[' errs'] = 0
1046                 stanzas.append(stz)
1047                 stz = None
1048                 hcurrent = None
1049
1050         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1051         while 1:
1052                 l = control.readline()
1053                 if not l: break
1054                 lno += 1
1055                 if not l.endswith('\n'): badctrl('unterminated line')
1056                 if regexp.compile('\s*\#').match(l): continue
1057                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1058                 initmat = initre.match(l)
1059                 if initmat:
1060                         (fname, l) = initmat.groups()
1061                         fname = string.capwords(fname)
1062                         if stz is None:
1063                                 stz = { ' lno': lno, ' tests': [] }
1064                         if not stz.has_key(fname): stz[fname] = [ ]
1065                         hcurrent = stz[fname]
1066                 elif regexp.compile('\s').match(l):
1067                         if not hcurrent: badctrl('unexpected continuation')
1068                 else:
1069                         badctrl('syntax error')
1070                 hcurrent.append((lno, l))
1071         end_stanza(stz)
1072
1073         def testbadctrl(stz, lno, m):
1074                 report_badctrl(lno, m)
1075                 stz[' errs'] += 1
1076
1077         for stz in stanzas:
1078                 try:
1079                         try: tnames = stz['Tests']
1080                         except KeyError:
1081                                 tnames = ['*']
1082                                 raise Unsupported(stz[' lno'],
1083                                         'no Tests field')
1084                         tnames = map((lambda lt: lt[1]), tnames)
1085                         tnames = string.join(tnames).split()
1086                         base = {
1087                                 'restrictions': [],
1088                                 'testsdir': 'debian/tests',
1089                                 'depends' : '@'
1090                         }
1091                         for fname in stz.keys():
1092                                 if fname.startswith(' '): continue
1093                                 vl = stz[fname]
1094                                 try: fclass = globals()['Field_'+
1095                                         fname.replace('-','_')]
1096                                 except KeyError: raise Unsupported(vl[0][0],
1097                                         'unknown metadata field %s' % fname)
1098                                 f = fclass(stz, fname, base, tnames, vl)
1099                                 f.parse()
1100                         for tname in tnames:
1101                                 t = Test(tname, base, act)
1102                                 stz[' tests'].append(t)
1103                 except Unsupported, u:
1104                         for tname in tnames: u.report(tname)
1105                         continue
1106
1107         return stanzas
1108
1109 def print_exception(ei, msgprefix=''):
1110         if msgprefix: pstderr(msgprefix)
1111         (et, q, tb) = ei
1112         if et is Quit:
1113                 pstderr('adt-run: ' + q.m)
1114                 return q.ec
1115         else:
1116                 pstderr("adt-run: unexpected, exceptional, error:")
1117                 traceback.print_exc(None, sys.stderr)
1118                 if trace_stream is not None:
1119                         traceback.print_exc(None, trace_stream)
1120                 return 20
1121
1122 def cleanup():
1123         global trace_stream
1124         try:
1125                 rm_ec = 0
1126                 if opts.tmpdir is None and tmpdir is not None:
1127                         rmtree('tmpdir', tmpdir)
1128                 if testbed is not None:
1129                         testbed.reset_apt()
1130                         testbed.stop()
1131                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1132                 if trace_stream is not None:
1133                         trace_stream.close()
1134                         trace_stream = None
1135         except:
1136                 print_exception(sys.exc_info(),
1137                         '\nadt-run: error cleaning up:\n')
1138                 os._exit(20)
1139
1140 #---------- registration, installation etc. of .deb's: Binaries
1141
1142 def determine_package(act):
1143         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1144         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1145         if rc: badpkg('failed to parse binary package, code %d' % rc)
1146         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1147         act.pkg = None
1148         for l in output.split('\n'):
1149                 m = re.match(l)
1150                 if not m: continue
1151                 if act.pkg: badpkg('two Package: lines in control file')
1152                 act.pkg = m.groups()[0]
1153         if not act.pkg: badpkg('no good Package: line in control file')
1154
1155 class Binaries:
1156  def __init__(b):
1157         b.dir = TemporaryDir('binaries')
1158         b.dir.write()
1159         ok = False
1160
1161         if opts.gnupghome is None:
1162                 opts.gnupghome = tmpdir+'/gnupg'
1163
1164         b._debug('initialising')
1165         try:
1166                 for x in ['pubring','secring']:
1167                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1168                 ok = True
1169         except OSError, oe:
1170                 if oe.errno != errno.ENOENT: raise
1171
1172         if ok: b._debug('no key generation needed')
1173         else: b.genkey()
1174
1175  def _debug(b, s):
1176         debug('* '+s)
1177
1178  def genkey(b):
1179         b._debug('preparing for key generation')
1180
1181         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1182         mkdir_okexist(opts.gnupghome, 0700)
1183
1184         script = '''
1185   exec >&2
1186   cd "$1"
1187   cat <<"END" >key-gen-params
1188 Key-Type: DSA
1189 Key-Length: 1024
1190 Key-Usage: sign
1191 Name-Real: autopkgtest per-run key
1192 Name-Comment: do not trust this key
1193 Name-Email: autopkgtest@example.com
1194 END
1195   set -x
1196   gpg --homedir="$1" --batch --gen-key key-gen-params
1197                 '''
1198         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1199         rc = subprocess_cooked(cmdl, dbg=(genkey,script))[0]
1200         if rc:
1201                 try:
1202                         f = open(opts.gnupghome+'/key-gen-log')
1203                         tp = file.read()
1204                 except OSError, e: tp = e
1205                 pstderr(tp)
1206                 bomb('key generation failed, code %d' % rc)
1207
1208  def apt_configs(b):
1209         return {
1210                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1211         }
1212
1213  def apt_pkg_gdebi_script(b, arg, middle):
1214         script = [
1215                 'import apt_pkg',
1216                 'import urllib',
1217                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1218                 ]
1219         for (k,v) in b.apt_configs().iteritems():
1220                 v = urllib.quote(v)
1221                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1222                                 % (k, v))
1223         script += [
1224                 'from GDebi.Cache import Cache',
1225                 'cache = Cache()',
1226                 ]
1227         for m in middle:
1228                 script += m + [
1229                 'print res',
1230                 'print d.missingDeps',
1231                 'print d.requiredChanges',
1232                 'assert(res)',
1233                 'cache.commit()',
1234                 ''
1235                 ]
1236         return '\n'.join(script)
1237  def apt_get(b):
1238         ag = ['apt-get','-qy']
1239         for kv in b.apt_configs().iteritems():
1240                 ag += ['-o', '%s=%s' % kv]
1241         return ag
1242
1243  def reset(b):
1244         b._debug('reset')
1245         rmtree('binaries', b.dir.read())
1246         b.dir.invalidate()
1247         b.dir.write()
1248         b.install = []
1249         b.blamed = []
1250         b.registered = set()
1251
1252  def register(b, act, pkg, af, forwhat, blamed):
1253         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1254                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1255
1256         if act.ah['deb_'+forwhat] == 'ignore': return
1257
1258         b.blamed += testbed.blamed
1259
1260         leafname = pkg+'.deb'
1261         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1262
1263         try: os.remove(dest.write())
1264         except OSError, oe:
1265                 if oe.errno != errno.ENOENT: raise e
1266
1267         try: os.link(af.read(), dest.write())
1268         except OSError, oe:
1269                 if oe.errno != errno.EXDEV: raise e
1270                 shutil.copy(af.read(), dest)
1271
1272         if act.ah['deb_'+forwhat] == 'install':
1273                 b.install.append(pkg)
1274
1275         b.registered.add(pkg)
1276
1277  def publish(b):
1278         b._debug('publish')
1279
1280         script = '''
1281   exec >&2
1282   cd "$1"
1283   apt-ftparchive packages . >Packages
1284   gzip <Packages >Packages.gz
1285   apt-ftparchive release . >Release
1286   rm -f Release.gpg
1287   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1288   gpg --homedir="$2" --batch --export >archive-key.pgp
1289                 '''
1290         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1291         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1292         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1293
1294         b.dir.invalidate(True)
1295         apt_source = b.dir.read(True)
1296
1297         so = TemporaryFile('vlds')
1298         script = '''
1299   exec 3>&1 >&2
1300   apt-key add archive-key.pgp
1301   echo "deb file://'''+apt_source+''' /" >sources.list
1302   cat /etc/apt/sources.list >>sources.list
1303   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1304     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1305   fi
1306   '''+ ' '.join(b.apt_get()) +''' update >&2
1307   cat /var/lib/dpkg/status >&3
1308                 '''
1309         testbed.mungeing_apt()
1310         rc = testbed.execute('apt-key', ['sh','-ec',script],
1311                                 so=so.write(True), cwd=b.dir.write(True),
1312                                 script=script)
1313         if rc: bomb('apt setup failed with exit code %d' % rc)
1314
1315         testbed.blamed += b.blamed
1316
1317         b._debug('publish reinstall checking...')
1318         pkgs_reinstall = set()
1319         pkg = None
1320         for l in open(so.read()):
1321                 if l.startswith('Package: '):
1322                         pkg = l[9:].rstrip()
1323                 elif l.startswith('Status: install '):
1324                         if pkg in b.registered:
1325                                 pkgs_reinstall.add(pkg)
1326                                 b._debug(' publish reinstall needs '+pkg)
1327
1328         if pkgs_reinstall:
1329                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1330                 what = 'apt-get-reinstall'
1331                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1332                         [pkg for pkg in pkgs_reinstall])
1333                 rc = testbed.execute(what, cmdl, script=None)
1334                 if rc: badpkg("installation of basic binarries failed,"
1335                                 " exit code %d" % rc)
1336
1337         b._debug('publish install...')
1338         for pkg in b.install:
1339                 what = 'apt-get-install-%s' % pkg
1340                 testbed.blame(pkg)
1341                 cmdl = b.apt_get() + ['install',pkg]
1342                 rc = testbed.execute(what, cmdl, script=None)
1343                 if rc: badpkg("installation of %s failed, exit code %d"
1344                                 % (pkg, rc))
1345
1346         b._debug('publish done')
1347
1348 #---------- processing of sources (building)
1349
1350 def source_rules_command(act,script,what,which,work,cwd,
1351                                 results_lines=0,xargs=[]):
1352         script = [      "exec 3>&1 >&2",
1353                         "set -x"        ] + script
1354         script = '\n'.join(script)
1355         so = TemporaryFile('%s-%s-results' % (what,which))
1356         rc = testbed.execute('%s-%s' % (what,which),
1357                         ['sh','-ec',script]+xargs, script=script,
1358                         so=so.write(True), cwd=cwd)
1359         results = open(so.read()).read().rstrip('\n').split("\n")
1360         if rc: badpkg("%s failed with exit code %d" % (which,rc))
1361         if results_lines is not None and len(results) != results_lines:
1362                 badpkg("got %d lines of results from %s where %d expected"
1363                         % (len(results), which, results_lines))
1364         if results_lines==1: return results[0]
1365         return results
1366
1367 def build_source(act):
1368         act.blame = 'arg:'+act.af.spec
1369         testbed.blame(act.blame)
1370         testbed.needs_reset()
1371
1372         what = act.what
1373         basename = act.af.spec
1374
1375         if act.kind == 'dsc':
1376                 dsc = act.af
1377                 dsc_file = open(dsc.read())
1378                 in_files = False
1379                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1380                 for l in dsc_file:
1381                         l = l.rstrip('\n')
1382                         if l.startswith('Files:'): in_files = True; continue
1383                         elif l.startswith('#'): pass
1384                         elif not l.startswith(' '):
1385                                 in_files = False
1386                                 if l.startswith('Source:'):
1387                                         act.blame = 'dsc:'+l[7:].strip()
1388                                         testbed.blame(act.blame)
1389                         if not in_files: continue
1390
1391                         m = fre.match(l)
1392                         if not m: badpkg(".dsc contains unparseable line"
1393                                         " in Files: `%s'" % l)
1394                         leaf = m.groups(0)[0]
1395                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1396                                         sibling=True)
1397                         subfile.read(True)
1398                 dsc.read(True)
1399
1400         if act.kind == 'ubtree':
1401                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1402                         act.af, 'debian/control')
1403                 dsc = TemporaryFile(what+'-fakedsc')
1404                 dsc_w = open(dsc.write(), 'w')
1405                 for l in open(debiancontrol.read()):
1406                         l = l.rstrip('\n')
1407                         if not len(l): break
1408                         print >>dsc_w, l
1409                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1410                 dsc_w.close()
1411
1412         work = TemporaryDir(what+'-build')
1413
1414         script = [
1415                 'spec="$1"',
1416                 'origpwd=`pwd`',
1417                 'cd '+work.write(True)
1418         ]
1419
1420         if act.kind == 'ubtree':
1421                 spec = '%s/real-tree' % work.write(True)
1422                 create_command = '''
1423                         rm -rf "$spec"
1424                         mkdir "$spec"
1425                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1426                         '''
1427                 initcwd = act.af.read(True)
1428
1429         if act.kind == 'dsc':
1430                 spec = dsc.read(True)
1431                 create_command = '''
1432                         dpkg-source -x $spec
1433                         '''
1434                 initcwd = work.write(True)
1435
1436         if opts.user: script += [
1437                         'chown '+opts.user+' .',
1438                         'spec="$spec" origpwd="$origpwd" '+opts.user_wrap(create_command)
1439                         ]
1440         else: script += [
1441                         create_command
1442                 ]
1443
1444         script += [
1445                         'cd */.',
1446                         'pwd >&3',
1447                 ]
1448         result_pwd = source_rules_command(act,script,what,'extract',work,
1449                         cwd=initcwd, results_lines=1, xargs=['x',spec])
1450
1451         script = binaries.apt_pkg_gdebi_script(
1452                 dsc.read(True), [[
1453                 'from GDebi.DscSrcPackage import DscSrcPackage',
1454                 'd = DscSrcPackage(cache, arg)',
1455                 'res = d.checkDeb()',
1456                  ],[
1457                 'from GDebi.DebPackage import DebPackage',
1458                 'd = DebPackage(cache)',
1459                 'res = d.satisfyDependsStr("build-essential")',
1460                 ]])
1461
1462         cmdl = ['python','-c',script]
1463         whatp = what+'-builddeps'
1464         rc = testbed.execute(what, cmdl, script=script)
1465         if rc: badpkg('build-depends install failed, exit code %d' % rc)
1466
1467         script = [
1468                         'cd "$1"',
1469                         'dpkg-checkbuilddeps',
1470                         opts.user_wrap('debian/rules build'),
1471                 ]
1472         source_rules_command(act,script,what,'build',work,
1473                         cwd=initcwd, xargs=['x',result_pwd])
1474
1475         if os.path.dirname(result_pwd)+'/' != work.read(True):
1476                 badpkg("results dir `%s' is not in expected parent dir `%s'"
1477                         % (result_pwd, work.read(True)))
1478
1479         act.work = work
1480         act.tests_tree = InputDir(what+'-tests-tree',
1481                                 work.read(True)+os.path.basename(result_pwd),
1482                                 True)
1483         if act.ah['dsc_tests']:
1484                 act.tests_tree.read()
1485                 testbed.register_ephemeral(act.work)
1486                 testbed.register_ephemeral(act.tests_tree)
1487
1488         act.blamed = copy.copy(testbed.blamed)
1489
1490         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1491         act.binaries = []
1492         filter = act.ah['dsc_filter']
1493         debug_b('filter=%s' % filter)
1494         if filter != '_':
1495                 script = [
1496                         'cd '+work.write(True)+'/*/.',
1497                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1498                         'cd ..',
1499                         'echo *.deb >&3',
1500                         ]
1501                 result_debs = source_rules_command(act,script,what,
1502                                 'binary',work,work.write(True),results_lines=1)
1503                 if result_debs == '*': debs = []
1504                 else: debs = result_debs.split(' ')
1505                 debug_b('debs='+`debs`)
1506                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1507                 for deb in debs:
1508                         m = re.match(deb)
1509                         if not m: badpkg("badly-named binary `%s'" % deb)
1510                         pkg = m.groups()[0]
1511                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1512                         for pat in filter.split(','):
1513                                 debug_b('  pat=%s' % pat)
1514                                 if not fnmatch.fnmatchcase(pkg,pat):
1515                                         debug_b('   no match')
1516                                         continue
1517                                 deb_what = pkg+'_'+what+'.deb'
1518                                 bin = RelativeInputFile(deb_what,work,deb,True)
1519                                 debug_b('  deb_what=%s, bin=%s' %
1520                                         (deb_what, str(bin)))
1521                                 binaries.register(act,pkg,bin,
1522                                         'forbuilds',testbed.blamed)
1523                                 act.binaries.append((pkg,bin))
1524                                 break
1525                 debug_b('all done.')
1526
1527 #---------- main processing loop and main program
1528
1529 def process_actions():
1530         global binaries
1531
1532         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1533         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1534         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1535
1536         debug_a1('starting')
1537         testbed.open()
1538         binaries = Binaries()
1539
1540         for act in opts.actions:
1541                 testbed.register_ephemeral(act.af)
1542
1543         binaries.reset()
1544
1545         debug_a1('builds ...')
1546         for act in opts.actions:
1547                 debug_a2('%s %s' %
1548                         (act.kind, act.what))
1549
1550                 testbed.prepare([])
1551                 if act.kind == 'deb':
1552                         testbed.blame('arg:'+act.af.spec)
1553                         determine_package(act)
1554                         testbed.blame('deb:'+act.pkg)
1555                         binaries.register(act,act.pkg,act.af,
1556                                 'forbuilds',testbed.blamed)
1557                 if act.kind == 'dsc' or act.kind == 'ubtree':
1558                         build_source(act)
1559                 if act.kind == 'tree':
1560                         act.binaries = []
1561
1562         debug_a1('builds done.')
1563
1564         binaries.reset()
1565         control_override = None
1566
1567         debug_a1('tests ...')
1568         for act in opts.actions:
1569                 debug_a2('test %s %s' % (act.kind, act.what))
1570
1571                 testbed.needs_reset()
1572                 if act.kind == 'control':
1573                         control_override = act.af
1574                 if act.kind == 'deb':
1575                         binaries.register(act,act.pkg,act.af,'fortests',
1576                                 ['deb:'+act.pkg])
1577                 if act.kind == 'dsc' or act.kind == 'ubtree':
1578                         for (pkg,bin) in act.binaries:
1579                                 binaries.register(act,pkg,bin,'fortests',
1580                                         act.blamed)
1581                 if act.kind == 'dsc':
1582                         if act.ah['dsc_tests']:
1583                                 debug_a3('read control ...')
1584                                 stanzas = read_control(act, act.tests_tree,
1585                                                 control_override)
1586                                 testbed.blamed += act.blamed
1587                                 debug_a3('run_tests ...')
1588                                 run_tests(stanzas, act.tests_tree)
1589                         control_override = None
1590                 if act.kind == 'tree' or act.kind == 'ubtree':
1591                         testbed.blame('arg:'+act.af.spec)
1592                         stanzas = read_control(act, act.af, control_override)
1593                         debug_a3('run_tests ...')
1594                         run_tests(stanzas, act.af)
1595                         control_override = None
1596         debug_a1('tests done.')
1597
1598 def main():
1599         global testbed
1600         global tmpdir
1601         try:
1602                 parse_args()
1603         except SystemExit, se:
1604                 os._exit(20)
1605         try:
1606                 setup_trace()
1607                 testbed = Testbed()
1608                 testbed.start()
1609                 finalise_options()
1610                 process_actions()
1611         except:
1612                 ec = print_exception(sys.exc_info(), '')
1613                 cleanup()
1614                 os._exit(ec)
1615         cleanup()
1616         os._exit(errorcode)
1617
1618 main()