runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 binaries = None         # Binaries (.debs we have registered)
  49
  50 #---------- output handling
  51 #
  52 #  There are at least the following kinds of output:
  53 #
  54 #   1. stderr output which consists of
  55 #   1a. our errors
  56 #   1b. stuff printed to stderr by the virtualisation server
  57 #   1c. stuff printed to stderr by our short-lived subprocesseses
  58 #           which we don't expect to fail
  59 #
  60 #   2. trace information, which consists of
  61 #   2a. our trace information from calls to debug()
  62 #   2b. progress information and stderr output from
  63 #        general scripts we run on the host
  64 #   2c. progress information and stderr output from things
  65 #        we run on the testbed including builds
  66 #   2d. stderr and stdout output from actual tests
  67 #
  68 #   xxxx
  69 #   3. actual test results (printed to our stdout)
  70 #
  71 # Cloning of 1a and 2a, where necessary, is done by us writing
  72 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  73 # is done by forking off a copy of ourselves to do plumbing,
  74 # which copy we wait for at the appropriate point.
  75
  76 trace_stream = None
  77
  78 def pstderr(m):
  79         print >>sys.stderr, m
  80         if trace_stream is not None: print >>trace_stream, m
  81
  82 def debug(m, minlevel=0):
  83         if opts.debuglevel < minlevel: return
  84         if opts.quiet and trace_stream is None: return
  85         p = 'adt-run: debug'
  86         if minlevel: p += `minlevel`
  87         p += ': '
  88         for l in m.rstrip('\n').split('\n'):
  89                 s = p + l
  90                 if not opts.quiet: print >>sys.stderr, s
  91                 if trace_stream is not None: print >>trace_stream, s
  92
  93 def debug_file(hp, minlevel=0):
  94         if opts.debuglevel < minlevel: return
  95         def do_copy(stream, what):
  96                 rc = subprocess.call(['cat',hp], stdout=stream)
  97                 if rc: bomb('cat failed copying data from %s'
  98                             ' to %s, exit code %d' % (hp, what, rc))
  99         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 100         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 101
 102 class Errplumb:
 103  def __init__(ep, critical=False):
 104         to_stderr = critical or not opts.quiet
 105         count = to_stderr + (trace_stream is not None)
 106         if count == 0:
 107                 ep.stream = open('/dev/null','w')
 108                 ep._sp = None
 109         elif count == 1:
 110                 if to_stderr: ep.stream = sys.stderr
 111                 else: ep.stream = trace_stream
 112                 ep._sp = None
 113         else:
 114                 ep._sp = subprocess.Popen(['tee','/dev/stderr'],
 115                         stdin=subprocess.PIPE, stdout=trace_stream,
 116                         close_fds=True)
 117                 ep.stream = ep._sp.stdin
 118  def wait(ep):
 119         if ep._sp is None: return
 120         ep._sp.stdin.close()
 121         rc = ep._sp.wait()
 122         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 123         ep._sp = None
 124
 125 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 126         if dbg is not None:
 127                 if isinstance(dbg,tuple): (what,script) = dbg
 128                 else: (what,script) = (dbg,None)
 129                 debug_subprocess(what, cmdl, script=script)
 130         ep = Errplumb(critical)
 131         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 132         output = running.communicate()[0]
 133         rc = running.wait()
 134         ep.wait()
 135         return (rc, output)
 136
 137 def preport(m):
 138         print m
 139         if trace_stream is not None: print >>trace_stream, m
 140
 141 def report(tname, result):
 142         preport('%-20s %s' % (tname, result))
 143
 144 #---------- errors we define
 145
 146 class Quit:
 147         def __init__(q,ec,m): q.ec = ec; q.m = m
 148
 149 def bomb(m):
 150         raise Quit(20, "unexpected error: %s" % m)
 151
 152 def badpkg(m):
 153         preport('blame: ' + ' '.join(testbed.blamed))
 154         preport('badpkg: ' + m)
 155         raise Quit(12, "erroneous package: %s" % m)
 156
 157 class Unsupported:
 158  def __init__(u, lno, m):
 159         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 160         else: u.m = m
 161  def report(u, tname):
 162         global errorcode
 163         errorcode != 2
 164         report(tname, 'SKIP %s' % u.m)
 165
 166 #---------- convenience function
 167
 168 def mkdir_okexist(pathname, mode=02755):
 169         try:
 170                 os.mkdir(pathname, mode)
 171         except OSError, oe:
 172                 if oe.errno != errno.EEXIST: raise
 173
 174 def rmtree(what, pathname):
 175         debug('/ %s rmtree %s' % (what, pathname), 2)
 176         shutil.rmtree(pathname)
 177
 178 def debug_subprocess(what, cmdl=None, script=None):
 179         o = '$ '+what+':'
 180         if cmdl is not None:
 181                 ol = []
 182                 for x in cmdl:
 183                         if x is script: x = '<SCRIPT>'
 184                         ol.append(x.    replace('\\','\\\\').
 185                                         replace(' ','\\ ')      )
 186                 o += ' '+ ' '.join(ol)
 187         debug(o)
 188         if script is not None and opts.debuglevel >= 1:
 189                 o = ''
 190                 for l in script.rstrip('\n').split('\n'):
 191                         o += '$     '+l+'\n'
 192                 debug(o,1)
 193
 194 def flatten(l):
 195         return reduce((lambda a,b: a + b), l, [])
 196
 197 #---------- fancy automatic file-copying class
 198
 199 class AutoFile:
 200         # p.what
 201         # p.path[tb]    None or path    not None => path known
 202         # p.file[tb]    None or path    not None => file exists
 203         # p.spec        string or None
 204         # p.spec_tbp    True or False, or not set if spec is None
 205         # p.dir         '' or '/'
 206
 207  def __init__(p, what):
 208         p.what = what
 209         p.path = [None,None]
 210         p.file = [None,None]
 211         p.spec = None
 212         p.dir = ''
 213
 214  def __repr__(p):
 215         return "<AF@%s>" % p.__str__()
 216  def __str__(p):
 217         def ptbp(tbp):
 218                 if p.path[tbp] is None: return '-'+p.dir
 219                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 220                 else: return p.path[tbp]+p.dir+'!'
 221         out = p.what
 222         if p.spec is not None:
 223                 if p.spec_tbp: out += '#'
 224                 else: out += '='
 225                 out += p.spec
 226         out += ':'
 227         out += ptbp(False)
 228         out += '|'
 229         out += ptbp(True)
 230         return out
 231
 232  def _wrong(p, how):
 233         xtra = ''
 234         if p.spec is not None: xtra = ' spec[%s]=%s' % (p.spec, p.spec_tb)
 235         raise ("internal error: %s (%s)" % (how, str(p)))
 236
 237  def _ensure_path(p, tbp):
 238         if p.path[tbp] is None:
 239                 if '/' in p.what:
 240                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 241                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 242                 if not tbp:
 243                         p.path[tbp] = tmpdir+'/'+p.what
 244                 else:
 245                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 246
 247  def write(p, tbp=False):
 248         p._debug('write %s...' % 'HT'[tbp])
 249         p._ensure_path(tbp)
 250
 251         if p.dir and not p.file[tbp]:
 252                 if not tbp:
 253                         p._debug('mkdir H')
 254                         mkdir_okexist(p.path[tbp])
 255                 else:
 256                         cmdl = ['sh','-ec',
 257                                 'test -d "$1" || mkdir -p "$1"',
 258                                 'x', p.path[tbp]]
 259                         tf_what = urllib.quote(p.what).replace('/','%2F')
 260                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 261                         if rc: bomb('failed to create directory %s' %
 262                                 p.path[tbp])
 263
 264         p.file[tbp] = p.path[tbp]
 265         return p.path[tbp]
 266
 267  def read(p, tbp=False):
 268         p._debug('read %s...' % 'HT'[tbp])
 269         p._ensure_path(tbp)
 270
 271         if p.file[tbp] is None:
 272                 if p.file[not tbp] is None:
 273                         p._wrong("requesting read but nonexistent")
 274                 cud = ['copyup','copydown'][tbp]
 275                 src = p.file[not tbp] + p.dir
 276                 dst = p.path[tbp] + p.dir
 277                 testbed.command(cud, (src, dst))
 278                 p.file[tbp] = p.path[tbp]
 279
 280         return p.file[tbp] + p.dir
 281
 282  def invalidate(p, tbp=False):
 283         p.file[tbp] = None
 284         p._debug('invalidated %s' % 'HT'[tbp])
 285
 286  def _debug(p, m):
 287         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 288
 289  def _constructed(p):
 290         p._debug('constructed: '+str(p))
 291         p._check()
 292
 293  def _check(p):
 294         for tbp in [False,True]:
 295          for pf in [p.path, p.file]:
 296                 if pf[tbp] is None: continue
 297                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 298                 if p.dir and pf[tbp].endswith('/'):
 299                         pf[tbp] = pf[tbp].rstrip('/')
 300                         if not pf[tbp]: pf[tbp] = '/'
 301                 if not p.dir and pf[tbp].endswith('/'):
 302                         bomb("directory `%s' specified for "
 303                              "non-directory %s" % (pf[tbp], p.what))
 304
 305  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 306         AutoFile.__init__(p,what)
 307         sh_on = ''; sh_sibl = ''
 308         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 309         if sibling: sh_sibl=' (sibling)'
 310         parent._debug('using as base: %s: %s%s%s' %
 311                         (str(parent), leaf, sh_on, sh_sibl))
 312         if not sibling and not parent.dir:
 313                 parent._wrong('asked for non-sibling relative path of non-dir')
 314         if sibling: trim = os.path.dirname
 315         else: trim = lambda x: x
 316         for tbp in [False,True]:
 317                 if parent.path[tbp] is None: continue
 318                 trimmed = trim(parent.path[tbp])
 319                 if trimmed: trimmed += '/'
 320                 p.path[tbp] = trimmed + leaf
 321                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 322                         p.file[tbp] = p.path[tbp]
 323
 324 class InputFile(AutoFile):
 325  def _init(p, what, spec, spec_tbp=False):
 326         AutoFile.__init__(p, what)
 327         p.spec = spec
 328         p.spec_tbp = spec_tbp
 329         p.path[spec_tbp] = spec
 330         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 331  def __init__(p, what, spec, spec_tbp=False):
 332         p._init(what,spec,spec_tbp)
 333         p._constructed()
 334
 335 class InputDir(InputFile):
 336  def __init__(p, what, spec, spec_tbp=False):
 337         InputFile._init(p,what,spec,spec_tbp)
 338         p.dir = '/'
 339         p._constructed()
 340
 341 class OutputFile(AutoFile):
 342  def _init(p, what, spec, spec_tbp=False):
 343         AutoFile.__init__(p, what)
 344         p.spec = spec
 345         p.spec_tbp = spec_tbp
 346         p.path[spec_tbp] = spec
 347  def __init__(p, what, spec, spec_tbp=False):
 348         p._init(what,spec,spec_tbp)
 349         p._constructed()
 350
 351 class OutputDir(OutputFile):
 352  def __init__(p, what, spec, spec_tbp=False):
 353         OutputFile._init(p,what,spec,spec_tbp)
 354         p.dir = '/'
 355         p._constructed()
 356
 357 class RelativeInputFile(AutoFile):
 358  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 359         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 360         p._constructed()
 361
 362 class RelativeOutputFile(AutoFile):
 363  def __init__(p, what, parent, leaf, sibling=False):
 364         p._relative_init(what, parent, leaf, None, False, sibling)
 365         p._constructed()
 366
 367 class TemporaryFile(AutoFile):
 368  def __init__(p, what):
 369         AutoFile.__init__(p, what)
 370         p._constructed()
 371
 372 class TemporaryDir(AutoFile):
 373  def __init__(p, what):
 374         AutoFile.__init__(p, what)
 375         p.dir = '/'
 376         p._constructed()
 377
 378 #---------- parsing and representation of the arguments
 379
 380 class Action:
 381  def __init__(a, kind, af, arghandling, what):
 382         # extra attributes get added during processing
 383         a.kind = kind
 384         a.af = af
 385         a.ah = arghandling
 386         a.what = what
 387         a.missing_tests_control = False
 388  def __repr__(a):
 389         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 390
 391 def parse_args():
 392         global opts
 393         usage = "%prog <options> --- <virt-server>..."
 394         parser = OptionParser(usage=usage)
 395         pa = parser.add_option
 396         pe = parser.add_option
 397
 398         arghandling = {
 399                 'dsc_tests': True,
 400                 'dsc_filter': '*',
 401                 'deb_forbuilds': 'auto',
 402                 'deb_fortests': 'auto',
 403                 'tb': False,
 404                 'override_control': None
 405         }
 406         initial_arghandling = arghandling.copy()
 407         n_non_actions = 0
 408
 409         #----------
 410         # actions (ie, test sets to run, sources to build, binaries to use):
 411
 412         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 413                 parser.largs.append((value,kindpath))
 414                 n_non_actions += not(is_act)
 415
 416         def pa_action(long, metavar, kindpath, help, is_act=True):
 417                 pa('','--'+long, action='callback', callback=cb_action,
 418                         nargs=1, type='string',
 419                         callback_args=(long,kindpath,is_act), help=help)
 420
 421         pa_action('built-tree',         'TREE', '@/',
 422                 help='run tests from build tree TREE')
 423
 424         pa_action('unbuilt-tree',       'TREE', '@//',
 425                 help='run tests from build tree TREE')
 426
 427         pa_action('source',             'DSC', '@.dsc',
 428                 help='build DSC and use its tests and/or'
 429                     ' generated binary packages')
 430
 431         pa_action('binary',             'DEB', '@.deb',
 432                help='use binary package DEB according'
 433                     ' to most recent --binaries-* settings')
 434
 435         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 436                help='run tests from control file CONTROL instead,'
 437                     ' (applies to next test suite only)')
 438
 439         #----------
 440         # argument handling settings (what ways to use action
 441         #  arguments, and pathname processing):
 442
 443         def cb_setah(option, opt_str, value, parser, toset,setval):
 444                 if type(setval) == list:
 445                         if not value in setval:
 446                                 parser.error('value for %s option (%s) is not '
 447                                  'one of the permitted values (%s)' %
 448                                  (value, opt_str, setval.join(' ')))
 449                 elif setval is not None:
 450                         value = setval
 451                 for v in toset:
 452                         arghandling[v] = value
 453                 parser.largs.append(arghandling.copy())
 454
 455         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 456                 type = metavar
 457                 if type is not None: type = 'string'
 458                 pa('',long, action='callback', callback=cb_setah,
 459                    callback_args=(affected,effect), **kwargs)
 460
 461         #---- paths: host or testbed:
 462         #
 463         pa_setah('--paths-testbed', ['tb'],True,
 464                 help='subsequent path specifications refer to the testbed')
 465         pa_setah('--paths-host', ['tb'],False,
 466                 help='subsequent path specifications refer to the host')
 467
 468         #---- source processing settings:
 469
 470         pa_setah('--sources-tests', ['dsc_tests'],True,
 471                 help='run tests from builds of subsequent sources')
 472         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 473                 help='do not run tests from builds of subsequent sources')
 474
 475         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 476                 type='string', metavar='PATTERN-LIST',
 477                 help='from subsequent sources, use binaries matching'
 478                      ' PATTERN-LIST (comma-separated glob patterns)'
 479                      ' according to most recent --binaries-* settings')
 480         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 481                 help='from subsequent sources, do not use any binaries')
 482
 483         #---- binary package processing settings:
 484
 485         def pa_setahbins(long,toset,how):
 486          pa_setah(long, toset,['ignore','auto','install'],
 487                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 488                 help=how+' ignore binaries, install them as needed'
 489                         ' for dependencies, or unconditionally install'
 490                         ' them, respectively')
 491         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 492         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 493         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 494
 495         #----------
 496         # general options:
 497
 498         def cb_vserv(op,optstr,value,parser):
 499                 parser.values.vserver = list(parser.rargs)
 500                 del parser.rargs[:]
 501
 502         def cb_path(op,optstr,value,parser, constructor,long,dir):
 503                 name = long.replace('-','_')
 504                 af = constructor(arghandling['tb'], value, long, dir)
 505                 setattr(parser.values, name, af)
 506
 507         def pa_path(long, constructor, help, dir=False):
 508                 pa('','--'+long, action='callback', callback=cb_path,
 509                         callback_args=(constructor,long,dir),
 510                         nargs=1, type='string',
 511                         help=help, metavar='PATH')
 512
 513         pa_path('output-dir', OutputDir, dir=True,
 514                 help='write stderr/out files in PATH')
 515
 516         pa('','--tmp-dir',              type='string', dest='tmpdir',
 517                 help='write temporary files to TMPDIR, emptying it'
 518                      ' beforehand and leaving it behind at the end')
 519         pa('','--log-file',             type='string', dest='logfile',
 520                 help='write the log LOGFILE, emptying it beforehand,'
 521                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 522
 523         pa('','--user',                 type='string', dest='user',
 524                 help='run tests as USER (needs root on testbed)')
 525         pa('','--gain-root',            type='string', dest='gainroot',
 526                 help='prefix debian/rules binary with GAINROOT')
 527         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 528         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 529         pa('','--gnupg-home',           type='string', dest='gnupghome',
 530                 default='~/.autopkgtest/gpg',
 531                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 532                         " signing private apt archive);"
 533                         " `fresh' means generate new key each time.")
 534
 535         #----------
 536         # actual meat:
 537
 538         class SpecialOption(optparse.Option): pass
 539         vs_op = SpecialOption('','--VSERVER-DUMMY')
 540         vs_op.action = 'callback'
 541         vs_op.type = None
 542         vs_op.default = None
 543         vs_op.nargs = 0
 544         vs_op.callback = cb_vserv
 545         vs_op.callback_args = ( )
 546         vs_op.callback_kwargs = { }
 547         vs_op.help = 'introduces virtualisation server and args'
 548         vs_op._short_opts = []
 549         vs_op._long_opts = ['---']
 550
 551         pa(vs_op)
 552
 553         (opts,args) = parser.parse_args()
 554         if not hasattr(opts,'vserver'):
 555                 parser.error('you must specifiy --- <virt-server>...')
 556         if n_non_actions >= len(parser.largs):
 557                 parser.error('nothing to do specified')
 558
 559         arghandling = initial_arghandling
 560         opts.actions = []
 561         ix = 0
 562         for act in args:
 563                 if type(act) == dict:
 564                         arghandling = act
 565                         continue
 566                 elif type(act) == tuple:
 567                         pass
 568                 elif type(act) == str:
 569                         act = (act,act)
 570                 else:
 571                         raise ("unknown action in list `%s' having"
 572                               " type `%s'" % (act, type(act)))
 573                 (pathstr, kindpath) = act
 574
 575                 constructor = InputFile
 576                 if type(kindpath) is tuple:             kind = kindpath[0]
 577                 elif kindpath.endswith('.deb'):         kind = 'deb'
 578                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 579                 elif kindpath.endswith('//'):
 580                         kind = 'ubtree'
 581                         constructor = InputDir
 582                 elif kindpath.endswith('/'):
 583                         kind = 'tree'
 584                         constructor = InputDir
 585                 else: parser.error("do not know how to handle filename \`%s';"
 586                         " specify --source --binary or --build-tree")
 587
 588                 what = '%s%s' % (kind,ix); ix += 1
 589
 590                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 591                 else: fwhatx = '-'+kind
 592
 593                 af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 594                 opts.actions.append(Action(kind, af, arghandling, what))
 595
 596 def setup_trace():
 597         global trace_stream, tmpdir
 598
 599         if opts.tmpdir is not None:
 600                 rmtree('tmpdir(specified)',opts.tmpdir)
 601                 mkdir_okexist(opts.tmpdir, 0700)
 602                 tmpdir = opts.tmpdir
 603         else:
 604                 assert(tmpdir is None)
 605                 tmpdir = tempfile.mkdtemp()
 606
 607         if opts.logfile is None:
 608                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 609                         opts.logfile = opts.output_dir.spec + '/log'
 610                 elif opts.tmpdir is not None:
 611                         opts.logfile = opts.tmpdir + '/log'
 612         if opts.logfile is not None:
 613                 trace_stream = open(opts.logfile, 'w', 0)
 614
 615         debug('options: '+`opts`, 1)
 616
 617 def finalise_options():
 618         global opts, tb
 619
 620         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 621                 opts.user = ''
 622
 623         if opts.user is None:
 624                 su = 'suggested-normal-user='
 625                 ul = [
 626                         e[len(su):]
 627                         for e in testbed.caps
 628                         if e.startswith(su)
 629                         ]
 630                 if ul:
 631                         opts.user = ul[0]
 632                 else:
 633                         opts.user = ''
 634
 635         if opts.user:
 636                 if 'root-on-testbed' not in testbed.caps:
 637                         pstderr("warning: virtualisation"
 638                                 " system does not offer root on testbed,"
 639                                 " but --user option specified: failure likely")
 640                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 641         else:
 642                 opts.user_wrap = lambda x: x
 643
 644         if opts.gainroot is None:
 645                 opts.gainroot = ''
 646                 if (opts.user or
 647                     'root-on-testbed' not in testbed.caps):
 648                         opts.gainroot = 'fakeroot'
 649
 650         if opts.gnupghome.startswith('~/'):
 651                 try: home = os.environ['HOME']
 652                 except KeyError:
 653                         parser.error("HOME environment variable"
 654                                 " not set, needed for --gnupghome=`%s"
 655                                 % opts.gnupghome)
 656                 opts.gnupghome = home + opts.gnupghome[1:]
 657         elif opts.gnupghome == 'fresh':
 658                 opts.gnupghome = None
 659
 660 #---------- testbed management - the Testbed class
 661
 662 class Testbed:
 663  def __init__(tb):
 664         tb.sp = None
 665         tb.lastsend = None
 666         tb.scratch = None
 667         tb.modified = False
 668         tb.blamed = []
 669         tb._ephemeral = []
 670         tb._debug('init')
 671         tb._need_reset_apt = False
 672  def _debug(tb, m, minlevel=0):
 673         debug('** '+m, minlevel)
 674  def start(tb):
 675         tb._debug('start')
 676         p = subprocess.PIPE
 677         debug_subprocess('vserver', opts.vserver)
 678         tb._errplumb = Errplumb(True)
 679         tb.sp = subprocess.Popen(opts.vserver,
 680                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 681         tb.expect('ok')
 682         tb.caps = tb.commandr('capabilities')
 683  def stop(tb):
 684         tb._debug('stop')
 685         tb.close()
 686         if tb.sp is None: return
 687         ec = tb.sp.returncode
 688         if ec is None:
 689                 tb.sp.stdout.close()
 690                 tb.send('quit')
 691                 tb.sp.stdin.close()
 692                 ec = tb.sp.wait()
 693         if ec:
 694                 tb.bomb('testbed gave exit status %d after quit' % ec)
 695         tb._errplumb.wait()
 696  def open(tb):
 697         tb._debug('open, scratch=%s' % tb.scratch)
 698         if tb.scratch is not None: return
 699         pl = tb.commandr('open')
 700         tb.scratch = InputDir('tb-scratch', pl[0], True)
 701         tb.deps_processed = []
 702  def mungeing_apt(tb):
 703         if not 'revert' in tb.caps:
 704                 tb._need_reset_apt = True
 705  def reset_apt(tb):
 706         if not tb._need_reset_apt: return
 707         what = 'aptget-update-reset'
 708         cmdl = ['apt-get','-qy','update']
 709         rc = tb.execute(what, cmdl)
 710         if rc:
 711                 pstderr("\n" "warning: failed to restore"
 712                         " testbed apt cache, exit code %d" % rc)
 713         tb._need_reset_apt = False
 714  def close(tb):
 715         tb._debug('close, scratch=%s' % tb.scratch)
 716         if tb.scratch is None: return
 717         tb.scratch = None
 718         if tb.sp is None: return
 719         tb.command('close')
 720  def prepare1(tb, deps_new):
 721         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 722                 (tb.modified, tb.deps_processed, deps_new), 1)
 723         if 'revert' in tb.caps and (tb.modified or
 724             [d for d in tb.deps_processed if d not in deps_new]):
 725                 tb._debug('reset **')
 726                 tb.command('revert')
 727                 tb.blamed = []
 728                 for af in tb._ephemeral: af.invalidate(True)
 729         tb.modified = False
 730  def prepare2(tb, deps_new):
 731         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 732         binaries.publish()
 733         tb._install_deps(deps_new)
 734  def prepare(tb, deps_new):
 735         tb.prepare1(deps_new)
 736         tb.prepare2(deps_new)
 737  def register_ephemeral(tb, af):
 738         if not getattr(af,'spec_tbp',False): tb._ephemeral.append(af)
 739  def _install_deps(tb, deps_new):
 740         tb._debug(' installing dependencies '+`deps_new`, 1)
 741         tb.deps_processed = deps_new
 742         if not deps_new: return
 743         dstr = ', '.join(deps_new)
 744         script = binaries.apt_pkg_gdebi_script(
 745                 dstr, [[
 746                 'from GDebi.DebPackage import DebPackage',
 747                 'd = DebPackage(cache)',
 748                 'res = d.satisfyDependsStr(arg)',
 749                 ]])
 750         cmdl = ['python','-c',script]
 751         what = 'install-deps'
 752         rc = testbed.execute(what+'-debinstall', cmdl, script=script)
 753         if rc: badpkg('dependency install failed, exit code %d' % rc)
 754  def needs_reset(tb):
 755         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 756         tb.modified = True
 757  def blame(tb, m):
 758         tb._debug('blame += %s' % m, 1)
 759         tb.blamed.append(m)
 760  def bomb(tb, m):
 761         tb._debug('bomb %s' % m)
 762         if tb.sp is not None:
 763                 tb.sp.stdout.close()
 764                 tb.sp.stdin.close()
 765                 ec = tb.sp.wait()
 766                 if ec: pstderr('adt-run: testbed failing,'
 767                                 ' exit status %d' % ec)
 768         tb.sp = None
 769         raise Quit(16, 'testbed failed: %s' % m)
 770  def send(tb, string):
 771         tb.sp.stdin
 772         try:
 773                 debug('>> '+string, 2)
 774                 print >>tb.sp.stdin, string
 775                 tb.sp.stdin.flush()
 776                 tb.lastsend = string
 777         except:
 778                 (type, value, dummy) = sys.exc_info()
 779                 tb.bomb('cannot send to testbed: %s' % traceback.
 780                         format_exception_only(type, value))
 781  def expect(tb, keyword, nresults=None):
 782         l = tb.sp.stdout.readline()
 783         if not l: tb.bomb('unexpected eof from the testbed')
 784         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 785         l = l.rstrip('\n')
 786         debug('<< '+l, 2)
 787         ll = l.split()
 788         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 789         if ll[0] != keyword:
 790                 if tb.lastsend is None:
 791                         tb.bomb("got banner `%s', expected `%s...'" %
 792                                 (l, keyword))
 793                 else:
 794                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 795                                 (tb.lastsend, l, keyword))
 796         ll = ll[1:]
 797         if nresults is not None and len(ll) != nresults:
 798                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 799                         " expected %d result parameters" %
 800                         (string, l, len(ll), nresults))
 801         return ll
 802  def commandr(tb, cmd, args=(), nresults=None):
 803         # pass args=[None,...] or =(None,...) to avoid more url quoting
 804         if type(cmd) is str: cmd = [cmd]
 805         if len(args) and args[0] is None: args = args[1:]
 806         else: args = map(urllib.quote, args)
 807         al = cmd + args
 808         tb.send(string.join(al))
 809         ll = tb.expect('ok', nresults)
 810         rl = map(urllib.unquote, ll)
 811         return rl
 812  def command(tb, cmd, args=()):
 813         tb.commandr(cmd, args, 0)
 814  def commandr1(tb, cmd, args=()):
 815         rl = tb.commandr(cmd, args, 1)
 816         return rl[0]
 817  def execute(tb, what, cmdl,
 818                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 819                 script=False, tmpdir=None):
 820         # Options for script:
 821         #   False - do not call debug_subprocess, no synch. reporting required
 822         #   None or string - call debug_subprocess with that value,
 823         #                       plumb stderr through synchronously if possible
 824         # Options for se:
 825         #   None - usual Errplumb (output is of kind 2c)
 826         #   string - path on testbed (output is of kind 2d)
 827
 828         if script is not False: debug_subprocess(what, cmdl, script=script)
 829         if cwd is None: cwd = tb.scratch.write(True)
 830
 831         xdump = None
 832         if se is None:
 833                 ep = Errplumb()
 834                 se_catch = TemporaryFile(what+'-xerr')
 835                 se_use = se_catch.write(True)
 836                 if not opts.quiet: xdump = 'debug=2-2'
 837                 elif trace_stream is not None:
 838                         xdump = 'debug=2-%d' % trace_stream.fileno()
 839         else:
 840                 ep = None
 841                 se_catch = None
 842                 se_use = se
 843
 844         cmdl = [None,
 845                 ','.join(map(urllib.quote, cmdl)),
 846                 si, so, se_use, cwd]
 847
 848         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 849         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 850
 851         rc = tb.commandr1('execute', cmdl)
 852         try: rc = int(rc)
 853         except ValueError: bomb("execute for %s gave invalid response `%s'"
 854                                         % (what,rc))
 855
 856         if se_catch is not None:
 857                 debug_file(se_catch.read())
 858         if ep is not None:
 859                 ep.wait()
 860
 861         return rc
 862
 863 #---------- representation of test control files: Field*, Test, etc.
 864
 865 class FieldBase:
 866  def __init__(f, fname, stz, base, tnames, vl):
 867         assert(vl)
 868         f.stz = stz
 869         f.base = base
 870         f.tnames = tnames
 871         f.vl = vl
 872  def words(f):
 873         def distribute(vle):
 874                 (lno, v) = vle
 875                 r = v.split()
 876                 r = map((lambda w: (lno, w)), r)
 877                 return r
 878         return flatten(map(distribute, f.vl))
 879  def atmostone(f):
 880         if len(vl) == 1:
 881                 (f.lno, f.v) = vl[0]
 882         else:
 883                 raise Unsupported(f.vl[1][0],
 884                         'only one %s field allowed' % fn)
 885         return f.v
 886
 887 class FieldIgnore(FieldBase):
 888  def parse(f): pass
 889
 890 class Restriction:
 891  def __init__(r,rname,base): pass
 892
 893 class Restriction_rw_build_tree(Restriction): pass
 894 class Restriction_breaks_testbed(Restriction):
 895  def __init__(r, rname, base):
 896         if 'revert' not in testbed.caps:
 897                 raise Unsupported(f.lno,
 898                         'Test breaks testbed but testbed cannot revert')
 899 class Restriction_needs_root(Restriction):
 900  def __init__(r, rname, base):
 901         if 'root-on-testbed' not in testbed.caps:
 902                 raise Unsupported(f.lno,
 903                         'Test needs root on testbed which is not available')
 904
 905 class Field_Restrictions(FieldBase):
 906  def parse(f):
 907         for wle in f.words():
 908                 (lno, rname) = wle
 909                 nrname = rname.replace('-','_')
 910                 try: rclass = globals()['Restriction_'+nrname]
 911                 except KeyError: raise Unsupported(lno,
 912                         'unknown restriction %s' % rname)
 913                 r = rclass(nrname, f.base)
 914                 f.base['restriction_names'].append(rname)
 915                 f.base['restrictions'].append(r)
 916
 917 class Field_Features(FieldIgnore):
 918  def parse(f):
 919         for wle in f.words():
 920                 (lno, fname) = wle
 921                 f.base['feature_names'].append(fname)
 922                 nfname = fname.replace('-','_')
 923                 try: fclass = globals()['Feature_'+nfname]
 924                 except KeyError: continue
 925                 ft = fclass(nfname, f.base)
 926                 f.base['features'].append(ft)
 927
 928 class Field_Tests(FieldIgnore): pass
 929
 930 class Field_Depends(FieldBase):
 931  def parse(f):
 932         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 933         dl = map(lambda x: x.strip(),
 934                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 935         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
 936         for d in dl:
 937                 if re.search(d):
 938                         badpkg("Test Depends field contains dependency"
 939                                " `%s' with invalid characters" % d)
 940         f.base['depends'] = dl
 941
 942 class Field_Tests_directory(FieldBase):
 943  def parse(f):
 944         td = atmostone(f)
 945         if td.startswith('/'): raise Unspported(f.lno,
 946                 'Tests-Directory may not be absolute')
 947         f.base['testsdir'] = td
 948
 949 def run_tests(stanzas, tree):
 950         global errorcode, testbed
 951         if stanzas == ():
 952                 report('*', 'SKIP no tests in this package')
 953         for stanza in stanzas:
 954                 tests = stanza[' tests']
 955                 if not tests:
 956                         report('*', 'SKIP package has metadata but no tests')
 957                         errorcode |= 8
 958                 for t in tests:
 959                         t.prepare()
 960                         t.run(tree)
 961                         if 'breaks-testbed' in t.restrictions:
 962                                 testbed.needs_reset()
 963                 testbed.needs_reset()
 964
 965 class Test:
 966  def __init__(t, tname, base, act):
 967         if '/' in tname: raise Unsupported(base[' lno'],
 968                 'test name may not contain / character')
 969         for k in base: setattr(t,k,base[k])
 970         t.tname = tname
 971         t.act = act
 972         t.what = act.what+'t-'+tname
 973         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
 974         else: t.path = tname
 975         t._debug('constructed; path=%s' % t.path)
 976         t._debug(' .depends=%s' % t.depends)
 977  def _debug(t, m):
 978         debug('& %s: %s' % (t.what, m))
 979  def report(t, m):
 980         report(t.what, m)
 981  def reportfail(t, m):
 982         global errorcode
 983         errorcode |= 4
 984         report(t.what, 'FAIL ' + m)
 985  def prepare(t):
 986         t._debug('preparing')
 987         dn = []
 988         for d in t.depends:
 989                 t._debug(' processing dependency '+d)
 990                 if not '@' in d:
 991                         t._debug('  literal dependency '+d)
 992                         dn.append(d)
 993                 else:
 994                         for (pkg,bin) in t.act.binaries:
 995                                 d = d.replace('@',pkg)
 996                                 t._debug('  synthesised dependency '+d)
 997                                 dn.append(d)
 998         testbed.prepare(dn)
 999  def run(t, tree):
1000         t._debug('[----------------------------------------')
1001         def stdouterr(oe):
1002                 idstr = t.what + '-' + oe
1003                 if opts.output_dir is not None and opts.output_dir.tb:
1004                         use_dir = opts.output_dir
1005                 else:
1006                         use_dir = testbed.scratch
1007                 return RelativeOutputFile(idstr, use_dir, idstr)
1008
1009         t.act.work.write(True)
1010
1011         af = RelativeInputFile(t.what, tree, t.path)
1012         so = stdouterr('stdout')
1013         se = stdouterr('stderr')
1014
1015         tf = af.read(True)
1016         tmpdir = None
1017
1018         if 'needs-root' not in t.restrictions:
1019                 tf = opts.user_wrap(tf)
1020                 tmpdir = '%s/%s-tmpdir' % (testbed.scratch.read(True), t.what)
1021                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1022                 if opts.user: script += '; chown %s "$1"' % opts.user
1023                 rc = testbed.execute('mktmpdir-'+t.what,
1024                         ['sh','-xec',script,'x',tmpdir])
1025                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1026                                 % (tmpdir, rc))
1027
1028         rc = testbed.execute('test-'+t.what, [tf],
1029                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1030                 tmpdir=tmpdir)
1031
1032         so_read = so.read()
1033         se_read = se.read()
1034
1035         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1036         stab = os.stat(se_read)
1037         if stab.st_size != 0:
1038                 l = open(se_read).readline()
1039                 l = l.rstrip('\n \t\r')
1040                 if len(l) > 40: l = l[:40] + '...'
1041                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1042                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1043                 debug_file(se_read)
1044         elif rc != 0:
1045                 t.reportfail('non-zero exit status %d' % rc)
1046         else:
1047                 t.report('PASS')
1048
1049         stab = os.stat(so_read)
1050         if stab.st_size != 0:
1051                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1052                 debug_file(so_read)
1053
1054         t._debug('----------------------------------------]')
1055
1056 def read_control(act, tree, control_override):
1057         stanzas = [ ]
1058
1059         if control_override is not None:
1060                 control_af = control_override
1061                 testbed.blame('arg:'+control_override.spec)
1062         else:
1063                 if act.missing_tests_control:
1064                         return ()
1065                 control_af = RelativeInputFile(act.what+'-testcontrol',
1066                         tree, 'debian/tests/control')
1067         try:
1068                 control = open(control_af.read(), 'r')
1069         except OSError, oe:
1070                 if oe[0] != errno.ENOENT: raise
1071                 return []
1072
1073         lno = 0
1074         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1075         stz = None      # stz[field_name][index] = (lno, value)
1076                         # special field names:
1077                         # stz[' lno'] = number
1078                         # stz[' tests'] = list of Test objects
1079         def end_stanza(stz):
1080                 if stz is None: return
1081                 stz[' errs'] = 0
1082                 stanzas.append(stz)
1083                 stz = None
1084                 hcurrent = None
1085
1086         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1087         while 1:
1088                 l = control.readline()
1089                 if not l: break
1090                 lno += 1
1091                 if not l.endswith('\n'): badctrl('unterminated line')
1092                 if regexp.compile('\s*\#').match(l): continue
1093                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1094                 initmat = initre.match(l)
1095                 if initmat:
1096                         (fname, l) = initmat.groups()
1097                         fname = string.capwords(fname)
1098                         if stz is None:
1099                                 stz = { ' lno': lno, ' tests': [] }
1100                         if not stz.has_key(fname): stz[fname] = [ ]
1101                         hcurrent = stz[fname]
1102                 elif regexp.compile('\s').match(l):
1103                         if not hcurrent: badctrl('unexpected continuation')
1104                 else:
1105                         badctrl('syntax error')
1106                 hcurrent.append((lno, l))
1107         end_stanza(stz)
1108
1109         def testbadctrl(stz, lno, m):
1110                 report_badctrl(lno, m)
1111                 stz[' errs'] += 1
1112
1113         for stz in stanzas:
1114                 try:
1115                         try: tnames = stz['Tests']
1116                         except KeyError:
1117                                 tnames = ['*']
1118                                 raise Unsupported(stz[' lno'],
1119                                         'no Tests field')
1120                         tnames = map((lambda lt: lt[1]), tnames)
1121                         tnames = string.join(tnames).split()
1122                         base = {
1123                                 'restriction_names': [],
1124                                 'restrictions': [],
1125                                 'feature_names': [],
1126                                 'features': [],
1127                                 'testsdir': 'debian/tests',
1128                                 'depends' : '@'
1129                         }
1130                         for fname in stz.keys():
1131                                 if fname.startswith(' '): continue
1132                                 vl = stz[fname]
1133                                 try: fclass = globals()['Field_'+
1134                                         fname.replace('-','_')]
1135                                 except KeyError: raise Unsupported(vl[0][0],
1136                                         'unknown metadata field %s' % fname)
1137                                 f = fclass(stz, fname, base, tnames, vl)
1138                                 f.parse()
1139                         for tname in tnames:
1140                                 t = Test(tname, base, act)
1141                                 stz[' tests'].append(t)
1142                 except Unsupported, u:
1143                         for tname in tnames: u.report(tname)
1144                         continue
1145
1146         return stanzas
1147
1148 def print_exception(ei, msgprefix=''):
1149         if msgprefix: pstderr(msgprefix)
1150         (et, q, tb) = ei
1151         if et is Quit:
1152                 pstderr('adt-run: ' + q.m)
1153                 return q.ec
1154         else:
1155                 pstderr("adt-run: unexpected, exceptional, error:")
1156                 traceback.print_exc(None, sys.stderr)
1157                 if trace_stream is not None:
1158                         traceback.print_exc(None, trace_stream)
1159                 return 20
1160
1161 def cleanup():
1162         global trace_stream
1163         try:
1164                 rm_ec = 0
1165                 if opts.tmpdir is None and tmpdir is not None:
1166                         rmtree('tmpdir', tmpdir)
1167                 if testbed is not None:
1168                         testbed.reset_apt()
1169                         testbed.stop()
1170                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1171                 if trace_stream is not None:
1172                         trace_stream.close()
1173                         trace_stream = None
1174         except:
1175                 print_exception(sys.exc_info(),
1176                         '\nadt-run: error cleaning up:\n')
1177                 os._exit(20)
1178
1179 #---------- registration, installation etc. of .deb's: Binaries
1180
1181 def determine_package(act):
1182         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1183         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1184         if rc: badpkg('failed to parse binary package, code %d' % rc)
1185         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1186         act.pkg = None
1187         for l in output.split('\n'):
1188                 m = re.match(l)
1189                 if not m: continue
1190                 if act.pkg: badpkg('two Package: lines in control file')
1191                 act.pkg = m.groups()[0]
1192         if not act.pkg: badpkg('no good Package: line in control file')
1193
1194 class Binaries:
1195  def __init__(b):
1196         b.dir = TemporaryDir('binaries')
1197         b.dir.write()
1198         ok = False
1199
1200         if opts.gnupghome is None:
1201                 opts.gnupghome = tmpdir+'/gnupg'
1202
1203         b._debug('initialising')
1204         try:
1205                 for x in ['pubring','secring']:
1206                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1207                 ok = True
1208         except OSError, oe:
1209                 if oe.errno != errno.ENOENT: raise
1210
1211         if ok: b._debug('no key generation needed')
1212         else: b.genkey()
1213
1214  def _debug(b, s):
1215         debug('* '+s)
1216
1217  def genkey(b):
1218         b._debug('preparing for key generation')
1219
1220         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1221         mkdir_okexist(opts.gnupghome, 0700)
1222
1223         script = '''
1224   exec >&2
1225   cd "$1"
1226   cat <<"END" >key-gen-params
1227 Key-Type: DSA
1228 Key-Length: 1024
1229 Key-Usage: sign
1230 Name-Real: autopkgtest per-run key
1231 Name-Comment: do not trust this key
1232 Name-Email: autopkgtest@example.com
1233 END
1234   set -x
1235   gpg --homedir="$1" --batch --gen-key key-gen-params
1236                 '''
1237         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1238         rc = subprocess_cooked(cmdl, dbg=(genkey,script))[0]
1239         if rc:
1240                 try:
1241                         f = open(opts.gnupghome+'/key-gen-log')
1242                         tp = file.read()
1243                 except OSError, e: tp = e
1244                 pstderr(tp)
1245                 bomb('key generation failed, code %d' % rc)
1246
1247  def apt_configs(b):
1248         return {
1249                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1250         }
1251
1252  def apt_pkg_gdebi_script(b, arg, middle):
1253         script = [
1254                 'import apt_pkg',
1255                 'import urllib',
1256                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1257                 ]
1258         for (k,v) in b.apt_configs().iteritems():
1259                 v = urllib.quote(v)
1260                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1261                                 % (k, v))
1262         script += [
1263                 'from GDebi.Cache import Cache',
1264                 'cache = Cache()',
1265                 ]
1266         for m in middle:
1267                 script += m + [
1268                 'print res',
1269                 'print d.missingDeps',
1270                 'print d.requiredChanges',
1271                 'assert(res)',
1272                 'cache.commit()',
1273                 ''
1274                 ]
1275         return '\n'.join(script)
1276  def apt_get(b):
1277         ag = ['apt-get','-qy']
1278         for kv in b.apt_configs().iteritems():
1279                 ag += ['-o', '%s=%s' % kv]
1280         return ag
1281
1282  def reset(b):
1283         b._debug('reset')
1284         rmtree('binaries', b.dir.read())
1285         b.dir.invalidate()
1286         b.dir.write()
1287         b.install = []
1288         b.blamed = []
1289         b.registered = set()
1290
1291  def register(b, act, pkg, af, forwhat, blamed):
1292         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1293                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1294
1295         if act.ah['deb_'+forwhat] == 'ignore': return
1296
1297         b.blamed += testbed.blamed
1298
1299         leafname = pkg+'.deb'
1300         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1301
1302         try: os.remove(dest.write())
1303         except OSError, oe:
1304                 if oe.errno != errno.ENOENT: raise e
1305
1306         try: os.link(af.read(), dest.write())
1307         except OSError, oe:
1308                 if oe.errno != errno.EXDEV: raise e
1309                 shutil.copy(af.read(), dest)
1310
1311         if act.ah['deb_'+forwhat] == 'install':
1312                 b.install.append(pkg)
1313
1314         b.registered.add(pkg)
1315
1316  def publish(b):
1317         b._debug('publish')
1318
1319         script = '''
1320   exec >&2
1321   cd "$1"
1322   apt-ftparchive packages . >Packages
1323   gzip <Packages >Packages.gz
1324   apt-ftparchive release . >Release
1325   rm -f Release.gpg
1326   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1327   gpg --homedir="$2" --batch --export >archive-key.pgp
1328                 '''
1329         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1330         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1331         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1332
1333         b.dir.invalidate(True)
1334         apt_source = b.dir.read(True)
1335
1336         so = TemporaryFile('vlds')
1337         script = '''
1338   exec 3>&1 >&2
1339   apt-key add archive-key.pgp
1340   echo "deb file://'''+apt_source+''' /" >sources.list
1341   cat /etc/apt/sources.list >>sources.list
1342   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1343     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1344   fi
1345   '''+ ' '.join(b.apt_get()) +''' update >&2
1346   cat /var/lib/dpkg/status >&3
1347                 '''
1348         testbed.mungeing_apt()
1349         rc = testbed.execute('apt-key', ['sh','-ec',script],
1350                                 so=so.write(True), cwd=b.dir.write(True),
1351                                 script=script)
1352         if rc: bomb('apt setup failed with exit code %d' % rc)
1353
1354         testbed.blamed += b.blamed
1355
1356         b._debug('publish reinstall checking...')
1357         pkgs_reinstall = set()
1358         pkg = None
1359         for l in open(so.read()):
1360                 if l.startswith('Package: '):
1361                         pkg = l[9:].rstrip()
1362                 elif l.startswith('Status: install '):
1363                         if pkg in b.registered:
1364                                 pkgs_reinstall.add(pkg)
1365                                 b._debug(' publish reinstall needs '+pkg)
1366
1367         if pkgs_reinstall:
1368                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1369                 what = 'apt-get-reinstall'
1370                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1371                         [pkg for pkg in pkgs_reinstall])
1372                 rc = testbed.execute(what, cmdl, script=None)
1373                 if rc: badpkg("installation of basic binarries failed,"
1374                                 " exit code %d" % rc)
1375
1376         b._debug('publish install...')
1377         for pkg in b.install:
1378                 what = 'apt-get-install-%s' % pkg
1379                 testbed.blame(pkg)
1380                 cmdl = b.apt_get() + ['install',pkg]
1381                 rc = testbed.execute(what, cmdl, script=None)
1382                 if rc: badpkg("installation of %s failed, exit code %d"
1383                                 % (pkg, rc))
1384
1385         b._debug('publish done')
1386
1387 #---------- processing of sources (building)
1388
1389 def source_rules_command(act,script,what,which,work,cwd,
1390                                 results_lines=0,xargs=[]):
1391         script = [      "exec 3>&1 >&2",
1392                         "set -x"        ] + script
1393         script = '\n'.join(script)
1394         so = TemporaryFile('%s-%s-results' % (what,which))
1395         rc = testbed.execute('%s-%s' % (what,which),
1396                         ['sh','-ec',script]+xargs, script=script,
1397                         so=so.write(True), cwd=cwd)
1398         results = open(so.read()).read().rstrip('\n')
1399         if len(results): results = results.split("\n")
1400         else: results = []
1401         if rc: badpkg("%s failed with exit code %d" % (which,rc))
1402         if results_lines is not None and len(results) != results_lines:
1403                 badpkg("got %d lines of results from %s where %d expected"
1404                         % (len(results), which, results_lines))
1405         if results_lines==1: return results[0]
1406         return results
1407
1408 def build_source(act, control_override):
1409         act.blame = 'arg:'+act.af.spec
1410         testbed.blame(act.blame)
1411         testbed.prepare1([])
1412         testbed.needs_reset()
1413
1414         what = act.what
1415         basename = act.af.spec
1416         debiancontrol = None
1417         act.binaries = []
1418
1419         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1420
1421         if act.kind == 'dsc':
1422                 dsc = act.af
1423                 dsc_file = open(dsc.read())
1424                 in_files = False
1425                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1426                 for l in dsc_file:
1427                         l = l.rstrip('\n')
1428                         if l.startswith('Files:'): in_files = True; continue
1429                         elif l.startswith('#'): pass
1430                         elif not l.startswith(' '):
1431                                 in_files = False
1432                                 if l.startswith('Source:'):
1433                                         act.blame = 'dsc:'+l[7:].strip()
1434                                         testbed.blame(act.blame)
1435                         if not in_files: continue
1436
1437                         m = fre.match(l)
1438                         if not m: badpkg(".dsc contains unparseable line"
1439                                         " in Files: `%s'" % l)
1440                         leaf = m.groups(0)[0]
1441                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1442                                         sibling=True)
1443                         subfile.read(True)
1444                 dsc.read(True)
1445
1446         if act.kind == 'ubtree':
1447                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1448                         act.af, 'debian/control')
1449                 dsc = TemporaryFile(what+'-fakedsc')
1450                 dsc_w = open(dsc.write(), 'w')
1451                 for l in open(debiancontrol.read()):
1452                         l = l.rstrip('\n')
1453                         if not len(l): break
1454                         print >>dsc_w, l
1455                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1456                 dsc_w.close()
1457
1458         if act.kind == 'dsc':
1459                 testbed.prepare2([])
1460                 script = binaries.apt_pkg_gdebi_script('', [[
1461                                 'from GDebi.DebPackage import DebPackage',
1462                                 'd = DebPackage(cache)',
1463                                 'res = d.satisfyDependsStr("dpkg-source")',
1464                         ]])
1465                 cmdl = ['python','-c',script]
1466                 whatp = what+'-dpkgsource'
1467                 rc = testbed.execute(what, cmdl, script=script)
1468                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1469
1470         work = TemporaryDir(what+'-build')
1471         act.work = work
1472
1473         tmpdir = work.write(True)+'/tmpdir'
1474         tmpdir_script = [
1475                         'TMPDIR="$1"',
1476                         'rm -rf -- "$TMPDIR"',
1477                         'export TMPDIR',
1478                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1479                 ]
1480
1481         if act.kind == 'ubtree':
1482                 spec = '%s/real-tree' % work.write(True)
1483                 create_command = '''
1484                         rm -rf "$spec"
1485                         mkdir "$spec"
1486                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1487                         '''
1488                 initcwd = act.af.read(True)
1489
1490         if act.kind == 'dsc':
1491                 spec = dsc.read(True)
1492                 create_command = '''
1493                         dpkg-source -x $spec
1494                         '''
1495                 initcwd = work.write(True)
1496
1497         script = [
1498                 'spec="$2"',
1499                 'origpwd=`pwd`',
1500                 'cd '+work.write(True)
1501         ]
1502
1503         if opts.user:
1504                 script += ([ 'chown '+opts.user+' .' ] +
1505                         tmpdir_script +
1506                         [ 'spec="$spec" origpwd="$origpwd" '
1507                                 +opts.user_wrap(create_command) ])
1508         else:
1509                 script += (tmpdir_script +
1510                         [ create_command ])
1511
1512         script += [
1513                         'cd */.',
1514                         'pwd >&3',
1515                         'set +e; test -f debian/tests/control; echo $? >&3'
1516                 ]
1517         (result_pwd, control_test_rc) = source_rules_command(
1518                         act,script,what,'extract',work,
1519                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1520
1521         filter = act.ah['dsc_filter']
1522
1523         if control_test_rc == '1': act.missing_tests_control = True
1524
1525         # For optional builds:
1526         #
1527         # We might need to build the package because:
1528         #   - we want its binaries (filter isn't _ and at least one of the
1529         #       deb_... isn't ignore)
1530         #   - the test control file says so
1531         #       (assuming we have any tests)
1532
1533         class NeedBuildException: pass
1534         def build_needed(m):
1535                 debug_b('build needed for %s' % m)
1536                 raise NeedBuildException()
1537
1538         try:
1539                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1540                                       act.ah['deb_fortests'] != 'ignore'):
1541                         build_needed('binaries')
1542
1543                 result_pwd_af = InputDir(what+'-treeforcontrol',
1544                         result_pwd, True)
1545                 stanzas = read_control(act, result_pwd_af, control_override)
1546                 for stanza in stanzas:
1547                         for t in stanza[' tests']:
1548                                 if 'no-build-needed' not in t.feature_names:
1549                                         build_needed('test %s' % t.name)
1550                                 for d in t.depends:
1551                                         if '@' in d:
1552                                                 build_needed('test %s '
1553                                                   'dependency %s' % (t.name,d))
1554
1555                 debug_b('build not needed')
1556                 built = False
1557
1558         except NeedBuildException:
1559
1560                 if act.kind != 'dsc':
1561                         testbed.prepare2([])
1562
1563                 script = binaries.apt_pkg_gdebi_script(
1564                         dsc.read(True), [[
1565                         'from GDebi.DscSrcPackage import DscSrcPackage',
1566                         'd = DscSrcPackage(cache, arg)',
1567                         'res = d.checkDeb()',
1568                          ],[
1569                         'from GDebi.DebPackage import DebPackage',
1570                         'd = DebPackage(cache)',
1571                         'res = d.satisfyDependsStr("build-essential")',
1572                         ]])
1573
1574                 cmdl = ['python','-c',script]
1575                 whatp = what+'-builddeps'
1576                 rc = testbed.execute(what, cmdl, script=script)
1577                 if rc: badpkg('build-depends install failed,'
1578                               ' exit code %d' % rc)
1579
1580                 script = tmpdir_script + [
1581                         'cd "$2"',
1582                         'dpkg-checkbuilddeps',
1583                         opts.user_wrap('debian/rules build'),
1584                         ]
1585                 source_rules_command(act,script,what,'build',work,
1586                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1587
1588                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1589                         badpkg("results dir `%s' is not in expected parent"
1590                                " dir `%s'" % (result_pwd, work.read(True)))
1591
1592                 built = True
1593
1594         act.tests_tree = InputDir(what+'-tests-tree',
1595                                 work.read(True)+os.path.basename(result_pwd),
1596                                 True)
1597         if act.ah['dsc_tests']:
1598                 act.tests_tree.read()
1599                 testbed.register_ephemeral(act.work)
1600                 testbed.register_ephemeral(act.tests_tree)
1601
1602         if not built: return
1603
1604         act.blamed = copy.copy(testbed.blamed)
1605
1606         debug_b('filter=%s' % filter)
1607         if filter != '_':
1608                 script = tmpdir_script + [
1609                         'cd '+work.write(True)+'/*/.',
1610                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1611                         'cd ..',
1612                         'echo *.deb >&3',
1613                         ]
1614                 result_debs = source_rules_command(act,script,what,
1615                                 'binary',work,work.write(True),
1616                                 results_lines=1, xargs=['x',tmpdir])
1617                 if result_debs == '*': debs = []
1618                 else: debs = result_debs.split(' ')
1619                 debug_b('debs='+`debs`)
1620                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1621                 for deb in debs:
1622                         m = re.match(deb)
1623                         if not m: badpkg("badly-named binary `%s'" % deb)
1624                         pkg = m.groups()[0]
1625                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1626                         for pat in filter.split(','):
1627                                 debug_b('  pat=%s' % pat)
1628                                 if not fnmatch.fnmatchcase(pkg,pat):
1629                                         debug_b('   no match')
1630                                         continue
1631                                 deb_what = pkg+'_'+what+'.deb'
1632                                 bin = RelativeInputFile(deb_what,work,deb,True)
1633                                 debug_b('  deb_what=%s, bin=%s' %
1634                                         (deb_what, str(bin)))
1635                                 binaries.register(act,pkg,bin,
1636                                         'forbuilds',testbed.blamed)
1637                                 act.binaries.append((pkg,bin))
1638                                 break
1639                 debug_b('all done.')
1640
1641 #---------- main processing loop and main program
1642
1643 def process_actions():
1644         global binaries
1645
1646         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1647         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1648         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1649
1650         debug_a1('starting')
1651         testbed.open()
1652         binaries = Binaries()
1653
1654         for act in opts.actions:
1655                 testbed.register_ephemeral(act.af)
1656
1657         binaries.reset()
1658         control_override = None
1659
1660         debug_a1('builds ...')
1661         for act in opts.actions:
1662                 debug_a2('%s %s' %
1663                         (act.kind, act.what))
1664
1665                 if act.kind == 'control':
1666                         control_override = act.af
1667                 if act.kind == 'deb':
1668                         testbed.blame('arg:'+act.af.spec)
1669                         determine_package(act)
1670                         testbed.blame('deb:'+act.pkg)
1671                         binaries.register(act,act.pkg,act.af,
1672                                 'forbuilds',testbed.blamed)
1673                 if act.kind == 'dsc' or act.kind == 'ubtree':
1674                         build_source(act, control_override)
1675                 if act.kind == 'tree':
1676                         act.binaries = []
1677                 if act.kind.endswith('tree') or act.kind == 'dsc':
1678                         control_override = None
1679
1680         debug_a1('builds done.')
1681
1682         binaries.reset()
1683         control_override = None
1684
1685         debug_a1('tests ...')
1686         for act in opts.actions:
1687                 debug_a2('test %s %s' % (act.kind, act.what))
1688
1689                 testbed.needs_reset()
1690                 if act.kind == 'control':
1691                         control_override = act.af
1692                 if act.kind == 'deb':
1693                         binaries.register(act,act.pkg,act.af,'fortests',
1694                                 ['deb:'+act.pkg])
1695                 if act.kind == 'dsc' or act.kind == 'ubtree':
1696                         for (pkg,bin) in act.binaries:
1697                                 binaries.register(act,pkg,bin,'fortests',
1698                                         act.blamed)
1699                 if act.kind == 'dsc':
1700                         if act.ah['dsc_tests']:
1701                                 debug_a3('read control ...')
1702                                 stanzas = read_control(act, act.tests_tree,
1703                                                 control_override)
1704                                 testbed.blamed += act.blamed
1705                                 debug_a3('run_tests ...')
1706                                 run_tests(stanzas, act.tests_tree)
1707                         control_override = None
1708                 if act.kind == 'tree' or act.kind == 'ubtree':
1709                         testbed.blame('arg:'+act.af.spec)
1710                         stanzas = read_control(act, act.af, control_override)
1711                         debug_a3('run_tests ...')
1712                         run_tests(stanzas, act.af)
1713                         control_override = None
1714         debug_a1('tests done.')
1715
1716 def main():
1717         global testbed
1718         global tmpdir
1719         try:
1720                 parse_args()
1721         except SystemExit, se:
1722                 os._exit(20)
1723         try:
1724                 setup_trace()
1725                 testbed = Testbed()
1726                 testbed.start()
1727                 finalise_options()
1728                 process_actions()
1729         except:
1730                 ec = print_exception(sys.exc_info(), '')
1731                 cleanup()
1732                 os._exit(ec)
1733         cleanup()
1734         os._exit(errorcode)
1735
1736 main()