runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 binaries = None         # Binaries (.debs we have registered)
  49 build_essential = ["build-essential"]
  50
  51 #---------- output handling
  52 #
  53 #  There are at least the following kinds of output:
  54 #
  55 #   1. stderr output which consists of
  56 #   1a. our errors
  57 #   1b. stuff printed to stderr by the virtualisation server
  58 #   1c. stuff printed to stderr by our short-lived subprocesseses
  59 #           which we don't expect to fail
  60 #
  61 #   2. trace information, which consists of
  62 #   2a. our trace information from calls to debug()
  63 #   2b. progress information and stderr output from
  64 #        general scripts we run on the host
  65 #   2c. progress information and stderr output from things
  66 #        we run on the testbed including builds
  67 #   2d. stderr and stdout output from actual tests
  68 #
  69 #   xxxx
  70 #   3. actual test results (printed to our stdout)
  71 #
  72 # Cloning of 1a and 2a, where necessary, is done by us writing
  73 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  74 # is done by forking off a copy of ourselves to do plumbing,
  75 # which copy we wait for at the appropriate point.
  76
  77 class DummyOpts:
  78  def __init__(do): do.debuglevel = 0
  79
  80 opts = DummyOpts()
  81 trace_stream = None
  82 summary_stream = None
  83
  84 def pstderr(m):
  85         print >>sys.stderr, m
  86         if trace_stream is not None: print >>trace_stream, m
  87
  88 def debug(m, minlevel=0):
  89         if opts.debuglevel < minlevel: return
  90         if opts.quiet and trace_stream is None: return
  91         p = 'adt-run: debug'
  92         if minlevel: p += `minlevel`
  93         p += ': '
  94         for l in m.rstrip('\n').split('\n'):
  95                 s = p + l
  96                 if not opts.quiet: print >>sys.stderr, s
  97                 if trace_stream is not None: print >>trace_stream, s
  98
  99 def debug_file(hp, minlevel=0):
 100         if opts.debuglevel < minlevel: return
 101         def do_copy(stream, what):
 102                 rc = subprocess.call(['cat',hp], stdout=stream)
 103                 if rc: bomb('cat failed copying data from %s'
 104                             ' to %s, exit code %d' % (hp, what, rc))
 105         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 106         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 107
 108 class Errplumb:
 109  def __init__(ep, critical=False):
 110         to_stderr = critical or not opts.quiet
 111         count = to_stderr + (trace_stream is not None)
 112         if count == 0:
 113                 ep.stream = open('/dev/null','w')
 114                 ep._sp = None
 115         elif count == 1:
 116                 if to_stderr: ep.stream = sys.stderr
 117                 else: ep.stream = trace_stream
 118                 ep._sp = None
 119         else:
 120                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 121                         stdin=subprocess.PIPE, stdout=trace_stream,
 122                         close_fds=True)
 123                 ep.stream = ep._sp.stdin
 124  def wait(ep):
 125         if ep._sp is None: return
 126         ep._sp.stdin.close()
 127         rc = ep._sp.wait()
 128         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 129         ep._sp = None
 130
 131 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 132         if dbg is not None:
 133                 if isinstance(dbg,tuple): (what,script) = dbg
 134                 else: (what,script) = (dbg,None)
 135                 debug_subprocess(what, cmdl, script=script)
 136         ep = Errplumb(critical)
 137         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 138         output = running.communicate()[0]
 139         rc = running.wait()
 140         ep.wait()
 141         return (rc, output)
 142
 143 def psummary(m):
 144         if summary_stream is not None: print >>summary_stream, m
 145
 146 def preport(m):
 147         print m
 148         sys.stdout.flush()
 149         if trace_stream is not None: print >>trace_stream, m
 150         psummary(m)
 151
 152 def report(tname, result):
 153         preport('%-20s %s' % (tname, result))
 154
 155 #---------- errors we define
 156
 157 class Quit:
 158         def __init__(q,ec,m): q.ec = ec; q.m = m
 159
 160 def bomb(m):
 161         raise Quit(20, "unexpected error: %s" % m)
 162
 163 def badpkg(m):
 164         preport('blame: ' + ' '.join(testbed.blamed))
 165         preport('badpkg: ' + m)
 166         raise Quit(12, "erroneous package: %s" % m)
 167
 168 class Unsupported:
 169  def __init__(u, lno, m):
 170         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 171         else: u.m = m
 172  def report(u, tname):
 173         global errorcode
 174         errorcode != 2
 175         report(tname, 'SKIP %s' % u.m)
 176
 177 #---------- convenience function
 178
 179 def mkdir_okexist(pathname, mode=02755):
 180         try:
 181                 os.mkdir(pathname, mode)
 182         except OSError, oe:
 183                 if oe.errno != errno.EEXIST: raise
 184
 185 def rmtree(what, pathname):
 186         debug('/ %s rmtree %s' % (what, pathname), 2)
 187         shutil.rmtree(pathname)
 188
 189 def debug_subprocess(what, cmdl=None, script=None):
 190         o = '$ '+what+':'
 191         if cmdl is not None:
 192                 ol = []
 193                 for x in cmdl:
 194                         if x is script: x = '<SCRIPT>'
 195                         ol.append(x.    replace('\\','\\\\').
 196                                         replace(' ','\\ ')      )
 197                 o += ' '+ ' '.join(ol)
 198         debug(o)
 199         if script is not None and opts.debuglevel >= 1:
 200                 o = ''
 201                 for l in script.rstrip('\n').split('\n'):
 202                         o += '$     '+l+'\n'
 203                 debug(o,1)
 204
 205 def flatten(l):
 206         return reduce((lambda a,b: a + b), l, [])
 207
 208 #---------- fancy automatic file-copying class
 209
 210 class AutoFile:
 211         # p.what
 212         # p.path[tb]    None or path    not None => path known
 213         # p.file[tb]    None or path    not None => file exists
 214         # p.spec        string or None
 215         # p.spec_tbp    True or False, or not set if spec is None
 216         # p.dir         '' or '/'
 217
 218  def __init__(p, what):
 219         p.what = what
 220         p.path = [None,None]
 221         p.file = [None,None]
 222         p.spec = None
 223         p.dir = ''
 224
 225  def __repr__(p):
 226         return "<AF@%s>" % p.__str__()
 227  def __str__(p):
 228         def ptbp(tbp):
 229                 if p.path[tbp] is None: return '-'+p.dir
 230                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 231                 else: return p.path[tbp]+p.dir+'!'
 232         out = p.what
 233         if p.spec is not None:
 234                 if not hasattr(p,'spec_tb'): out += '~'
 235                 elif p.spec_tbp: out += '#'
 236                 else: out += '='
 237                 out += p.spec
 238         out += ':'
 239         out += ptbp(False)
 240         out += '|'
 241         out += ptbp(True)
 242         return out
 243
 244  def _wrong(p, how):
 245         xtra = ''
 246         if p.spec is not None:
 247                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 248         raise ("internal error: %s (%s)" % (how, str(p)))
 249
 250  def _ensure_path(p, tbp):
 251         if p.path[tbp] is None:
 252                 if '/' in p.what:
 253                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 254                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 255                 if not tbp:
 256                         p.path[tbp] = tmpdir+'/'+p.what
 257                 else:
 258                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 259
 260  def write(p, tbp=False):
 261         p._debug('write %s...' % 'HT'[tbp])
 262         p._ensure_path(tbp)
 263
 264         if p.dir and not p.file[tbp]:
 265                 if not tbp:
 266                         p._debug('mkdir H')
 267                         mkdir_okexist(p.path[tbp])
 268                 else:
 269                         cmdl = ['sh','-ec',
 270                                 'test -d "$1" || mkdir -p "$1"',
 271                                 'x', p.path[tbp]]
 272                         tf_what = urllib.quote(p.what).replace('/','%2F')
 273                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 274                         if rc: bomb('failed to create directory %s' %
 275                                 p.path[tbp])
 276
 277         p.file[tbp] = p.path[tbp]
 278         return p.path[tbp]
 279
 280  def read(p, tbp=False):
 281         p._debug('read %s...' % 'HT'[tbp])
 282         p._ensure_path(tbp)
 283
 284         if p.file[tbp] is None:
 285                 if p.file[not tbp] is None:
 286                         p._wrong("requesting read but nonexistent")
 287                 cud = ['copyup','copydown'][tbp]
 288                 src = p.file[not tbp] + p.dir
 289                 dst = p.path[tbp] + p.dir
 290                 testbed.command(cud, (src, dst))
 291                 p.file[tbp] = p.path[tbp]
 292
 293         return p.file[tbp] + p.dir
 294
 295  def invalidate(p, tbp=False):
 296         p.file[tbp] = None
 297         p._debug('invalidated %s' % 'HT'[tbp])
 298
 299  def _debug(p, m):
 300         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 301
 302  def _constructed(p):
 303         p._debug('constructed: '+str(p))
 304         p._check()
 305
 306  def _check(p):
 307         for tbp in [False,True]:
 308          for pf in [p.path, p.file]:
 309                 if pf[tbp] is None: continue
 310                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 311                 if p.dir and pf[tbp].endswith('/'):
 312                         pf[tbp] = pf[tbp].rstrip('/')
 313                         if not pf[tbp]: pf[tbp] = '/'
 314                 if not p.dir and pf[tbp].endswith('/'):
 315                         bomb("directory `%s' specified for "
 316                              "non-directory %s" % (pf[tbp], p.what))
 317
 318  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 319         AutoFile.__init__(p,what)
 320         sh_on = ''; sh_sibl = ''
 321         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 322         if sibling: sh_sibl=' (sibling)'
 323         parent._debug('using as base: %s: %s%s%s' %
 324                         (str(parent), leaf, sh_on, sh_sibl))
 325         if not sibling and not parent.dir:
 326                 parent._wrong('asked for non-sibling relative path of non-dir')
 327         if sibling: trim = os.path.dirname
 328         else: trim = lambda x: x
 329         for tbp in [False,True]:
 330                 if parent.path[tbp] is None: continue
 331                 trimmed = trim(parent.path[tbp])
 332                 if trimmed: trimmed += '/'
 333                 p.path[tbp] = trimmed + leaf
 334                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 335                         p.file[tbp] = p.path[tbp]
 336
 337 class InputFile(AutoFile):
 338  def _init(p, what, spec, spec_tbp=False):
 339         AutoFile.__init__(p, what)
 340         p.spec = spec
 341         p.spec_tbp = spec_tbp
 342         p.path[spec_tbp] = spec
 343         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 344  def __init__(p, what, spec, spec_tbp=False):
 345         p._init(what,spec,spec_tbp)
 346         p._constructed()
 347
 348 class InputDir(InputFile):
 349  def __init__(p, what, spec, spec_tbp=False):
 350         InputFile._init(p,what,spec,spec_tbp)
 351         p.dir = '/'
 352         p._constructed()
 353
 354 class OutputFile(AutoFile):
 355  def _init(p, what, spec, spec_tbp=False):
 356         AutoFile.__init__(p, what)
 357         p.spec = spec
 358         p.spec_tbp = spec_tbp
 359         p.path[spec_tbp] = spec
 360  def __init__(p, what, spec, spec_tbp=False):
 361         p._init(what,spec,spec_tbp)
 362         p._constructed()
 363
 364 class OutputDir(OutputFile):
 365  def __init__(p, what, spec, spec_tbp=False):
 366         OutputFile._init(p,what,spec,spec_tbp)
 367         p.dir = '/'
 368         p._constructed()
 369
 370 class RelativeInputFile(AutoFile):
 371  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 372         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 373         p._constructed()
 374
 375 class RelativeOutputFile(AutoFile):
 376  def __init__(p, what, parent, leaf, sibling=False):
 377         p._relative_init(what, parent, leaf, None, False, sibling)
 378         p._constructed()
 379
 380 class TemporaryFile(AutoFile):
 381  def __init__(p, what):
 382         AutoFile.__init__(p, what)
 383         p._constructed()
 384
 385 class TemporaryDir(AutoFile):
 386  def __init__(p, what):
 387         AutoFile.__init__(p, what)
 388         p.dir = '/'
 389         p._constructed()
 390
 391 #---------- parsing and representation of the arguments
 392
 393 class Action:
 394  def __init__(a, kind, af, arghandling, what):
 395         # extra attributes get added during processing
 396         a.kind = kind
 397         a.af = af
 398         a.ah = arghandling
 399         a.what = what
 400         a.missing_tests_control = False
 401  def __repr__(a):
 402         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 403
 404 def parse_args():
 405         global opts
 406         global n_non_actions # argh, stupid python scoping rules
 407         usage = "%prog <options> --- <virt-server>..."
 408         parser = OptionParser(usage=usage)
 409         pa = parser.add_option
 410         pe = parser.add_option
 411
 412         arghandling = {
 413                 'dsc_tests': True,
 414                 'dsc_filter': '*',
 415                 'deb_forbuilds': 'auto',
 416                 'deb_fortests': 'auto',
 417                 'tb': False,
 418                 'override_control': None
 419         }
 420         initial_arghandling = arghandling.copy()
 421         n_non_actions = 0
 422
 423         #----------
 424         # actions (ie, test sets to run, sources to build, binaries to use):
 425
 426         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 427                 global n_non_actions
 428                 parser.largs.append((value,kindpath))
 429                 n_non_actions += not(is_act)
 430
 431         def pa_action(long, metavar, kindpath, help, is_act=True):
 432                 pa('','--'+long, action='callback', callback=cb_action,
 433                         nargs=1, type='string',
 434                         callback_args=(long,kindpath,is_act), help=help)
 435
 436         pa_action('built-tree',         'TREE', '@/',
 437                 help='run tests from build tree TREE')
 438
 439         pa_action('unbuilt-tree',       'TREE', '@//',
 440                 help='run tests from build tree TREE')
 441
 442         pa_action('source',             'DSC', '@.dsc',
 443                 help='build DSC and use its tests and/or'
 444                     ' generated binary packages')
 445
 446         pa_action('binary',             'DEB', '@.deb',
 447                help='use binary package DEB according'
 448                     ' to most recent --binaries-* settings')
 449
 450         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 451                help='run tests from control file CONTROL instead,'
 452                     ' (applies to next test suite only)')
 453
 454         #----------
 455         # argument handling settings (what ways to use action
 456         #  arguments, and pathname processing):
 457
 458         def cb_setah(option, opt_str, value, parser, toset,setval):
 459                 if type(setval) == list:
 460                         if not value in setval:
 461                                 parser.error('value for %s option (%s) is not '
 462                                  'one of the permitted values (%s)' %
 463                                  (value, opt_str, setval.join(' ')))
 464                 elif setval is not None:
 465                         value = setval
 466                 for v in toset:
 467                         arghandling[v] = value
 468                 parser.largs.append(arghandling.copy())
 469
 470         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 471                 type = metavar
 472                 if type is not None: type = 'string'
 473                 pa('',long, action='callback', callback=cb_setah,
 474                    callback_args=(affected,effect), **kwargs)
 475
 476         #---- paths: host or testbed:
 477         #
 478         pa_setah('--paths-testbed', ['tb'],True,
 479                 help='subsequent path specifications refer to the testbed')
 480         pa_setah('--paths-host', ['tb'],False,
 481                 help='subsequent path specifications refer to the host')
 482
 483         #---- source processing settings:
 484
 485         pa_setah('--sources-tests', ['dsc_tests'],True,
 486                 help='run tests from builds of subsequent sources')
 487         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 488                 help='do not run tests from builds of subsequent sources')
 489
 490         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 491                 type='string', metavar='PATTERN-LIST',
 492                 help='from subsequent sources, use binaries matching'
 493                      ' PATTERN-LIST (comma-separated glob patterns)'
 494                      ' according to most recent --binaries-* settings')
 495         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 496                 help='from subsequent sources, do not use any binaries')
 497
 498         #---- binary package processing settings:
 499
 500         def pa_setahbins(long,toset,how):
 501          pa_setah(long, toset,['ignore','auto','install'],
 502                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 503                 help=how+' ignore binaries, install them as needed'
 504                         ' for dependencies, or unconditionally install'
 505                         ' them, respectively')
 506         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 507         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 508         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 509
 510         #----------
 511         # general options:
 512
 513         def cb_vserv(op,optstr,value,parser):
 514                 parser.values.vserver = list(parser.rargs)
 515                 del parser.rargs[:]
 516
 517         def cb_path(op,optstr,value,parser, constructor,long,dir):
 518                 name = long.replace('-','_')
 519                 af = constructor(long, value,arghandling['tb'])
 520                 setattr(parser.values, name, af)
 521
 522         def pa_path(long, constructor, help, dir=False):
 523                 pa('','--'+long, action='callback', callback=cb_path,
 524                         callback_args=(constructor,long,dir),
 525                         nargs=1, type='string',
 526                         help=help, metavar='PATH')
 527
 528         pa_path('output-dir', OutputDir, dir=True,
 529                 help='write stderr/out files in PATH')
 530
 531         pa('','--tmp-dir',              type='string', dest='tmpdir',
 532                 help='write temporary files to TMPDIR, emptying it'
 533                      ' beforehand and leaving it behind at the end')
 534         pa('','--log-file',             type='string', dest='logfile',
 535                 help='write the log LOGFILE, emptying it beforehand,'
 536                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 537         pa('','--summary-file',         type='string', dest='summary',
 538                 help='write a summary report to SUMMARY,'
 539                      ' emptying it beforehand')
 540
 541         pa('','--user',                 type='string', dest='user',
 542                 help='run tests as USER (needs root on testbed)')
 543         pa('','--gain-root',            type='string', dest='gainroot',
 544                 help='prefix debian/rules binary with GAINROOT')
 545         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 546         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 547         pa('','--gnupg-home',           type='string', dest='gnupghome',
 548                 default='~/.autopkgtest/gpg',
 549                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 550                         " signing private apt archive);"
 551                         " `fresh' means generate new key each time.")
 552
 553         #----------
 554         # actual meat:
 555
 556         class SpecialOption(optparse.Option): pass
 557         vs_op = SpecialOption('','--VSERVER-DUMMY')
 558         vs_op.action = 'callback'
 559         vs_op.type = None
 560         vs_op.default = None
 561         vs_op.nargs = 0
 562         vs_op.callback = cb_vserv
 563         vs_op.callback_args = ( )
 564         vs_op.callback_kwargs = { }
 565         vs_op.help = 'introduces virtualisation server and args'
 566         vs_op._short_opts = []
 567         vs_op._long_opts = ['---']
 568
 569         pa(vs_op)
 570
 571         (opts,args) = parser.parse_args()
 572         if not hasattr(opts,'vserver'):
 573                 parser.error('you must specifiy --- <virt-server>...')
 574         if n_non_actions >= len(parser.largs):
 575                 parser.error('nothing to do specified')
 576
 577         arghandling = initial_arghandling
 578         opts.actions = []
 579         ix = 0
 580         for act in args:
 581                 if type(act) == dict:
 582                         arghandling = act
 583                         continue
 584                 elif type(act) == tuple:
 585                         pass
 586                 elif type(act) == str:
 587                         act = (act,act)
 588                 else:
 589                         raise ("unknown action in list `%s' having"
 590                               " type `%s'" % (act, type(act)))
 591                 (pathstr, kindpath) = act
 592
 593                 constructor = InputFile
 594                 if type(kindpath) is tuple:             kind = kindpath[0]
 595                 elif kindpath.endswith('.deb'):         kind = 'deb'
 596                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 597                 elif kindpath.endswith('//'):
 598                         kind = 'ubtree'
 599                         constructor = InputDir
 600                 elif kindpath.endswith('/'):
 601                         kind = 'tree'
 602                         constructor = InputDir
 603                 else: parser.error("do not know how to handle filename \`%s';"
 604                         " specify --source --binary or --build-tree")
 605
 606                 what = '%s%s' % (kind,ix); ix += 1
 607
 608                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 609                 else: fwhatx = '-'+kind
 610
 611                 af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 612                 opts.actions.append(Action(kind, af, arghandling, what))
 613
 614 def setup_trace():
 615         global trace_stream, tmpdir, summary_stream
 616
 617         if opts.tmpdir is not None:
 618                 rmtree('tmpdir(specified)',opts.tmpdir)
 619                 mkdir_okexist(opts.tmpdir, 0700)
 620                 tmpdir = opts.tmpdir
 621         else:
 622                 assert(tmpdir is None)
 623                 tmpdir = tempfile.mkdtemp()
 624
 625         if opts.logfile is None:
 626                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 627                         opts.logfile = opts.output_dir.spec + '/log'
 628                 elif opts.tmpdir is not None:
 629                         opts.logfile = opts.tmpdir + '/log'
 630         if opts.logfile is not None:
 631                 trace_stream = open(opts.logfile, 'w', 0)
 632         if opts.summary is not None:
 633                 summary_stream = open(opts.summary, 'w', 0)
 634
 635         debug('options: '+`opts`, 1)
 636
 637 def finalise_options():
 638         global opts, tb, build_essential
 639
 640         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 641                 opts.user = ''
 642
 643         if opts.user is None:
 644                 su = 'suggested-normal-user='
 645                 ul = [
 646                         e[len(su):]
 647                         for e in testbed.caps
 648                         if e.startswith(su)
 649                         ]
 650                 if ul:
 651                         opts.user = ul[0]
 652                 else:
 653                         opts.user = ''
 654
 655         if opts.user:
 656                 if 'root-on-testbed' not in testbed.caps:
 657                         pstderr("warning: virtualisation"
 658                                 " system does not offer root on testbed,"
 659                                 " but --user option specified: failure likely")
 660                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 661         else:
 662                 opts.user_wrap = lambda x: x
 663
 664         if opts.gainroot is None:
 665                 opts.gainroot = ''
 666                 if (opts.user or
 667                     'root-on-testbed' not in testbed.caps):
 668                         opts.gainroot = 'fakeroot'
 669                         build_essential += ['fakeroot']
 670
 671         if opts.gnupghome.startswith('~/'):
 672                 try: home = os.environ['HOME']
 673                 except KeyError:
 674                         parser.error("HOME environment variable"
 675                                 " not set, needed for --gnupghome=`%s"
 676                                 % opts.gnupghome)
 677                 opts.gnupghome = home + opts.gnupghome[1:]
 678         elif opts.gnupghome == 'fresh':
 679                 opts.gnupghome = None
 680
 681 #---------- testbed management - the Testbed class
 682
 683 class Testbed:
 684  def __init__(tb):
 685         tb.sp = None
 686         tb.lastsend = None
 687         tb.scratch = None
 688         tb.modified = False
 689         tb.blamed = []
 690         tb._ephemeral = []
 691         tb._debug('init')
 692         tb._need_reset_apt = False
 693  def _debug(tb, m, minlevel=0):
 694         debug('** '+m, minlevel)
 695  def start(tb):
 696         tb._debug('start')
 697         p = subprocess.PIPE
 698         debug_subprocess('vserver', opts.vserver)
 699         tb._errplumb = Errplumb(True)
 700         tb.sp = subprocess.Popen(opts.vserver,
 701                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 702         tb.expect('ok')
 703         tb.caps = tb.commandr('capabilities')
 704  def stop(tb):
 705         tb._debug('stop')
 706         tb.close()
 707         if tb.sp is None: return
 708         ec = tb.sp.returncode
 709         if ec is None:
 710                 tb.sp.stdout.close()
 711                 tb.send('quit')
 712                 tb.sp.stdin.close()
 713                 ec = tb.sp.wait()
 714         if ec:
 715                 tb.bomb('testbed gave exit status %d after quit' % ec)
 716         tb._errplumb.wait()
 717  def open(tb):
 718         tb._debug('open, scratch=%s' % tb.scratch)
 719         if tb.scratch is not None: return
 720         pl = tb.commandr('open')
 721         tb.scratch = InputDir('tb-scratch', pl[0], True)
 722         tb.deps_processed = []
 723  def mungeing_apt(tb):
 724         if not 'revert' in tb.caps:
 725                 tb._need_reset_apt = True
 726  def reset_apt(tb):
 727         if not tb._need_reset_apt: return
 728         what = 'aptget-update-reset'
 729         cmdl = ['apt-get','-qy','update']
 730         rc = tb.execute(what, cmdl)
 731         if rc:
 732                 pstderr("\n" "warning: failed to restore"
 733                         " testbed apt cache, exit code %d" % rc)
 734         tb._need_reset_apt = False
 735  def close(tb):
 736         tb._debug('close, scratch=%s' % tb.scratch)
 737         if tb.scratch is None: return
 738         tb.scratch = None
 739         if tb.sp is None: return
 740         tb.command('close')
 741  def prepare1(tb, deps_new):
 742         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 743                 (tb.modified, tb.deps_processed, deps_new), 1)
 744         if 'revert' in tb.caps and (tb.modified or
 745             [d for d in tb.deps_processed if d not in deps_new]):
 746                 for af in tb._ephemeral: af.read(False)
 747                 tb._debug('reset **')
 748                 tb.command('revert')
 749                 tb.blamed = []
 750                 for af in tb._ephemeral: af.invalidate(True)
 751         tb.modified = False
 752  def prepare2(tb, deps_new):
 753         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 754         binaries.publish()
 755         tb._install_deps(deps_new)
 756  def prepare(tb, deps_new):
 757         tb.prepare1(deps_new)
 758         tb.prepare2(deps_new)
 759  def register_ephemeral(tb, af):
 760         tb._ephemeral.append(af)
 761  def _install_deps(tb, deps_new):
 762         tb._debug(' installing dependencies '+`deps_new`, 1)
 763         tb.deps_processed = deps_new
 764         if not deps_new: return
 765         dstr = ', '.join(deps_new)
 766         script = binaries.apt_pkg_gdebi_script(
 767                 dstr, [[
 768                 'from GDebi.DebPackage import DebPackage',
 769                 'd = DebPackage(cache)',
 770                 'res = d.satisfyDependsStr(arg)',
 771                 ]])
 772         cmdl = ['python','-c',script]
 773         what = 'install-deps'
 774         rc = testbed.execute(what+'-debinstall', cmdl, script=script)
 775         if rc: badpkg('dependency install failed, exit code %d' % rc)
 776  def needs_reset(tb):
 777         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 778         tb.modified = True
 779  def blame(tb, m):
 780         tb._debug('blame += %s' % m, 1)
 781         tb.blamed.append(m)
 782  def bomb(tb, m):
 783         tb._debug('bomb %s' % m)
 784         if tb.sp is not None:
 785                 tb.sp.stdout.close()
 786                 tb.sp.stdin.close()
 787                 ec = tb.sp.wait()
 788                 if ec: pstderr('adt-run: testbed failing,'
 789                                 ' exit status %d' % ec)
 790         tb.sp = None
 791         raise Quit(16, 'testbed failed: %s' % m)
 792  def send(tb, string):
 793         tb.sp.stdin
 794         try:
 795                 debug('>> '+string, 2)
 796                 print >>tb.sp.stdin, string
 797                 tb.sp.stdin.flush()
 798                 tb.lastsend = string
 799         except:
 800                 (type, value, dummy) = sys.exc_info()
 801                 tb.bomb('cannot send to testbed: %s' % traceback.
 802                         format_exception_only(type, value))
 803  def expect(tb, keyword, nresults=None):
 804         l = tb.sp.stdout.readline()
 805         if not l: tb.bomb('unexpected eof from the testbed')
 806         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 807         l = l.rstrip('\n')
 808         debug('<< '+l, 2)
 809         ll = l.split()
 810         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 811         if ll[0] != keyword:
 812                 if tb.lastsend is None:
 813                         tb.bomb("got banner `%s', expected `%s...'" %
 814                                 (l, keyword))
 815                 else:
 816                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 817                                 (tb.lastsend, l, keyword))
 818         ll = ll[1:]
 819         if nresults is not None and len(ll) != nresults:
 820                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 821                         " expected %d result parameters" %
 822                         (string, l, len(ll), nresults))
 823         return ll
 824  def commandr(tb, cmd, args=(), nresults=None):
 825         # pass args=[None,...] or =(None,...) to avoid more url quoting
 826         if type(cmd) is str: cmd = [cmd]
 827         if len(args) and args[0] is None: args = args[1:]
 828         else: args = map(urllib.quote, args)
 829         al = cmd + args
 830         tb.send(string.join(al))
 831         ll = tb.expect('ok', nresults)
 832         rl = map(urllib.unquote, ll)
 833         return rl
 834  def command(tb, cmd, args=()):
 835         tb.commandr(cmd, args, 0)
 836  def commandr1(tb, cmd, args=()):
 837         rl = tb.commandr(cmd, args, 1)
 838         return rl[0]
 839  def execute(tb, what, cmdl,
 840                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 841                 script=False, tmpdir=None):
 842         # Options for script:
 843         #   False - do not call debug_subprocess, no synch. reporting required
 844         #   None or string - call debug_subprocess with that value,
 845         #                       plumb stderr through synchronously if possible
 846         # Options for se:
 847         #   None - usual Errplumb (output is of kind 2c)
 848         #   string - path on testbed (output is of kind 2d)
 849
 850         if script is not False: debug_subprocess(what, cmdl, script=script)
 851         if cwd is None: cwd = tb.scratch.write(True)
 852
 853         xdump = None
 854         if se is None:
 855                 ep = Errplumb()
 856                 se_catch = TemporaryFile(what+'-xerr')
 857                 se_use = se_catch.write(True)
 858                 if not opts.quiet: xdump = 'debug=2-2'
 859                 elif trace_stream is not None:
 860                         xdump = 'debug=2-%d' % trace_stream.fileno()
 861         else:
 862                 ep = None
 863                 se_catch = None
 864                 se_use = se
 865
 866         cmdl = [None,
 867                 ','.join(map(urllib.quote, cmdl)),
 868                 si, so, se_use, cwd]
 869
 870         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 871         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 872
 873         rc = tb.commandr1('execute', cmdl)
 874         try: rc = int(rc)
 875         except ValueError: bomb("execute for %s gave invalid response `%s'"
 876                                         % (what,rc))
 877
 878         if se_catch is not None:
 879                 debug_file(se_catch.read())
 880         if ep is not None:
 881                 ep.wait()
 882
 883         return rc
 884
 885 #---------- representation of test control files: Field*, Test, etc.
 886
 887 class FieldBase:
 888  def __init__(f, fname, stz, base, tnames, vl):
 889         assert(vl)
 890         f.stz = stz
 891         f.base = base
 892         f.tnames = tnames
 893         f.vl = vl
 894  def words(f):
 895         def distribute(vle):
 896                 (lno, v) = vle
 897                 r = v.split()
 898                 r = map((lambda w: (lno, w)), r)
 899                 return r
 900         return flatten(map(distribute, f.vl))
 901  def atmostone(f):
 902         if len(vl) == 1:
 903                 (f.lno, f.v) = vl[0]
 904         else:
 905                 raise Unsupported(f.vl[1][0],
 906                         'only one %s field allowed' % fn)
 907         return f.v
 908
 909 class FieldIgnore(FieldBase):
 910  def parse(f): pass
 911
 912 class Restriction:
 913  def __init__(r,rname,base): pass
 914
 915 class Restriction_rw_build_tree(Restriction): pass
 916 class Restriction_breaks_testbed(Restriction):
 917  def __init__(r, rname, base):
 918         if 'revert' not in testbed.caps:
 919                 raise Unsupported(f.lno,
 920                         'Test breaks testbed but testbed cannot revert')
 921 class Restriction_needs_root(Restriction):
 922  def __init__(r, rname, base):
 923         if 'root-on-testbed' not in testbed.caps:
 924                 raise Unsupported(f.lno,
 925                         'Test needs root on testbed which is not available')
 926
 927 class Field_Restrictions(FieldBase):
 928  def parse(f):
 929         for wle in f.words():
 930                 (lno, rname) = wle
 931                 nrname = rname.replace('-','_')
 932                 try: rclass = globals()['Restriction_'+nrname]
 933                 except KeyError: raise Unsupported(lno,
 934                         'unknown restriction %s' % rname)
 935                 r = rclass(nrname, f.base)
 936                 f.base['restriction_names'].append(rname)
 937                 f.base['restrictions'].append(r)
 938
 939 class Field_Features(FieldIgnore):
 940  def parse(f):
 941         for wle in f.words():
 942                 (lno, fname) = wle
 943                 f.base['feature_names'].append(fname)
 944                 nfname = fname.replace('-','_')
 945                 try: fclass = globals()['Feature_'+nfname]
 946                 except KeyError: continue
 947                 ft = fclass(nfname, f.base)
 948                 f.base['features'].append(ft)
 949
 950 class Field_Tests(FieldIgnore): pass
 951
 952 class Field_Depends(FieldBase):
 953  def parse(f):
 954         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 955         dl = map(lambda x: x.strip(),
 956                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 957         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
 958         for d in dl:
 959                 if re.search(d):
 960                         badpkg("Test Depends field contains dependency"
 961                                " `%s' with invalid characters" % d)
 962         f.base['depends'] = dl
 963
 964 class Field_Tests_directory(FieldBase):
 965  def parse(f):
 966         td = atmostone(f)
 967         if td.startswith('/'): raise Unspported(f.lno,
 968                 'Tests-Directory may not be absolute')
 969         f.base['testsdir'] = td
 970
 971 def run_tests(stanzas, tree):
 972         global errorcode, testbed
 973         if stanzas == ():
 974                 report('*', 'SKIP no tests in this package')
 975                 errorcode |= 8
 976         for stanza in stanzas:
 977                 tests = stanza[' tests']
 978                 if not tests:
 979                         report('*', 'SKIP package has metadata but no tests')
 980                         errorcode |= 8
 981                 for t in tests:
 982                         t.prepare()
 983                         t.run(tree)
 984                         if 'breaks-testbed' in t.restriction_names:
 985                                 testbed.needs_reset()
 986                 testbed.needs_reset()
 987
 988 class Test:
 989  def __init__(t, tname, base, act):
 990         if '/' in tname: raise Unsupported(base[' lno'],
 991                 'test name may not contain / character')
 992         for k in base: setattr(t,k,base[k])
 993         t.tname = tname
 994         t.act = act
 995         t.what = act.what+'t-'+tname
 996         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
 997         else: t.path = tname
 998         t._debug('constructed; path=%s' % t.path)
 999         t._debug(' .depends=%s' % t.depends)
1000  def _debug(t, m):
1001         debug('& %s: %s' % (t.what, m))
1002  def report(t, m):
1003         report(t.what, m)
1004  def reportfail(t, m):
1005         global errorcode
1006         errorcode |= 4
1007         report(t.what, 'FAIL ' + m)
1008  def prepare(t):
1009         t._debug('preparing')
1010         dn = []
1011         for d in t.depends:
1012                 t._debug(' processing dependency '+d)
1013                 if not '@' in d:
1014                         t._debug('  literal dependency '+d)
1015                         dn.append(d)
1016                 else:
1017                         for (pkg,bin) in t.act.binaries:
1018                                 d = d.replace('@',pkg)
1019                                 t._debug('  synthesised dependency '+d)
1020                                 dn.append(d)
1021         testbed.prepare(dn)
1022  def run(t, tree):
1023         t._debug('[----------------------------------------')
1024         def stdouterr(oe):
1025                 idstr = t.what + '-' + oe
1026                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1027                         use_dir = opts.output_dir
1028                 else:
1029                         use_dir = testbed.scratch
1030                 return RelativeOutputFile(idstr, use_dir, idstr)
1031
1032         t.act.work.write(True)
1033
1034         af = RelativeInputFile(t.what, tree, t.path)
1035         so = stdouterr('stdout')
1036         se = stdouterr('stderr')
1037
1038         tf = af.read(True)
1039         tmpdir = None
1040         tree.read(True)
1041
1042         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1043         if rc: bomb('failed to chmod +x %s' % tf)
1044
1045         if 'needs-root' not in t.restriction_names and opts.user is not None:
1046                 tfl = ['su',opts.user,'-c',tf]
1047                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1048                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1049                 if opts.user: script += '; chown %s "$1"' % opts.user
1050                 if 'rw-build-tree' in t.restriction_names:
1051                         script += '; chown -R %s "$2"' % opts.user
1052                 rc = testbed.execute('mktmpdir-'+t.what,
1053                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1054                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1055                                 % (tmpdir, rc))
1056         else:
1057                 tfl = [tf]
1058
1059         rc = testbed.execute('test-'+t.what, tfl,
1060                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1061                 tmpdir=tmpdir)
1062
1063         so_read = so.read()
1064         se_read = se.read()
1065
1066         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1067         stab = os.stat(se_read)
1068         if stab.st_size != 0:
1069                 l = open(se_read).readline()
1070                 l = l.rstrip('\n \t\r')
1071                 if len(l) > 35: l = l[:35] + '...'
1072                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1073                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1074                 debug_file(se_read)
1075         elif rc != 0:
1076                 t.reportfail('non-zero exit status %d' % rc)
1077         else:
1078                 t.report('PASS')
1079
1080         stab = os.stat(so_read)
1081         if stab.st_size != 0:
1082                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1083                 debug_file(so_read)
1084
1085         t._debug('----------------------------------------]')
1086
1087 def read_control(act, tree, control_override):
1088         stanzas = [ ]
1089
1090         if control_override is not None:
1091                 control_af = control_override
1092                 testbed.blame('arg:'+control_override.spec)
1093         else:
1094                 if act.missing_tests_control:
1095                         return ()
1096                 control_af = RelativeInputFile(act.what+'-testcontrol',
1097                         tree, 'debian/tests/control')
1098         try:
1099                 control = open(control_af.read(), 'r')
1100         except OSError, oe:
1101                 if oe[0] != errno.ENOENT: raise
1102                 return []
1103
1104         lno = 0
1105         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1106         stz = None      # stz[field_name][index] = (lno, value)
1107                         # special field names:
1108                         # stz[' lno'] = number
1109                         # stz[' tests'] = list of Test objects
1110         def end_stanza(stz):
1111                 if stz is None: return
1112                 stz[' errs'] = 0
1113                 stanzas.append(stz)
1114                 stz = None
1115                 hcurrent = None
1116
1117         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1118         while 1:
1119                 l = control.readline()
1120                 if not l: break
1121                 lno += 1
1122                 if not l.endswith('\n'): badctrl('unterminated line')
1123                 if regexp.compile('\s*\#').match(l): continue
1124                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1125                 initmat = initre.match(l)
1126                 if initmat:
1127                         (fname, l) = initmat.groups()
1128                         fname = string.capwords(fname)
1129                         if stz is None:
1130                                 stz = { ' lno': lno, ' tests': [] }
1131                         if not stz.has_key(fname): stz[fname] = [ ]
1132                         hcurrent = stz[fname]
1133                 elif regexp.compile('\s').match(l):
1134                         if not hcurrent: badctrl('unexpected continuation')
1135                 else:
1136                         badctrl('syntax error')
1137                 hcurrent.append((lno, l))
1138         end_stanza(stz)
1139
1140         def testbadctrl(stz, lno, m):
1141                 report_badctrl(lno, m)
1142                 stz[' errs'] += 1
1143
1144         for stz in stanzas:
1145                 try:
1146                         try: tnames = stz['Tests']
1147                         except KeyError:
1148                                 tnames = ['*']
1149                                 raise Unsupported(stz[' lno'],
1150                                         'no Tests field')
1151                         tnames = map((lambda lt: lt[1]), tnames)
1152                         tnames = string.join(tnames).split()
1153                         base = {
1154                                 'restriction_names': [],
1155                                 'restrictions': [],
1156                                 'feature_names': [],
1157                                 'features': [],
1158                                 'testsdir': 'debian/tests',
1159                                 'depends' : '@'
1160                         }
1161                         for fname in stz.keys():
1162                                 if fname.startswith(' '): continue
1163                                 vl = stz[fname]
1164                                 try: fclass = globals()['Field_'+
1165                                         fname.replace('-','_')]
1166                                 except KeyError: raise Unsupported(vl[0][0],
1167                                         'unknown metadata field %s' % fname)
1168                                 f = fclass(stz, fname, base, tnames, vl)
1169                                 f.parse()
1170                         for tname in tnames:
1171                                 t = Test(tname, base, act)
1172                                 stz[' tests'].append(t)
1173                 except Unsupported, u:
1174                         for tname in tnames: u.report(tname)
1175                         continue
1176
1177         return stanzas
1178
1179 def print_exception(ei, msgprefix=''):
1180         if msgprefix: pstderr(msgprefix)
1181         (et, q, tb) = ei
1182         if et is Quit:
1183                 pstderr('adt-run: ' + q.m)
1184                 psummary('quitting: '+q.m)
1185                 return q.ec
1186         else:
1187                 pstderr("adt-run: unexpected, exceptional, error:")
1188                 psummary('quitting: unexpected error, consult transcript')
1189                 traceback.print_exc(None, sys.stderr)
1190                 if trace_stream is not None:
1191                         traceback.print_exc(None, trace_stream)
1192                 return 20
1193
1194 def cleanup():
1195         global trace_stream
1196         try:
1197                 rm_ec = 0
1198                 if opts.tmpdir is None and tmpdir is not None:
1199                         rmtree('tmpdir', tmpdir)
1200                 if testbed is not None:
1201                         testbed.reset_apt()
1202                         testbed.stop()
1203                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1204                 if trace_stream is not None:
1205                         trace_stream.close()
1206                         trace_stream = None
1207         except:
1208                 print_exception(sys.exc_info(),
1209                         '\nadt-run: error cleaning up:\n')
1210                 os._exit(20)
1211
1212 #---------- registration, installation etc. of .deb's: Binaries
1213
1214 def determine_package(act):
1215         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1216         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1217         if rc: badpkg('failed to parse binary package, code %d' % rc)
1218         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1219         act.pkg = None
1220         for l in output.split('\n'):
1221                 m = re.match(l)
1222                 if not m: continue
1223                 if act.pkg: badpkg('two Package: lines in control file')
1224                 act.pkg = m.groups()[0]
1225         if not act.pkg: badpkg('no good Package: line in control file')
1226
1227 class Binaries:
1228  def __init__(b):
1229         b.dir = TemporaryDir('binaries')
1230         b.dir.write()
1231         ok = False
1232
1233         if opts.gnupghome is None:
1234                 opts.gnupghome = tmpdir+'/gnupg'
1235
1236         b._debug('initialising')
1237         try:
1238                 for x in ['pubring','secring']:
1239                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1240                 ok = True
1241         except OSError, oe:
1242                 if oe.errno != errno.ENOENT: raise
1243
1244         if ok: b._debug('no key generation needed')
1245         else: b.genkey()
1246
1247  def _debug(b, s):
1248         debug('* '+s)
1249
1250  def genkey(b):
1251         b._debug('preparing for key generation')
1252
1253         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1254         mkdir_okexist(opts.gnupghome, 0700)
1255
1256         script = '''
1257   exec >&2
1258   cd "$1"
1259   cat <<"END" >key-gen-params
1260 Key-Type: DSA
1261 Key-Length: 1024
1262 Key-Usage: sign
1263 Name-Real: autopkgtest per-run key
1264 Name-Comment: do not trust this key
1265 Name-Email: autopkgtest@example.com
1266 END
1267   set -x
1268   gpg --homedir="$1" --batch --gen-key key-gen-params
1269                 '''
1270         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1271         rc = subprocess_cooked(cmdl, dbg=(genkey,script))[0]
1272         if rc:
1273                 try:
1274                         f = open(opts.gnupghome+'/key-gen-log')
1275                         tp = file.read()
1276                 except OSError, e: tp = e
1277                 pstderr(tp)
1278                 bomb('key generation failed, code %d' % rc)
1279
1280  def apt_configs(b):
1281         return {
1282                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1283         }
1284
1285  def apt_pkg_gdebi_script(b, arg, middle):
1286         script = [
1287                 'import apt_pkg',
1288                 'import urllib',
1289                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1290                 ]
1291         for (k,v) in b.apt_configs().iteritems():
1292                 v = urllib.quote(v)
1293                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1294                                 % (k, v))
1295         script += [
1296                 'from GDebi.Cache import Cache',
1297                 'cache = Cache()',
1298                 ]
1299         for m in middle:
1300                 script += m + [
1301                 'print res',
1302                 'print d.missingDeps',
1303                 'print d.requiredChanges',
1304                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1305                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1306                           'd._failureString)',
1307                 'cache.commit()',
1308                 ''
1309                 ]
1310         return '\n'.join(script)
1311  def apt_get(b):
1312         ag = ['apt-get','-qy']
1313         for kv in b.apt_configs().iteritems():
1314                 ag += ['-o', '%s=%s' % kv]
1315         return ag
1316
1317  def reset(b):
1318         b._debug('reset')
1319         rmtree('binaries', b.dir.read())
1320         b.dir.invalidate()
1321         b.dir.write()
1322         b.install = []
1323         b.blamed = []
1324         b.registered = set()
1325
1326  def register(b, act, pkg, af, forwhat, blamed):
1327         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1328                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1329
1330         if act.ah['deb_'+forwhat] == 'ignore': return
1331
1332         b.blamed += testbed.blamed
1333
1334         leafname = pkg+'.deb'
1335         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1336
1337         try: os.remove(dest.write())
1338         except OSError, oe:
1339                 if oe.errno != errno.ENOENT: raise e
1340
1341         try: os.link(af.read(), dest.write())
1342         except OSError, oe:
1343                 if oe.errno != errno.EXDEV: raise e
1344                 shutil.copy(af.read(), dest)
1345
1346         if act.ah['deb_'+forwhat] == 'install':
1347                 b.install.append(pkg)
1348
1349         b.registered.add(pkg)
1350
1351  def publish(b):
1352         b._debug('publish')
1353
1354         script = '''
1355   exec >&2
1356   cd "$1"
1357   apt-ftparchive packages . >Packages
1358   gzip <Packages >Packages.gz
1359   apt-ftparchive release . >Release
1360   rm -f Release.gpg
1361   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1362   gpg --homedir="$2" --batch --export >archive-key.pgp
1363                 '''
1364         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1365         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1366         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1367
1368         b.dir.invalidate(True)
1369         apt_source = b.dir.read(True)
1370
1371         so = TemporaryFile('vlds')
1372         script = '''
1373   exec 3>&1 >&2
1374   apt-key add archive-key.pgp
1375   echo "deb file://'''+apt_source+''' /" >sources.list
1376   cat /etc/apt/sources.list >>sources.list
1377   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1378     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1379   fi
1380   '''+ ' '.join(b.apt_get()) +''' update >&2
1381   cat /var/lib/dpkg/status >&3
1382                 '''
1383         testbed.mungeing_apt()
1384         rc = testbed.execute('apt-key', ['sh','-ec',script],
1385                                 so=so.write(True), cwd=b.dir.write(True),
1386                                 script=script)
1387         if rc: bomb('apt setup failed with exit code %d' % rc)
1388
1389         testbed.blamed += b.blamed
1390
1391         b._debug('publish reinstall checking...')
1392         pkgs_reinstall = set()
1393         pkg = None
1394         for l in open(so.read()):
1395                 if l.startswith('Package: '):
1396                         pkg = l[9:].rstrip()
1397                 elif l.startswith('Status: install '):
1398                         if pkg in b.registered:
1399                                 pkgs_reinstall.add(pkg)
1400                                 b._debug(' publish reinstall needs '+pkg)
1401
1402         if pkgs_reinstall:
1403                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1404                 what = 'apt-get-reinstall'
1405                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1406                         [pkg for pkg in pkgs_reinstall])
1407                 rc = testbed.execute(what, cmdl, script=None)
1408                 if rc: badpkg("installation of basic binarries failed,"
1409                                 " exit code %d" % rc)
1410
1411         b._debug('publish install...')
1412         for pkg in b.install:
1413                 what = 'apt-get-install-%s' % pkg
1414                 testbed.blame(pkg)
1415                 cmdl = b.apt_get() + ['install',pkg]
1416                 rc = testbed.execute(what, cmdl, script=None)
1417                 if rc: badpkg("installation of %s failed, exit code %d"
1418                                 % (pkg, rc))
1419
1420         b._debug('publish done')
1421
1422 #---------- processing of sources (building)
1423
1424 def source_rules_command(act,script,what,which,work,cwd,
1425                                 results_lines=0,xargs=[]):
1426         script = [      "exec 3>&1 >&2",
1427                         "set -x"        ] + script
1428         script = '\n'.join(script)
1429         so = TemporaryFile('%s-%s-results' % (what,which))
1430         rc = testbed.execute('%s-%s' % (what,which),
1431                         ['sh','-ec',script]+xargs, script=script,
1432                         so=so.write(True), cwd=cwd)
1433         results = open(so.read()).read().rstrip('\n')
1434         if len(results): results = results.split("\n")
1435         else: results = []
1436         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1437         if results_lines is not None and len(results) != results_lines:
1438                 badpkg("got %d lines of results from %s where %d expected"
1439                         % (len(results), which, results_lines))
1440         if results_lines==1: return results[0]
1441         return results
1442
1443 def build_source(act, control_override):
1444         act.blame = 'arg:'+act.af.spec
1445         testbed.blame(act.blame)
1446         testbed.prepare1([])
1447         testbed.needs_reset()
1448
1449         what = act.what
1450         basename = act.af.spec
1451         debiancontrol = None
1452         act.binaries = []
1453
1454         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1455
1456         if act.kind == 'dsc':
1457                 dsc = act.af
1458                 dsc_file = open(dsc.read())
1459                 in_files = False
1460                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1461                 for l in dsc_file:
1462                         l = l.rstrip('\n')
1463                         if l.startswith('Files:'): in_files = True; continue
1464                         elif l.startswith('#'): pass
1465                         elif not l.startswith(' '):
1466                                 in_files = False
1467                                 if l.startswith('Source:'):
1468                                         act.blame = 'dsc:'+l[7:].strip()
1469                                         testbed.blame(act.blame)
1470                         if not in_files: continue
1471
1472                         m = fre.match(l)
1473                         if not m: badpkg(".dsc contains unparseable line"
1474                                         " in Files: `%s'" % l)
1475                         leaf = m.groups(0)[0]
1476                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1477                                         sibling=True)
1478                         subfile.read(True)
1479                 dsc.read(True)
1480
1481         if act.kind == 'ubtree':
1482                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1483                         act.af, 'debian/control')
1484                 dsc = TemporaryFile(what+'-fakedsc')
1485                 dsc_w = open(dsc.write(), 'w')
1486                 for l in open(debiancontrol.read()):
1487                         l = l.rstrip('\n')
1488                         if not len(l): break
1489                         print >>dsc_w, l
1490                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1491                 dsc_w.close()
1492
1493         if act.kind == 'dsc':
1494                 testbed.prepare2([])
1495                 script = binaries.apt_pkg_gdebi_script('', [[
1496                                 'from GDebi.DebPackage import DebPackage',
1497                                 'd = DebPackage(cache)',
1498                                 'res = d.satisfyDependsStr("dpkg-dev")',
1499                         ]])
1500                 cmdl = ['python','-c',script]
1501                 whatp = what+'-dpkgsource'
1502                 rc = testbed.execute(what, cmdl, script=script)
1503                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1504
1505         work = TemporaryDir(what+'-build')
1506         act.work = work
1507
1508         tmpdir = work.write(True)+'/tmpdir'
1509         tmpdir_script = [
1510                         'TMPDIR="$1"',
1511                         'rm -rf -- "$TMPDIR"',
1512                         'export TMPDIR',
1513                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1514                 ]
1515
1516         if act.kind == 'ubtree':
1517                 spec = '%s/real-tree' % work.write(True)
1518                 create_command = '''
1519                         rm -rf "$spec"
1520                         mkdir "$spec"
1521                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1522                         '''
1523                 initcwd = act.af.read(True)
1524
1525         if act.kind == 'dsc':
1526                 spec = dsc.read(True)
1527                 create_command = '''
1528                         dpkg-source -x $spec
1529                         '''
1530                 initcwd = work.write(True)
1531
1532         script = [
1533                 'spec="$2"',
1534                 'origpwd=`pwd`',
1535                 'cd '+work.write(True)
1536         ]
1537
1538         if opts.user:
1539                 script += ([ 'chown '+opts.user+' .' ] +
1540                         tmpdir_script +
1541                         [ 'spec="$spec" origpwd="$origpwd" '
1542                                 +opts.user_wrap(create_command) ])
1543         else:
1544                 script += (tmpdir_script +
1545                         [ create_command ])
1546
1547         script += [
1548                         'cd */.',
1549                         'pwd >&3',
1550                         'set +e; test -f debian/tests/control; echo $? >&3'
1551                 ]
1552         (result_pwd, control_test_rc) = source_rules_command(
1553                         act,script,what,'extract',work,
1554                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1555
1556         filter = act.ah['dsc_filter']
1557
1558         if control_test_rc == '1': act.missing_tests_control = True
1559
1560         # For optional builds:
1561         #
1562         # We might need to build the package because:
1563         #   - we want its binaries (filter isn't _ and at least one of the
1564         #       deb_... isn't ignore)
1565         #   - the test control file says so
1566         #       (assuming we have any tests)
1567
1568         class NeedBuildException: pass
1569         def build_needed(m):
1570                 debug_b('build needed for %s' % m)
1571                 raise NeedBuildException()
1572
1573         try:
1574                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1575                                       act.ah['deb_fortests'] != 'ignore'):
1576                         build_needed('binaries')
1577
1578                 result_pwd_af = InputDir(what+'-treeforcontrol',
1579                         result_pwd, True)
1580                 stanzas = read_control(act, result_pwd_af, control_override)
1581                 for stanza in stanzas:
1582                         for t in stanza[' tests']:
1583                                 if 'no-build-needed' not in t.feature_names:
1584                                         build_needed('test %s' % t.tname)
1585                                 for d in t.depends:
1586                                         if '@' in d:
1587                                                 build_needed('test %s '
1588                                                  'dependency %s' % (t.tname,d))
1589
1590                 debug_b('build not needed')
1591                 built = False
1592
1593         except NeedBuildException:
1594
1595                 if act.kind != 'dsc':
1596                         testbed.prepare2([])
1597
1598                 script = binaries.apt_pkg_gdebi_script(
1599                         dsc.read(True), [[
1600                         'from GDebi.DscSrcPackage import DscSrcPackage',
1601                         'd = DscSrcPackage(cache, arg)',
1602                         'res = d.checkDeb()',
1603                          ],[
1604                         'from GDebi.DebPackage import DebPackage',
1605                         'd = DebPackage(cache)',
1606                         'res = d.satisfyDependsStr("'+
1607                                         ','.join(build_essential)+
1608                                 '")',
1609                         ]])
1610
1611                 cmdl = ['python','-c',script]
1612                 whatp = what+'-builddeps'
1613                 rc = testbed.execute(what, cmdl, script=script)
1614                 if rc: badpkg('build-depends install failed,'
1615                               ' exit code %d' % rc)
1616
1617                 script = tmpdir_script + [
1618                         'cd "$2"',
1619                         'dpkg-checkbuilddeps',
1620                         opts.user_wrap('debian/rules build'),
1621                         ]
1622                 source_rules_command(act,script,what,'build',work,
1623                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1624
1625                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1626                         badpkg("results dir `%s' is not in expected parent"
1627                                " dir `%s'" % (result_pwd, work.read(True)))
1628
1629                 built = True
1630
1631         act.tests_tree = InputDir(what+'-tests-tree',
1632                                 work.read(True)+os.path.basename(result_pwd),
1633                                 True)
1634         if act.ah['dsc_tests']:
1635                 testbed.register_ephemeral(act.work)
1636                 testbed.register_ephemeral(act.tests_tree)
1637
1638         if not built:
1639                 act.blamed = []
1640                 return
1641
1642         act.blamed = copy.copy(testbed.blamed)
1643
1644         debug_b('filter=%s' % filter)
1645         if filter != '_':
1646                 script = tmpdir_script + [
1647                         'cd '+work.write(True)+'/*/.',
1648                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1649                         'cd ..',
1650                         'echo *.deb >&3',
1651                         ]
1652                 result_debs = source_rules_command(act,script,what,
1653                                 'binary',work,work.write(True),
1654                                 results_lines=1, xargs=['x',tmpdir])
1655                 if result_debs == '*': debs = []
1656                 else: debs = result_debs.split(' ')
1657                 debug_b('debs='+`debs`)
1658                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1659                 for deb in debs:
1660                         m = re.match(deb)
1661                         if not m: badpkg("badly-named binary `%s'" % deb)
1662                         pkg = m.groups()[0]
1663                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1664                         for pat in filter.split(','):
1665                                 debug_b('  pat=%s' % pat)
1666                                 if not fnmatch.fnmatchcase(pkg,pat):
1667                                         debug_b('   no match')
1668                                         continue
1669                                 deb_what = pkg+'_'+what+'.deb'
1670                                 bin = RelativeInputFile(deb_what,work,deb,True)
1671                                 debug_b('  deb_what=%s, bin=%s' %
1672                                         (deb_what, str(bin)))
1673                                 binaries.register(act,pkg,bin,
1674                                         'forbuilds',testbed.blamed)
1675                                 act.binaries.append((pkg,bin))
1676                                 break
1677                 debug_b('all done.')
1678
1679 #---------- main processing loop and main program
1680
1681 def process_actions():
1682         global binaries
1683
1684         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1685         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1686         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1687
1688         debug_a1('starting')
1689         testbed.open()
1690         binaries = Binaries()
1691
1692         for act in opts.actions:
1693                 if not act.af.spec_tbp:
1694                         testbed.register_ephemeral(act.af)
1695
1696         binaries.reset()
1697         control_override = None
1698
1699         debug_a1('builds ...')
1700         for act in opts.actions:
1701                 debug_a2('%s %s' %
1702                         (act.kind, act.what))
1703
1704                 if act.kind == 'control':
1705                         control_override = act.af
1706                 if act.kind == 'deb':
1707                         testbed.blame('arg:'+act.af.spec)
1708                         determine_package(act)
1709                         testbed.blame('deb:'+act.pkg)
1710                         binaries.register(act,act.pkg,act.af,
1711                                 'forbuilds',testbed.blamed)
1712                 if act.kind == 'dsc' or act.kind == 'ubtree':
1713                         build_source(act, control_override)
1714                 if act.kind == 'tree':
1715                         act.binaries = []
1716                 if act.kind.endswith('tree') or act.kind == 'dsc':
1717                         control_override = None
1718
1719         debug_a1('builds done.')
1720
1721         binaries.reset()
1722         control_override = None
1723
1724         debug_a1('tests ...')
1725         for act in opts.actions:
1726                 debug_a2('test %s %s' % (act.kind, act.what))
1727
1728                 testbed.needs_reset()
1729                 if act.kind == 'control':
1730                         control_override = act.af
1731                 if act.kind == 'deb':
1732                         binaries.register(act,act.pkg,act.af,'fortests',
1733                                 ['deb:'+act.pkg])
1734                 if act.kind == 'dsc' or act.kind == 'ubtree':
1735                         for (pkg,bin) in act.binaries:
1736                                 binaries.register(act,pkg,bin,'fortests',
1737                                         act.blamed)
1738                 if act.kind == 'dsc':
1739                         if act.ah['dsc_tests']:
1740                                 debug_a3('read control ...')
1741                                 stanzas = read_control(act, act.tests_tree,
1742                                                 control_override)
1743                                 testbed.blamed += act.blamed
1744                                 debug_a3('run_tests ...')
1745                                 run_tests(stanzas, act.tests_tree)
1746                         control_override = None
1747                 if act.kind == 'tree' or act.kind == 'ubtree':
1748                         testbed.blame('arg:'+act.af.spec)
1749                         stanzas = read_control(act, act.af, control_override)
1750                         debug_a3('run_tests ...')
1751                         run_tests(stanzas, act.af)
1752                         control_override = None
1753         debug_a1('tests done.')
1754
1755 def main():
1756         global testbed
1757         global tmpdir
1758         try:
1759                 parse_args()
1760         except SystemExit, se:
1761                 os._exit(20)
1762         try:
1763                 setup_trace()
1764                 testbed = Testbed()
1765                 testbed.start()
1766                 finalise_options()
1767                 process_actions()
1768         except:
1769                 ec = print_exception(sys.exc_info(), '')
1770                 cleanup()
1771                 os._exit(ec)
1772         cleanup()
1773         os._exit(errorcode)
1774
1775 main()