runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':10, 'install':900, 'test':600, 'build':3000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51
  52 #---------- output handling
  53 #
  54 #  There are at least the following kinds of output:
  55 #
  56 #   1. stderr output which consists of
  57 #   1a. our errors
  58 #   1b. stuff printed to stderr by the virtualisation server
  59 #   1c. stuff printed to stderr by our short-lived subprocesseses
  60 #           which we don't expect to fail
  61 #
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from
  65 #        general scripts we run on the host
  66 #   2c. progress information and stderr output from things
  67 #        we run on the testbed including builds
  68 #   2d. stderr and stdout output from actual tests
  69 #
  70 #   xxxx
  71 #   3. actual test results (printed to our stdout)
  72 #
  73 # Cloning of 1a and 2a, where necessary, is done by us writing
  74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  75 # is done by forking off a copy of ourselves to do plumbing,
  76 # which copy we wait for at the appropriate point.
  77
  78 class DummyOpts:
  79  def __init__(do): do.debuglevel = 0
  80
  81 opts = DummyOpts()
  82 trace_stream = None
  83 summary_stream = None
  84
  85 def pstderr(m):
  86         print >>sys.stderr, m
  87         if trace_stream is not None: print >>trace_stream, m
  88
  89 def debug(m, minlevel=0):
  90         if opts.debuglevel < minlevel: return
  91         if opts.quiet and trace_stream is None: return
  92         p = 'adt-run: trace'
  93         if minlevel: p += `minlevel`
  94         p += ': '
  95         for l in m.rstrip('\n').split('\n'):
  96                 s = p + l
  97                 if not opts.quiet: print >>sys.stderr, s
  98                 if trace_stream is not None: print >>trace_stream, s
  99
 100 def debug_file(hp, minlevel=0):
 101         if opts.debuglevel < minlevel: return
 102         def do_copy(stream, what):
 103                 rc = subprocess.call(['cat',hp], stdout=stream)
 104                 if rc: bomb('cat failed copying data from %s'
 105                             ' to %s, exit code %d' % (hp, what, rc))
 106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 108
 109 class Errplumb:
 110  def __init__(ep, critical=False):
 111         to_stderr = critical or not opts.quiet
 112         count = to_stderr + (trace_stream is not None)
 113         if count == 0:
 114                 ep.stream = open('/dev/null','w')
 115                 ep._sp = None
 116         elif count == 1:
 117                 if to_stderr: ep.stream = os.dup(2)
 118                 else: ep.stream = trace_stream
 119                 ep._sp = None
 120         else:
 121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 122                         stdin=subprocess.PIPE, stdout=trace_stream,
 123                         close_fds=True)
 124                 ep.stream = ep._sp.stdin
 125  def wait(ep):
 126         if ep._sp is None: return
 127         if type(ep.stream) == type(2):
 128                 os.close(ep.stream)
 129                 ep.stream = ()
 130         ep._sp.stdin.close()
 131         rc = ep._sp.wait()
 132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 133         ep._sp = None
 134
 135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 136         if dbg is not None:
 137                 if isinstance(dbg,tuple): (what,script) = dbg
 138                 else: (what,script) = (dbg,None)
 139                 debug_subprocess(what, cmdl, script=script)
 140         ep = Errplumb(critical)
 141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 142         output = running.communicate()[0]
 143         rc = running.wait()
 144         ep.wait()
 145         return (rc, output)
 146
 147 def psummary(m):
 148         if summary_stream is not None: print >>summary_stream, m
 149
 150 def preport(m):
 151         print m
 152         sys.stdout.flush()
 153         if trace_stream is not None: print >>trace_stream, m
 154         psummary(m)
 155
 156 def report(tname, result):
 157         preport('%-20s %s' % (tname, result))
 158
 159 #---------- errors we define
 160
 161 class Quit:
 162         def __init__(q,ec,m): q.ec = ec; q.m = m
 163
 164 def bomb(m):
 165         raise Quit(20, "unexpected error: %s" % m)
 166
 167 def badpkg(m):
 168         preport('blame: ' + ' '.join(testbed.blamed))
 169         preport('badpkg: ' + m)
 170         raise Quit(12, "erroneous package: %s" % m)
 171
 172 class Unsupported:
 173  def __init__(u, lno, m):
 174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 175         else: u.m = m
 176  def report(u, tname):
 177         global errorcode
 178         errorcode != 2
 179         report(tname, 'SKIP %s' % u.m)
 180
 181 #---------- convenience function
 182
 183 def mkdir_okexist(pathname, mode=02755):
 184         try:
 185                 os.mkdir(pathname, mode)
 186         except (IOError,OSError), oe:
 187                 if oe.errno != errno.EEXIST: raise
 188
 189 def rmtree(what, pathname):
 190         debug('/ %s rmtree %s' % (what, pathname), 2)
 191         try: shutil.rmtree(pathname)
 192         except (IOError,OSError), oe:
 193                 if oe.errno != errno.EEXIST: raise
 194
 195 def debug_subprocess(what, cmdl=None, script=None):
 196         o = '$ '+what+':'
 197         if cmdl is not None:
 198                 ol = []
 199                 for x in cmdl:
 200                         if x is script: x = '<SCRIPT>'
 201                         ol.append(x.    replace('\\','\\\\').
 202                                         replace(' ','\\ ')      )
 203                 o += ' '+ ' '.join(ol)
 204         debug(o)
 205         if script is not None and opts.debuglevel >= 1:
 206                 o = ''
 207                 for l in script.rstrip('\n').split('\n'):
 208                         o += '$     '+l+'\n'
 209                 debug(o,1)
 210
 211 def flatten(l):
 212         return reduce((lambda a,b: a + b), l, [])
 213
 214 #---------- fancy automatic file-copying class
 215
 216 class AutoFile:
 217         # p.what
 218         # p.path[tb]    None or path    not None => path known
 219         # p.file[tb]    None or path    not None => file exists
 220         # p.spec        string or None
 221         # p.spec_tbp    True or False, or not set if spec is None
 222         # p.dir         '' or '/'
 223
 224  def __init__(p, what):
 225         p.what = what
 226         p.path = [None,None]
 227         p.file = [None,None]
 228         p.spec = None
 229         p.dir = ''
 230
 231  def __repr__(p):
 232         return "<AF@%s>" % p.__str__()
 233  def __str__(p):
 234         def ptbp(tbp):
 235                 if p.path[tbp] is None: return '-'+p.dir
 236                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 237                 else: return p.path[tbp]+p.dir+'!'
 238         out = p.what
 239         if p.spec is not None:
 240                 if not hasattr(p,'spec_tb'): out += '~'
 241                 elif p.spec_tbp: out += '#'
 242                 else: out += '='
 243                 out += p.spec
 244         out += ':'
 245         out += ptbp(False)
 246         out += '|'
 247         out += ptbp(True)
 248         return out
 249
 250  def _wrong(p, how):
 251         xtra = ''
 252         if p.spec is not None:
 253                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 254         raise ("internal error: %s (%s)" % (how, str(p)))
 255
 256  def _ensure_path(p, tbp):
 257         if p.path[tbp] is None:
 258                 if '/' in p.what:
 259                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 260                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 261                 if not tbp:
 262                         p.path[tbp] = tmpdir+'/'+p.what
 263                 else:
 264                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 265
 266  def write(p, tbp=False):
 267         p._debug('write %s...' % 'HT'[tbp])
 268         p._ensure_path(tbp)
 269
 270         if p.dir and not p.file[tbp]:
 271                 if not tbp:
 272                         p._debug('mkdir H')
 273                         mkdir_okexist(p.path[tbp])
 274                 else:
 275                         cmdl = ['sh','-ec',
 276                                 'test -d "$1" || mkdir -p "$1"',
 277                                 'x', p.path[tbp]]
 278                         tf_what = urllib.quote(p.what).replace('/','%2F')
 279                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 280                         if rc: bomb('failed to create directory %s' %
 281                                 p.path[tbp])
 282
 283         p.file[tbp] = p.path[tbp]
 284         return p.path[tbp]
 285
 286  def read(p, tbp=False):
 287         p._debug('read %s...' % 'HT'[tbp])
 288         p._ensure_path(tbp)
 289
 290         if p.file[tbp] is None:
 291                 if p.file[not tbp] is None:
 292                         p._wrong("requesting read but nonexistent")
 293                 cud = ['copyup','copydown'][tbp]
 294                 src = p.file[not tbp] + p.dir
 295                 dst = p.path[tbp] + p.dir
 296                 testbed.command(cud, (src, dst))
 297                 p.file[tbp] = p.path[tbp]
 298
 299         return p.file[tbp] + p.dir
 300
 301  def invalidate(p, tbp=False):
 302         p.file[tbp] = None
 303         p._debug('invalidated %s' % 'HT'[tbp])
 304
 305  def _debug(p, m):
 306         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 307
 308  def _constructed(p):
 309         p._debug('constructed: '+str(p))
 310         p._check()
 311
 312  def _check(p):
 313         for tbp in [False,True]:
 314          for pf in [p.path, p.file]:
 315                 if pf[tbp] is None: continue
 316                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 317                 if p.dir and pf[tbp].endswith('/'):
 318                         pf[tbp] = pf[tbp].rstrip('/')
 319                         if not pf[tbp]: pf[tbp] = '/'
 320                 if not p.dir and pf[tbp].endswith('/'):
 321                         bomb("directory `%s' specified for "
 322                              "non-directory %s" % (pf[tbp], p.what))
 323
 324  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 325         AutoFile.__init__(p,what)
 326         sh_on = ''; sh_sibl = ''
 327         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 328         if sibling: sh_sibl=' (sibling)'
 329         parent._debug('using as base: %s: %s%s%s' %
 330                         (str(parent), leaf, sh_on, sh_sibl))
 331         if not sibling and not parent.dir:
 332                 parent._wrong('asked for non-sibling relative path of non-dir')
 333         if sibling: trim = os.path.dirname
 334         else: trim = lambda x: x
 335         for tbp in [False,True]:
 336                 if parent.path[tbp] is None: continue
 337                 trimmed = trim(parent.path[tbp])
 338                 if trimmed: trimmed += '/'
 339                 p.path[tbp] = trimmed + leaf
 340                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 341                         p.file[tbp] = p.path[tbp]
 342
 343 class InputFile(AutoFile):
 344  def _init(p, what, spec, spec_tbp=False):
 345         AutoFile.__init__(p, what)
 346         p.spec = spec
 347         p.spec_tbp = spec_tbp
 348         p.path[spec_tbp] = spec
 349         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 350  def __init__(p, what, spec, spec_tbp=False):
 351         p._init(what,spec,spec_tbp)
 352         p._constructed()
 353
 354 class InputDir(InputFile):
 355  def __init__(p, what, spec, spec_tbp=False):
 356         InputFile._init(p,what,spec,spec_tbp)
 357         p.dir = '/'
 358         p._constructed()
 359
 360 class OutputFile(AutoFile):
 361  def _init(p, what, spec, spec_tbp=False):
 362         AutoFile.__init__(p, what)
 363         p.spec = spec
 364         p.spec_tbp = spec_tbp
 365         p.path[spec_tbp] = spec
 366  def __init__(p, what, spec, spec_tbp=False):
 367         p._init(what,spec,spec_tbp)
 368         p._constructed()
 369
 370 class OutputDir(OutputFile):
 371  def __init__(p, what, spec, spec_tbp=False):
 372         OutputFile._init(p,what,spec,spec_tbp)
 373         p.dir = '/'
 374         p._constructed()
 375
 376 class RelativeInputFile(AutoFile):
 377  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 378         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 379         p._constructed()
 380
 381 class RelativeOutputFile(AutoFile):
 382  def __init__(p, what, parent, leaf, sibling=False):
 383         p._relative_init(what, parent, leaf, None, False, sibling)
 384         p._constructed()
 385
 386 class TemporaryFile(AutoFile):
 387  def __init__(p, what):
 388         AutoFile.__init__(p, what)
 389         p._constructed()
 390
 391 class TemporaryDir(AutoFile):
 392  def __init__(p, what):
 393         AutoFile.__init__(p, what)
 394         p.dir = '/'
 395         p._constructed()
 396
 397 #---------- parsing and representation of the arguments
 398
 399 class Action:
 400  def __init__(a, kind, af, arghandling, what):
 401         # extra attributes get added during processing
 402         a.kind = kind
 403         a.af = af
 404         a.ah = arghandling
 405         a.what = what
 406         a.missing_tests_control = False
 407  def __repr__(a):
 408         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 409
 410 def parse_args():
 411         global opts
 412         global n_non_actions # argh, stupid python scoping rules
 413         usage = "%prog <options> --- <virt-server>..."
 414         parser = OptionParser(usage=usage)
 415         pa = parser.add_option
 416         pe = parser.add_option
 417
 418         arghandling = {
 419                 'dsc_tests': True,
 420                 'dsc_filter': '*',
 421                 'deb_forbuilds': 'auto',
 422                 'deb_fortests': 'auto',
 423                 'tb': False,
 424                 'override_control': None
 425         }
 426         initial_arghandling = arghandling.copy()
 427         n_non_actions = 0
 428
 429         #----------
 430         # actions (ie, test sets to run, sources to build, binaries to use):
 431
 432         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 433                 global n_non_actions
 434                 parser.largs.append((value,kindpath))
 435                 n_non_actions += not(is_act)
 436
 437         def pa_action(long, metavar, kindpath, help, is_act=True):
 438                 pa('','--'+long, action='callback', callback=cb_action,
 439                         nargs=1, type='string',
 440                         callback_args=(long,kindpath,is_act), help=help)
 441
 442         pa_action('built-tree',         'TREE', '@/',
 443                 help='run tests from build tree TREE')
 444
 445         pa_action('unbuilt-tree',       'TREE', '@//',
 446                 help='run tests from build tree TREE')
 447
 448         pa_action('source',             'DSC', '@.dsc',
 449                 help='build DSC and use its tests and/or'
 450                     ' generated binary packages')
 451
 452         pa_action('binary',             'DEB', '@.deb',
 453                help='use binary package DEB according'
 454                     ' to most recent --binaries-* settings')
 455
 456         def cb_actnoarg(op,optstr,value,parser, largsentry):
 457                 parser.largs.append(largsentry)
 458         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 459                 callback_args=((None, ('instantiate',)),),
 460                 help='instantiate testbed now (during testing phase)'
 461                      ' and install packages'
 462                      ' selected for automatic installation, even'
 463                      ' if this might apparently not be required otherwise')
 464
 465         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 466                help='run tests from control file CONTROL instead,'
 467                     ' (applies to next test suite only)')
 468
 469         #----------
 470         # argument handling settings (what ways to use action
 471         #  arguments, and pathname processing):
 472
 473         def cb_setah(option, opt_str, value, parser, toset,setval):
 474                 if type(setval) == list:
 475                         if not value in setval:
 476                                 parser.error('value for %s option (%s) is not '
 477                                  'one of the permitted values (%s)' %
 478                                  (value, opt_str, setval.join(' ')))
 479                 elif setval is not None:
 480                         value = setval
 481                 for v in toset:
 482                         arghandling[v] = value
 483                 parser.largs.append(arghandling.copy())
 484
 485         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 486                 type = metavar
 487                 if type is not None: type = 'string'
 488                 pa('',long, action='callback', callback=cb_setah,
 489                    callback_args=(affected,effect), **kwargs)
 490
 491         #---- paths: host or testbed:
 492         #
 493         pa_setah('--paths-testbed', ['tb'],True,
 494                 help='subsequent path specifications refer to the testbed')
 495         pa_setah('--paths-host', ['tb'],False,
 496                 help='subsequent path specifications refer to the host')
 497
 498         #---- source processing settings:
 499
 500         pa_setah('--sources-tests', ['dsc_tests'],True,
 501                 help='run tests from builds of subsequent sources')
 502         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 503                 help='do not run tests from builds of subsequent sources')
 504
 505         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 506                 type='string', metavar='PATTERN-LIST',
 507                 help='from subsequent sources, use binaries matching'
 508                      ' PATTERN-LIST (comma-separated glob patterns)'
 509                      ' according to most recent --binaries-* settings')
 510         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 511                 help='from subsequent sources, do not use any binaries')
 512
 513         #---- binary package processing settings:
 514
 515         def pa_setahbins(long,toset,how):
 516          pa_setah(long, toset,['ignore','auto','install'],
 517                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 518                 help=how+' ignore binaries, install them as needed'
 519                         ' for dependencies, or unconditionally install'
 520                         ' them, respectively')
 521         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 522         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 523         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 524
 525         #----------
 526         # general options:
 527
 528         def cb_vserv(op,optstr,value,parser):
 529                 parser.values.vserver = list(parser.rargs)
 530                 del parser.rargs[:]
 531
 532         def cb_path(op,optstr,value,parser, constructor,long,dir):
 533                 name = long.replace('-','_')
 534                 af = constructor(long, value,arghandling['tb'])
 535                 setattr(parser.values, name, af)
 536
 537         def pa_path(long, constructor, help, dir=False):
 538                 pa('','--'+long, action='callback', callback=cb_path,
 539                         callback_args=(constructor,long,dir),
 540                         nargs=1, type='string',
 541                         help=help, metavar='PATH')
 542
 543         pa_path('output-dir', OutputDir, dir=True,
 544                 help='write stderr/out files in PATH')
 545
 546         pa('','--tmp-dir',              type='string', dest='tmpdir',
 547                 help='write temporary files to TMPDIR, emptying it'
 548                      ' beforehand and leaving it behind at the end')
 549         pa('','--log-file',             type='string', dest='logfile',
 550                 help='write the log LOGFILE, emptying it beforehand,'
 551                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 552         pa('','--summary-file',         type='string', dest='summary',
 553                 help='write a summary report to SUMMARY,'
 554                      ' emptying it beforehand')
 555
 556         pa('','--user',                 type='string', dest='user',
 557                 help='run tests as USER (needs root on testbed)')
 558         pa('','--gain-root',            type='string', dest='gainroot',
 559                 help='prefix debian/rules binary with GAINROOT')
 560         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 561         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 562         pa('','--gnupg-home',           type='string', dest='gnupghome',
 563                 default='~/.autopkgtest/gpg',
 564                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 565                         " signing private apt archive);"
 566                         " `fresh' means generate new key each time.")
 567
 568         #----------
 569         # actual meat:
 570
 571         class SpecialOption(optparse.Option): pass
 572         vs_op = SpecialOption('','--VSERVER-DUMMY')
 573         vs_op.action = 'callback'
 574         vs_op.type = None
 575         vs_op.default = None
 576         vs_op.nargs = 0
 577         vs_op.callback = cb_vserv
 578         vs_op.callback_args = ( )
 579         vs_op.callback_kwargs = { }
 580         vs_op.help = 'introduces virtualisation server and args'
 581         vs_op._short_opts = []
 582         vs_op._long_opts = ['---']
 583
 584         pa(vs_op)
 585
 586         (opts,args) = parser.parse_args()
 587         if not hasattr(opts,'vserver'):
 588                 parser.error('you must specifiy --- <virt-server>...')
 589         if n_non_actions >= len(parser.largs):
 590                 parser.error('nothing to do specified')
 591
 592         arghandling = initial_arghandling
 593         opts.actions = []
 594         ix = 0
 595         for act in args:
 596                 if type(act) == dict:
 597                         arghandling = act
 598                         continue
 599                 elif type(act) == tuple:
 600                         pass
 601                 elif type(act) == str:
 602                         act = (act,act)
 603                 else:
 604                         raise ("unknown action in list `%s' having"
 605                               " type `%s'" % (act, type(act)))
 606                 (pathstr, kindpath) = act
 607
 608                 constructor = InputFile
 609                 if type(kindpath) is tuple:             kind = kindpath[0]
 610                 elif kindpath.endswith('.deb'):         kind = 'deb'
 611                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 612                 elif kindpath.endswith('//'):
 613                         kind = 'ubtree'
 614                         constructor = InputDir
 615                 elif kindpath.endswith('/'):
 616                         kind = 'tree'
 617                         constructor = InputDir
 618                 else: parser.error("do not know how to handle filename `%s';"
 619                         " specify --source --binary or --build-tree" %
 620                         kindpath)
 621
 622                 what = '%s%s' % (kind,ix); ix += 1
 623
 624                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 625                 else: fwhatx = '-'+kind
 626
 627                 if pathstr is None: af = None
 628                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 629                 opts.actions.append(Action(kind, af, arghandling, what))
 630
 631 def setup_trace():
 632         global trace_stream, tmpdir, summary_stream
 633
 634         if opts.tmpdir is not None:
 635                 rmtree('tmpdir(specified)',opts.tmpdir)
 636                 mkdir_okexist(opts.tmpdir, 0700)
 637                 tmpdir = opts.tmpdir
 638         else:
 639                 assert(tmpdir is None)
 640                 tmpdir = tempfile.mkdtemp()
 641
 642         if opts.logfile is None:
 643                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 644                         opts.logfile = opts.output_dir.spec + '/log'
 645                 elif opts.tmpdir is not None:
 646                         opts.logfile = opts.tmpdir + '/log'
 647         if opts.logfile is not None:
 648                 trace_stream = open(opts.logfile, 'w', 0)
 649         if opts.summary is not None:
 650                 summary_stream = open(opts.summary, 'w', 0)
 651
 652         debug('options: '+`opts`, 1)
 653
 654 def finalise_options():
 655         global opts, tb, build_essential
 656
 657         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 658                 opts.user = ''
 659
 660         if opts.user is None:
 661                 su = 'suggested-normal-user='
 662                 ul = [
 663                         e[len(su):]
 664                         for e in testbed.caps
 665                         if e.startswith(su)
 666                         ]
 667                 if ul:
 668                         opts.user = ul[0]
 669                 else:
 670                         opts.user = ''
 671
 672         if opts.user:
 673                 if 'root-on-testbed' not in testbed.caps:
 674                         pstderr("warning: virtualisation"
 675                                 " system does not offer root on testbed,"
 676                                 " but --user option specified: failure likely")
 677                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 678         else:
 679                 opts.user_wrap = lambda x: x
 680
 681         if opts.gainroot is None:
 682                 opts.gainroot = ''
 683                 if (opts.user or
 684                     'root-on-testbed' not in testbed.caps):
 685                         opts.gainroot = 'fakeroot'
 686                         build_essential += ['fakeroot']
 687
 688         if opts.gnupghome.startswith('~/'):
 689                 try: home = os.environ['HOME']
 690                 except KeyError:
 691                         parser.error("HOME environment variable"
 692                                 " not set, needed for --gnupghome=`%s"
 693                                 % opts.gnupghome)
 694                 opts.gnupghome = home + opts.gnupghome[1:]
 695         elif opts.gnupghome == 'fresh':
 696                 opts.gnupghome = None
 697
 698 #---------- testbed management - the Testbed class
 699
 700 class Testbed:
 701  def __init__(tb):
 702         tb.sp = None
 703         tb.lastsend = None
 704         tb.scratch = None
 705         tb.modified = False
 706         tb.blamed = []
 707         tb._ephemeral = []
 708         tb._debug('init')
 709         tb._need_reset_apt = False
 710  def _debug(tb, m, minlevel=0):
 711         debug('** '+m, minlevel)
 712  def start(tb):
 713         tb._debug('start')
 714         p = subprocess.PIPE
 715         debug_subprocess('vserver', opts.vserver)
 716         tb._errplumb = Errplumb(True)
 717         tb.sp = subprocess.Popen(opts.vserver,
 718                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 719         tb.expect('ok')
 720         tb.caps = tb.commandr('capabilities')
 721  def stop(tb):
 722         tb._debug('stop')
 723         tb.close()
 724         if tb.sp is None: return
 725         ec = tb.sp.returncode
 726         if ec is None:
 727                 tb.sp.stdout.close()
 728                 tb.send('quit')
 729                 tb.sp.stdin.close()
 730                 ec = tb.sp.wait()
 731         if ec:
 732                 tb.bomb('testbed gave exit status %d after quit' % ec)
 733         tb._errplumb.wait()
 734  def open(tb):
 735         tb._debug('open, scratch=%s' % tb.scratch)
 736         if tb.scratch is not None: return
 737         pl = tb.commandr('open')
 738         tb.scratch = InputDir('tb-scratch', pl[0], True)
 739         tb.deps_processed = []
 740  def mungeing_apt(tb):
 741         if not 'revert' in tb.caps:
 742                 tb._need_reset_apt = True
 743  def reset_apt(tb):
 744         if not tb._need_reset_apt: return
 745         what = 'aptget-update-reset'
 746         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 747         rc = tb.execute(what, cmdl, kind='install')
 748         if rc:
 749                 pstderr("\n" "warning: failed to restore"
 750                         " testbed apt cache, exit code %d" % rc)
 751         tb._need_reset_apt = False
 752  def close(tb):
 753         tb._debug('close, scratch=%s' % tb.scratch)
 754         if tb.scratch is None: return
 755         tb.scratch = None
 756         if tb.sp is None: return
 757         tb.command('close')
 758  def prepare1(tb, deps_new):
 759         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 760                 (tb.modified, tb.deps_processed, deps_new), 1)
 761         if 'revert' in tb.caps and (tb.modified or
 762             [d for d in tb.deps_processed if d not in deps_new]):
 763                 for af in tb._ephemeral: af.read(False)
 764                 tb._debug('reset **')
 765                 tb.command('revert')
 766                 tb.blamed = []
 767                 for af in tb._ephemeral: af.invalidate(True)
 768         tb.modified = False
 769  def prepare2(tb, deps_new):
 770         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 771         binaries.publish()
 772         tb._install_deps(deps_new)
 773  def prepare(tb, deps_new):
 774         tb.prepare1(deps_new)
 775         tb.prepare2(deps_new)
 776  def register_ephemeral(tb, af):
 777         tb._ephemeral.append(af)
 778  def _install_deps(tb, deps_new):
 779         tb._debug(' installing dependencies '+`deps_new`, 1)
 780         tb.deps_processed = deps_new
 781         if not deps_new: return
 782         dstr = ', '.join(deps_new)
 783         script = binaries.apt_pkg_gdebi_script(
 784                 dstr, [[
 785                 'from GDebi.DebPackage import DebPackage',
 786                 'd = DebPackage(cache)',
 787                 'res = d.satisfyDependsStr(arg)',
 788                 ]])
 789         cmdl = ['python','-c',script]
 790         what = 'install-deps'
 791         rc = testbed.execute(what+'-debinstall', cmdl, script=script,
 792                                 kind='install')
 793         if rc: badpkg('dependency install failed, exit code %d' % rc)
 794  def needs_reset(tb):
 795         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 796         tb.modified = True
 797  def blame(tb, m):
 798         tb._debug('blame += %s' % m, 1)
 799         tb.blamed.append(m)
 800  def bomb(tb, m):
 801         tb._debug('bomb %s' % m)
 802         if tb.sp is not None:
 803                 tb.sp.stdout.close()
 804                 tb.sp.stdin.close()
 805                 ec = tb.sp.wait()
 806                 if ec: pstderr('adt-run: testbed failing,'
 807                                 ' exit status %d' % ec)
 808         tb.sp = None
 809         raise Quit(16, 'testbed failed: %s' % m)
 810  def send(tb, string):
 811         tb.sp.stdin
 812         try:
 813                 debug('>> '+string, 2)
 814                 print >>tb.sp.stdin, string
 815                 tb.sp.stdin.flush()
 816                 tb.lastsend = string
 817         except:
 818                 (type, value, dummy) = sys.exc_info()
 819                 tb.bomb('cannot send to testbed: %s' % traceback.
 820                         format_exception_only(type, value))
 821  def expect(tb, keyword, nresults=None):
 822         l = tb.sp.stdout.readline()
 823         if not l: tb.bomb('unexpected eof from the testbed')
 824         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 825         l = l.rstrip('\n')
 826         debug('<< '+l, 2)
 827         ll = l.split()
 828         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 829         if ll[0] != keyword:
 830                 if tb.lastsend is None:
 831                         tb.bomb("got banner `%s', expected `%s...'" %
 832                                 (l, keyword))
 833                 else:
 834                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 835                                 (tb.lastsend, l, keyword))
 836         ll = ll[1:]
 837         if nresults is not None and len(ll) != nresults:
 838                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 839                         " expected %d result parameters" %
 840                         (string, l, len(ll), nresults))
 841         return ll
 842  def commandr(tb, cmd, args=(), nresults=None):
 843         # pass args=[None,...] or =(None,...) to avoid more url quoting
 844         if type(cmd) is str: cmd = [cmd]
 845         if len(args) and args[0] is None: args = args[1:]
 846         else: args = map(urllib.quote, args)
 847         al = cmd + args
 848         tb.send(string.join(al))
 849         ll = tb.expect('ok', nresults)
 850         rl = map(urllib.unquote, ll)
 851         return rl
 852  def command(tb, cmd, args=()):
 853         tb.commandr(cmd, args, 0)
 854  def commandr1(tb, cmd, args=()):
 855         rl = tb.commandr(cmd, args, 1)
 856         return rl[0]
 857  def execute(tb, what, cmdl,
 858                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 859                 script=False, tmpdir=None, kind='short'):
 860         # Options for script:
 861         #   False - do not call debug_subprocess, no synch. reporting required
 862         #   None or string - call debug_subprocess with that value,
 863         #                       plumb stderr through synchronously if possible
 864         # Options for se:
 865         #   None - usual Errplumb (output is of kind 2c)
 866         #   string - path on testbed (output is of kind 2d)
 867
 868         timeout = timeouts[kind]
 869
 870         if script is not False: debug_subprocess(what, cmdl, script=script)
 871         if cwd is None: cwd = tb.scratch.write(True)
 872
 873         xdump = None
 874         if se is None:
 875                 ep = Errplumb()
 876                 se_catch = TemporaryFile(what+'-xerr')
 877                 se_use = se_catch.write(True)
 878                 if not opts.quiet: xdump = 'debug=2-2'
 879                 elif trace_stream is not None:
 880                         xdump = 'debug=2-%d' % trace_stream.fileno()
 881         else:
 882                 ep = None
 883                 se_catch = None
 884                 se_use = se
 885
 886         cmdl = [None,
 887                 ','.join(map(urllib.quote, cmdl)),
 888                 si, so, se_use, cwd]
 889
 890         if timeout is not None and timeout > 0:
 891                 cmdl.append('timeout=%d' % timeout)
 892
 893         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 894         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 895         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 896 #       cmdl.append('env=LANG=C')
 897
 898         rc = tb.commandr1('execute', cmdl)
 899         try: rc = int(rc)
 900         except ValueError: bomb("execute for %s gave invalid response `%s'"
 901                                         % (what,rc))
 902
 903         if se_catch is not None:
 904                 debug_file(se_catch.read())
 905         if ep is not None:
 906                 ep.wait()
 907
 908         return rc
 909
 910 #---------- representation of test control files: Field*, Test, etc.
 911
 912 class FieldBase:
 913  def __init__(f, fname, stz, base, tnames, vl):
 914         assert(vl)
 915         f.stz = stz
 916         f.base = base
 917         f.tnames = tnames
 918         f.vl = vl
 919  def words(f):
 920         def distribute(vle):
 921                 (lno, v) = vle
 922                 r = v.split()
 923                 r = map((lambda w: (lno, w)), r)
 924                 return r
 925         return flatten(map(distribute, f.vl))
 926  def atmostone(f):
 927         if len(vl) == 1:
 928                 (f.lno, f.v) = vl[0]
 929         else:
 930                 raise Unsupported(f.vl[1][0],
 931                         'only one %s field allowed' % fn)
 932         return f.v
 933
 934 class FieldIgnore(FieldBase):
 935  def parse(f): pass
 936
 937 class Restriction:
 938  def __init__(r,rname,base): pass
 939
 940 class Restriction_rw_build_tree(Restriction): pass
 941 class Restriction_breaks_testbed(Restriction):
 942  def __init__(r, rname, base):
 943         if 'revert' not in testbed.caps:
 944                 raise Unsupported(f.lno,
 945                         'Test breaks testbed but testbed cannot revert')
 946 class Restriction_needs_root(Restriction):
 947  def __init__(r, rname, base):
 948         if 'root-on-testbed' not in testbed.caps:
 949                 raise Unsupported(f.lno,
 950                         'Test needs root on testbed which is not available')
 951
 952 class Field_Restrictions(FieldBase):
 953  def parse(f):
 954         for wle in f.words():
 955                 (lno, rname) = wle
 956                 nrname = rname.replace('-','_')
 957                 try: rclass = globals()['Restriction_'+nrname]
 958                 except KeyError: raise Unsupported(lno,
 959                         'unknown restriction %s' % rname)
 960                 r = rclass(nrname, f.base)
 961                 f.base['restriction_names'].append(rname)
 962                 f.base['restrictions'].append(r)
 963
 964 class Field_Features(FieldIgnore):
 965  def parse(f):
 966         for wle in f.words():
 967                 (lno, fname) = wle
 968                 f.base['feature_names'].append(fname)
 969                 nfname = fname.replace('-','_')
 970                 try: fclass = globals()['Feature_'+nfname]
 971                 except KeyError: continue
 972                 ft = fclass(nfname, f.base)
 973                 f.base['features'].append(ft)
 974
 975 class Field_Tests(FieldIgnore): pass
 976
 977 class Field_Depends(FieldBase):
 978  def parse(f):
 979         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 980         dl = map(lambda x: x.strip(),
 981                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 982         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
 983         for d in dl:
 984                 if re.search(d):
 985                         badpkg("Test Depends field contains dependency"
 986                                " `%s' with invalid characters" % d)
 987         f.base['depends'] = dl
 988
 989 class Field_Tests_directory(FieldBase):
 990  def parse(f):
 991         td = atmostone(f)
 992         if td.startswith('/'): raise Unspported(f.lno,
 993                 'Tests-Directory may not be absolute')
 994         f.base['testsdir'] = td
 995
 996 def run_tests(stanzas, tree):
 997         global errorcode, testbed
 998         if stanzas == ():
 999                 report('*', 'SKIP no tests in this package')
1000                 errorcode |= 8
1001         for stanza in stanzas:
1002                 tests = stanza[' tests']
1003                 if not tests:
1004                         report('*', 'SKIP package has metadata but no tests')
1005                         errorcode |= 8
1006                 for t in tests:
1007                         t.prepare()
1008                         t.run(tree)
1009                         if 'breaks-testbed' in t.restriction_names:
1010                                 testbed.needs_reset()
1011                 testbed.needs_reset()
1012
1013 class Test:
1014  def __init__(t, tname, base, act):
1015         if '/' in tname: raise Unsupported(base[' lno'],
1016                 'test name may not contain / character')
1017         for k in base: setattr(t,k,base[k])
1018         t.tname = tname
1019         t.act = act
1020         t.what = act.what+'t-'+tname
1021         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1022         else: t.path = tname
1023         t._debug('constructed; path=%s' % t.path)
1024         t._debug(' .depends=%s' % t.depends)
1025  def _debug(t, m):
1026         debug('& %s: %s' % (t.what, m))
1027  def report(t, m):
1028         report(t.what, m)
1029  def reportfail(t, m):
1030         global errorcode
1031         errorcode |= 4
1032         report(t.what, 'FAIL ' + m)
1033  def prepare(t):
1034         t._debug('preparing')
1035         dn = []
1036         for d in t.depends:
1037                 t._debug(' processing dependency '+d)
1038                 if not '@' in d:
1039                         t._debug('  literal dependency '+d)
1040                         dn.append(d)
1041                 else:
1042                         for (pkg,bin) in t.act.binaries:
1043                                 d = d.replace('@',pkg)
1044                                 t._debug('  synthesised dependency '+d)
1045                                 dn.append(d)
1046         testbed.prepare(dn)
1047  def run(t, tree):
1048         t._debug('[----------------------------------------')
1049         def stdouterr(oe):
1050                 idstr = t.what + '-' + oe
1051                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1052                         use_dir = opts.output_dir
1053                 else:
1054                         use_dir = testbed.scratch
1055                 return RelativeOutputFile(idstr, use_dir, idstr)
1056
1057         if hasattr(t.act,'work'): t.act.work.write(True)
1058         tree.read(True)
1059
1060         af = RelativeInputFile(t.what, tree, t.path)
1061         so = stdouterr('stdout')
1062         se = stdouterr('stderr')
1063
1064         tf = af.read(True)
1065         tmpdir = None
1066
1067         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1068         if rc: bomb('failed to chmod +x %s' % tf)
1069
1070         if 'needs-root' not in t.restriction_names and opts.user is not None:
1071                 tfl = ['su',opts.user,'-c',tf]
1072                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1073                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1074                 if opts.user: script += '; chown %s "$1"' % opts.user
1075                 if 'rw-build-tree' in t.restriction_names:
1076                         script += '; chown -R %s "$2"' % opts.user
1077                 rc = testbed.execute('mktmpdir-'+t.what,
1078                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1079                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1080                                 % (tmpdir, rc))
1081         else:
1082                 tfl = [tf]
1083
1084         rc = testbed.execute('test-'+t.what, tfl,
1085                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1086                 tmpdir=tmpdir, kind='test')
1087
1088         so_read = so.read()
1089         se_read = se.read()
1090
1091         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1092         stab = os.stat(se_read)
1093         if stab.st_size != 0:
1094                 l = open(se_read).readline()
1095                 l = l.rstrip('\n \t\r')
1096                 if len(l) > 35: l = l[:35] + '...'
1097                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1098                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1099                 debug_file(se_read)
1100         elif rc != 0:
1101                 t.reportfail('non-zero exit status %d' % rc)
1102         else:
1103                 t.report('PASS')
1104
1105         stab = os.stat(so_read)
1106         if stab.st_size != 0:
1107                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1108                 debug_file(so_read)
1109
1110         t._debug('----------------------------------------]')
1111
1112 def read_control(act, tree, control_override):
1113         stanzas = [ ]
1114
1115         if control_override is not None:
1116                 control_af = control_override
1117                 testbed.blame('arg:'+control_override.spec)
1118         else:
1119                 if act.missing_tests_control:
1120                         return ()
1121                 control_af = RelativeInputFile(act.what+'-testcontrol',
1122                         tree, 'debian/tests/control')
1123         try:
1124                 control = open(control_af.read(), 'r')
1125         except (IOError,OSError), oe:
1126                 if oe[0] != errno.ENOENT: raise
1127                 return []
1128
1129         lno = 0
1130         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1131         stz = None      # stz[field_name][index] = (lno, value)
1132                         # special field names:
1133                         # stz[' lno'] = number
1134                         # stz[' tests'] = list of Test objects
1135         def end_stanza(stz):
1136                 if stz is None: return
1137                 stz[' errs'] = 0
1138                 stanzas.append(stz)
1139                 stz = None
1140                 hcurrent = None
1141
1142         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1143         while 1:
1144                 l = control.readline()
1145                 if not l: break
1146                 lno += 1
1147                 if not l.endswith('\n'): badctrl('unterminated line')
1148                 if regexp.compile('\s*\#').match(l): continue
1149                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1150                 initmat = initre.match(l)
1151                 if initmat:
1152                         (fname, l) = initmat.groups()
1153                         fname = string.capwords(fname)
1154                         if stz is None:
1155                                 stz = { ' lno': lno, ' tests': [] }
1156                         if not stz.has_key(fname): stz[fname] = [ ]
1157                         hcurrent = stz[fname]
1158                 elif regexp.compile('\s').match(l):
1159                         if not hcurrent: badctrl('unexpected continuation')
1160                 else:
1161                         badctrl('syntax error')
1162                 hcurrent.append((lno, l))
1163         end_stanza(stz)
1164
1165         def testbadctrl(stz, lno, m):
1166                 report_badctrl(lno, m)
1167                 stz[' errs'] += 1
1168
1169         for stz in stanzas:
1170                 try:
1171                         try: tnames = stz['Tests']
1172                         except KeyError:
1173                                 tnames = ['*']
1174                                 raise Unsupported(stz[' lno'],
1175                                         'no Tests field')
1176                         tnames = map((lambda lt: lt[1]), tnames)
1177                         tnames = string.join(tnames).split()
1178                         base = {
1179                                 'restriction_names': [],
1180                                 'restrictions': [],
1181                                 'feature_names': [],
1182                                 'features': [],
1183                                 'testsdir': 'debian/tests',
1184                                 'depends' : '@'
1185                         }
1186                         for fname in stz.keys():
1187                                 if fname.startswith(' '): continue
1188                                 vl = stz[fname]
1189                                 try: fclass = globals()['Field_'+
1190                                         fname.replace('-','_')]
1191                                 except KeyError: raise Unsupported(vl[0][0],
1192                                         'unknown metadata field %s' % fname)
1193                                 f = fclass(stz, fname, base, tnames, vl)
1194                                 f.parse()
1195                         for tname in tnames:
1196                                 t = Test(tname, base, act)
1197                                 stz[' tests'].append(t)
1198                 except Unsupported, u:
1199                         for tname in tnames: u.report(tname)
1200                         continue
1201
1202         return stanzas
1203
1204 def print_exception(ei, msgprefix=''):
1205         if msgprefix: pstderr(msgprefix)
1206         (et, q, tb) = ei
1207         if et is Quit:
1208                 pstderr('adt-run: ' + q.m)
1209                 psummary('quitting: '+q.m)
1210                 return q.ec
1211         else:
1212                 pstderr("adt-run: unexpected, exceptional, error:")
1213                 psummary('quitting: unexpected error, consult transcript')
1214                 traceback.print_exc(None, sys.stderr)
1215                 if trace_stream is not None:
1216                         traceback.print_exc(None, trace_stream)
1217                 return 20
1218
1219 def cleanup():
1220         global trace_stream
1221         try:
1222                 if testbed is not None:
1223                         testbed.reset_apt()
1224                         testbed.stop()
1225                 if opts.tmpdir is None and tmpdir is not None:
1226                         rmtree('tmpdir', tmpdir)
1227                 if trace_stream is not None:
1228                         trace_stream.close()
1229                         trace_stream = None
1230         except:
1231                 print_exception(sys.exc_info(),
1232                         '\nadt-run: error cleaning up:\n')
1233                 os._exit(20)
1234
1235 #---------- registration, installation etc. of .deb's: Binaries
1236
1237 def determine_package(act):
1238         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1239         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1240         if rc: badpkg('failed to parse binary package, code %d' % rc)
1241         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1242         act.pkg = None
1243         for l in output.split('\n'):
1244                 m = re.match(l)
1245                 if not m: continue
1246                 if act.pkg: badpkg('two Package: lines in control file')
1247                 act.pkg = m.groups()[0]
1248         if not act.pkg: badpkg('no good Package: line in control file')
1249
1250 class Binaries:
1251  def __init__(b):
1252         b.dir = TemporaryDir('binaries')
1253         b.dir.write()
1254         ok = False
1255
1256         if opts.gnupghome is None:
1257                 opts.gnupghome = tmpdir+'/gnupg'
1258
1259         b._debug('initialising')
1260         try:
1261                 for x in ['pubring','secring']:
1262                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1263                 ok = True
1264         except (IOError,OSError), oe:
1265                 if oe.errno != errno.ENOENT: raise
1266
1267         if ok: b._debug('no key generation needed')
1268         else: b.genkey()
1269
1270  def _debug(b, s):
1271         debug('* '+s)
1272
1273  def genkey(b):
1274         b._debug('preparing for key generation')
1275
1276         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1277         mkdir_okexist(opts.gnupghome, 0700)
1278
1279         script = '''
1280   exec >&2
1281   cd "$1"
1282   cat <<"END" >key-gen-params
1283 Key-Type: DSA
1284 Key-Length: 1024
1285 Key-Usage: sign
1286 Name-Real: autopkgtest per-run key
1287 Name-Comment: do not trust this key
1288 Name-Email: autopkgtest@example.com
1289 END
1290   set -x
1291   gpg --homedir="$1" --batch --gen-key key-gen-params
1292                 '''
1293         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1294         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1295         if rc: bomb('key generation failed, code %d' % rc)
1296
1297  def apt_configs(b):
1298         return {
1299                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1300         }
1301
1302  def apt_pkg_gdebi_script(b, arg, middle):
1303         script = [
1304                 'import apt_pkg',
1305                 'import urllib',
1306                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1307                 ]
1308         for (k,v) in b.apt_configs().iteritems():
1309                 v = urllib.quote(v)
1310                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1311                                 % (k, v))
1312         script += [
1313                 'from GDebi.Cache import Cache',
1314                 'cache = Cache()',
1315                 ]
1316         for m in middle:
1317                 script += m + [
1318                 'print res',
1319                 'print d.missingDeps',
1320                 'print d.requiredChanges',
1321                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1322                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1323                           'd._failureString)',
1324                 'cache.commit()',
1325                 ''
1326                 ]
1327         return '\n'.join(script)
1328  def apt_get(b):
1329         ag = ['apt-get','-qy']
1330         for kv in b.apt_configs().iteritems():
1331                 ag += ['-o', '%s=%s' % kv]
1332         return ' '.join(ag)
1333
1334  def reset(b):
1335         b._debug('reset')
1336         rmtree('binaries', b.dir.read())
1337         b.dir.invalidate()
1338         b.dir.write()
1339         b.install = []
1340         b.blamed = []
1341         b.registered = set()
1342
1343  def register(b, act, pkg, af, forwhat, blamed):
1344         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1345                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1346
1347         if act.ah['deb_'+forwhat] == 'ignore': return
1348
1349         b.blamed += testbed.blamed
1350
1351         leafname = pkg+'.deb'
1352         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1353
1354         try: os.remove(dest.write())
1355         except (IOError,OSError), oe:
1356                 if oe.errno != errno.ENOENT: raise e
1357
1358         try: os.link(af.read(), dest.write())
1359         except (IOError,OSError), oe:
1360                 if oe.errno != errno.EXDEV: raise e
1361                 shutil.copy(af.read(), dest)
1362
1363         if act.ah['deb_'+forwhat] == 'install':
1364                 b.install.append(pkg)
1365
1366         b.registered.add(pkg)
1367
1368  def publish(b):
1369         b._debug('publish')
1370
1371         script = '''
1372   exec >&2
1373   cd "$1"
1374   apt-ftparchive packages . >Packages
1375   gzip <Packages >Packages.gz
1376   apt-ftparchive release . >Release
1377   rm -f Release.gpg
1378   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1379   gpg --homedir="$2" --batch --export >archive-key.pgp
1380                 '''
1381         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1382         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1383         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1384
1385         b.dir.invalidate(True)
1386         apt_source = b.dir.read(True)
1387
1388         so = TemporaryFile('vlds')
1389         script = '''
1390   exec 3>&1 >&2
1391   apt-key add archive-key.pgp
1392   echo "deb file://'''+apt_source+''' /" >sources.list
1393   cat /etc/apt/sources.list >>sources.list
1394   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1395     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1396   fi
1397   '''+ b.apt_get() +''' update >&2
1398   cat /var/lib/dpkg/status >&3
1399                 '''
1400         testbed.mungeing_apt()
1401         rc = testbed.execute('apt-key', ['sh','-ec',script],
1402                                 so=so.write(True), cwd=b.dir.write(True),
1403                                 script=script, kind='install')
1404         if rc: bomb('apt setup failed with exit code %d' % rc)
1405
1406         testbed.blamed += b.blamed
1407
1408         b._debug('publish reinstall checking...')
1409         pkgs_reinstall = set()
1410         pkg = None
1411         for l in open(so.read()):
1412                 if l.startswith('Package: '):
1413                         pkg = l[9:].rstrip()
1414                 elif l.startswith('Status: install '):
1415                         if pkg in b.registered:
1416                                 pkgs_reinstall.add(pkg)
1417                                 b._debug(' publish reinstall needs '+pkg)
1418
1419         if pkgs_reinstall:
1420                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1421                 what = 'apt-get-reinstall'
1422                 cmdl = (b.apt_get() + ' --reinstall install '+
1423                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1424                 cmdl = ['sh','-c',cmdl]
1425                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1426                 if rc: badpkg("installation of basic binarries failed,"
1427                                 " exit code %d" % rc)
1428
1429         b._debug('publish install...')
1430         for pkg in b.install:
1431                 what = 'apt-get-install-%s' % pkg
1432                 testbed.blame(pkg)
1433                 cmdl = b.apt_get() + ' install ' + pkg + ' >&2'
1434                 cmdl = ['sh','-c',cmdl]
1435                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1436                 if rc: badpkg("installation of %s failed, exit code %d"
1437                                 % (pkg, rc))
1438
1439         b._debug('publish done')
1440
1441 #---------- processing of sources (building)
1442
1443 def source_rules_command(act,script,what,which,work,cwd,
1444                                 results_lines=0,xargs=[]):
1445         script = [      "exec 3>&1 >&2",
1446                         "set -x"        ] + script
1447         script = '\n'.join(script)
1448         so = TemporaryFile('%s-%s-results' % (what,which))
1449         rc = testbed.execute('%s-%s' % (what,which),
1450                         ['sh','-ec',script]+xargs, script=script,
1451                         so=so.write(True), cwd=cwd, kind='build')
1452         results = open(so.read()).read().rstrip('\n')
1453         if len(results): results = results.split("\n")
1454         else: results = []
1455         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1456         if results_lines is not None and len(results) != results_lines:
1457                 badpkg("got %d lines of results from %s where %d expected"
1458                         % (len(results), which, results_lines))
1459         if results_lines==1: return results[0]
1460         return results
1461
1462 def build_source(act, control_override):
1463         act.blame = 'arg:'+act.af.spec
1464         testbed.blame(act.blame)
1465         testbed.prepare1([])
1466         testbed.needs_reset()
1467
1468         what = act.what
1469         basename = act.af.spec
1470         debiancontrol = None
1471         act.binaries = []
1472
1473         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1474
1475         if act.kind == 'dsc':
1476                 dsc = act.af
1477                 dsc_file = open(dsc.read())
1478                 in_files = False
1479                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1480                 for l in dsc_file:
1481                         l = l.rstrip('\n')
1482                         if l.startswith('Files:'): in_files = True; continue
1483                         elif l.startswith('#'): pass
1484                         elif not l.startswith(' '):
1485                                 in_files = False
1486                                 if l.startswith('Source:'):
1487                                         act.blame = 'dsc:'+l[7:].strip()
1488                                         testbed.blame(act.blame)
1489                         if not in_files: continue
1490
1491                         m = fre.match(l)
1492                         if not m: badpkg(".dsc contains unparseable line"
1493                                         " in Files: `%s'" % l)
1494                         leaf = m.groups(0)[0]
1495                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1496                                         sibling=True)
1497                         subfile.read(True)
1498                 dsc.read(True)
1499
1500         if act.kind == 'ubtree':
1501                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1502                         act.af, 'debian/control')
1503                 dsc = TemporaryFile(what+'-fakedsc')
1504                 dsc_w = open(dsc.write(), 'w')
1505                 for l in open(debiancontrol.read()):
1506                         l = l.rstrip('\n')
1507                         if not len(l): break
1508                         print >>dsc_w, l
1509                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1510                 dsc_w.close()
1511
1512         if act.kind == 'dsc':
1513                 testbed.prepare2([])
1514                 script = binaries.apt_pkg_gdebi_script('', [[
1515                                 'from GDebi.DebPackage import DebPackage',
1516                                 'd = DebPackage(cache)',
1517                                 'res = d.satisfyDependsStr("dpkg-dev")',
1518                         ]])
1519                 cmdl = ['python','-c',script]
1520                 whatp = what+'-dpkgsource'
1521                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1522                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1523
1524         work = TemporaryDir(what+'-build')
1525         act.work = work
1526
1527         tmpdir = work.write(True)+'/tmpdir'
1528         tmpdir_script = [
1529                         'TMPDIR="$1"',
1530                         'rm -rf -- "$TMPDIR"',
1531                         'export TMPDIR',
1532                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1533                 ]
1534
1535         if act.kind == 'ubtree':
1536                 spec = '%s/real-tree' % work.write(True)
1537                 create_command = '''
1538                         rm -rf "$spec"
1539                         mkdir "$spec"
1540                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1541                         '''
1542                 initcwd = act.af.read(True)
1543
1544         if act.kind == 'dsc':
1545                 spec = dsc.read(True)
1546                 create_command = '''
1547                         dpkg-source -x $spec
1548                         '''
1549                 initcwd = work.write(True)
1550
1551         script = [
1552                 'spec="$2"',
1553                 'origpwd=`pwd`',
1554                 'cd '+work.write(True)
1555         ]
1556
1557         if opts.user:
1558                 script += ([ 'chown '+opts.user+' .' ] +
1559                         tmpdir_script +
1560                         [ 'spec="$spec" origpwd="$origpwd" '
1561                                 +opts.user_wrap(create_command) ])
1562         else:
1563                 script += (tmpdir_script +
1564                         [ create_command ])
1565
1566         script += [
1567                         'cd */.',
1568                         'pwd >&3',
1569                         'set +e; test -f debian/tests/control; echo $? >&3'
1570                 ]
1571         (result_pwd, control_test_rc) = source_rules_command(
1572                         act,script,what,'extract',work,
1573                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1574
1575         filter = act.ah['dsc_filter']
1576
1577         if control_test_rc == '1': act.missing_tests_control = True
1578
1579         # For optional builds:
1580         #
1581         # We might need to build the package because:
1582         #   - we want its binaries (filter isn't _ and at least one of the
1583         #       deb_... isn't ignore)
1584         #   - the test control file says so
1585         #       (assuming we have any tests)
1586
1587         class NeedBuildException: pass
1588         def build_needed(m):
1589                 debug_b('build needed for %s' % m)
1590                 raise NeedBuildException()
1591
1592         try:
1593                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1594                                       act.ah['deb_fortests'] != 'ignore'):
1595                         build_needed('binaries')
1596
1597                 result_pwd_af = InputDir(what+'-treeforcontrol',
1598                         result_pwd, True)
1599                 stanzas = read_control(act, result_pwd_af, control_override)
1600                 for stanza in stanzas:
1601                         for t in stanza[' tests']:
1602                                 if 'no-build-needed' not in t.feature_names:
1603                                         build_needed('test %s' % t.tname)
1604                                 for d in t.depends:
1605                                         if '@' in d:
1606                                                 build_needed('test %s '
1607                                                  'dependency %s' % (t.tname,d))
1608
1609                 debug_b('build not needed')
1610                 built = False
1611
1612         except NeedBuildException:
1613
1614                 if act.kind != 'dsc':
1615                         testbed.prepare2([])
1616
1617                 script = binaries.apt_pkg_gdebi_script(
1618                         dsc.read(True), [[
1619                         'from GDebi.DscSrcPackage import DscSrcPackage',
1620                         'd = DscSrcPackage(cache, arg)',
1621                         'res = d.checkDeb()',
1622                          ],[
1623                         'from GDebi.DebPackage import DebPackage',
1624                         'd = DebPackage(cache)',
1625                         'res = d.satisfyDependsStr("'+
1626                                         ','.join(build_essential)+
1627                                 '")',
1628                         ]])
1629
1630                 cmdl = ['python','-c',script]
1631                 whatp = what+'-builddeps'
1632                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1633                 if rc: badpkg('build-depends install failed,'
1634                               ' exit code %d' % rc)
1635
1636                 script = tmpdir_script + [
1637                         'cd "$2"',
1638                         'dpkg-checkbuilddeps',
1639                         opts.user_wrap('debian/rules build'),
1640                         ]
1641                 source_rules_command(act,script,what,'build',work,
1642                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1643
1644                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1645                         badpkg("results dir `%s' is not in expected parent"
1646                                " dir `%s'" % (result_pwd, work.read(True)))
1647
1648                 built = True
1649
1650         act.tests_tree = InputDir(what+'-tests-tree',
1651                                 work.read(True)+os.path.basename(result_pwd),
1652                                 True)
1653         if act.ah['dsc_tests']:
1654                 testbed.register_ephemeral(act.work)
1655                 testbed.register_ephemeral(act.tests_tree)
1656
1657         if not built:
1658                 act.blamed = []
1659                 return
1660
1661         act.blamed = copy.copy(testbed.blamed)
1662
1663         debug_b('filter=%s' % filter)
1664         if filter != '_':
1665                 script = tmpdir_script + [
1666                         'cd '+work.write(True)+'/*/.',
1667                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1668                         'cd ..',
1669                         'echo *.deb >&3',
1670                         ]
1671                 result_debs = source_rules_command(act,script,what,
1672                                 'binary',work,work.write(True),
1673                                 results_lines=1, xargs=['x',tmpdir])
1674                 if result_debs == '*': debs = []
1675                 else: debs = result_debs.split(' ')
1676                 debug_b('debs='+`debs`)
1677                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1678                 for deb in debs:
1679                         m = re.match(deb)
1680                         if not m: badpkg("badly-named binary `%s'" % deb)
1681                         pkg = m.groups()[0]
1682                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1683                         for pat in filter.split(','):
1684                                 debug_b('  pat=%s' % pat)
1685                                 if not fnmatch.fnmatchcase(pkg,pat):
1686                                         debug_b('   no match')
1687                                         continue
1688                                 deb_what = pkg+'_'+what+'.deb'
1689                                 bin = RelativeInputFile(deb_what,work,deb,True)
1690                                 debug_b('  deb_what=%s, bin=%s' %
1691                                         (deb_what, str(bin)))
1692                                 binaries.register(act,pkg,bin,
1693                                         'forbuilds',testbed.blamed)
1694                                 act.binaries.append((pkg,bin))
1695                                 break
1696                 debug_b('all done.')
1697
1698 #---------- main processing loop and main program
1699
1700 def process_actions():
1701         global binaries
1702
1703         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1704         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1705         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1706
1707         debug_a1('starting')
1708         testbed.open()
1709         binaries = Binaries()
1710
1711         for act in opts.actions:
1712                 if act.af is not None and not act.af.spec_tbp:
1713                         testbed.register_ephemeral(act.af)
1714
1715         binaries.reset()
1716         control_override = None
1717
1718         debug_a1('builds ...')
1719         for act in opts.actions:
1720                 debug_a2('%s %s' %
1721                         (act.kind, act.what))
1722
1723                 if act.kind == 'control':
1724                         control_override = act.af
1725                 if act.kind == 'deb':
1726                         testbed.blame('arg:'+act.af.spec)
1727                         determine_package(act)
1728                         testbed.blame('deb:'+act.pkg)
1729                         binaries.register(act,act.pkg,act.af,
1730                                 'forbuilds',testbed.blamed)
1731                 if act.kind == 'dsc' or act.kind == 'ubtree':
1732                         build_source(act, control_override)
1733                 if act.kind == 'tree':
1734                         act.binaries = []
1735                 if act.kind.endswith('tree') or act.kind == 'dsc':
1736                         control_override = None
1737                 if act.kind == 'instantiate':
1738                         pass
1739
1740         debug_a1('builds done.')
1741
1742         binaries.reset()
1743         control_override = None
1744
1745         debug_a1('tests ...')
1746         for act in opts.actions:
1747                 debug_a2('test %s %s' % (act.kind, act.what))
1748
1749                 testbed.needs_reset()
1750                 if act.kind == 'control':
1751                         control_override = act.af
1752                 if act.kind == 'deb':
1753                         binaries.register(act,act.pkg,act.af,'fortests',
1754                                 ['deb:'+act.pkg])
1755                 if act.kind == 'dsc' or act.kind == 'ubtree':
1756                         for (pkg,bin) in act.binaries:
1757                                 binaries.register(act,pkg,bin,'fortests',
1758                                         act.blamed)
1759                 if act.kind == 'dsc':
1760                         if act.ah['dsc_tests']:
1761                                 debug_a3('read control ...')
1762                                 stanzas = read_control(act, act.tests_tree,
1763                                                 control_override)
1764                                 testbed.blamed += act.blamed
1765                                 debug_a3('run_tests ...')
1766                                 run_tests(stanzas, act.tests_tree)
1767                         control_override = None
1768                 if act.kind == 'tree' or act.kind == 'ubtree':
1769                         testbed.blame('arg:'+act.af.spec)
1770                         stanzas = read_control(act, act.af, control_override)
1771                         debug_a3('run_tests ...')
1772                         run_tests(stanzas, act.af)
1773                         control_override = None
1774                 if act.kind == 'instantiate':
1775                         testbed.prepare([])
1776         debug_a1('tests done.')
1777
1778 def main():
1779         global testbed
1780         global tmpdir
1781         try:
1782                 parse_args()
1783         except SystemExit, se:
1784                 os._exit(20)
1785         try:
1786                 setup_trace()
1787                 testbed = Testbed()
1788                 testbed.start()
1789                 finalise_options()
1790                 process_actions()
1791         except:
1792                 ec = print_exception(sys.exc_info(), '')
1793                 cleanup()
1794                 os._exit(ec)
1795         cleanup()
1796         os._exit(errorcode)
1797
1798 main()