runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 binaries = None         # Binaries (.debs we have registered)
  49
  50 #---------- output handling
  51 #
  52 #  There are at least the following kinds of output:
  53 #
  54 #   1. stderr output which consists of
  55 #   1a. our errors
  56 #   1b. stuff printed to stderr by the virtualisation server
  57 #   1c. stuff printed to stderr by our short-lived subprocesseses
  58 #           which we don't expect to fail
  59 #
  60 #   2. trace information, which consists of
  61 #   2a. our trace information from calls to debug()
  62 #   2b. progress information and stderr output from
  63 #        general scripts we run on the host
  64 #   2c. progress information and stderr output from things
  65 #        we run on the testbed including builds
  66 #   2d. stderr and stdout output from actual tests
  67 #
  68 #   xxxx
  69 #   3. actual test results (printed to our stdout)
  70 #
  71 # Cloning of 1a and 2a, where necessary, is done by us writing
  72 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  73 # is done by forking off a copy of ourselves to do plumbing,
  74 # which copy we wait for at the appropriate point.
  75
  76 trace_stream = None
  77
  78 def pstderr(m):
  79         print >>sys.stderr, m
  80         if trace_stream is not None: print >>trace_stream, m
  81
  82 def debug(m):
  83         if not opts.debug and trace_stream is None: return
  84         for l in m.rstrip('\n').split('\n'):
  85                 s = 'atd-run: debug: ' + l
  86                 if opts.debug: print >>sys.stderr, s
  87                 if trace_stream is not None: print >>trace_stream, s
  88
  89 def debug_file(hp):
  90         def do_copy(stream, what):
  91                 rc = subprocess.call(['cat',hp], stdout=stream)
  92                 if rc: bomb('cat failed copying data from %s'
  93                             ' to %s, exit code %d' % (hp, what, rc))
  94         if opts.debug: do_copy(sys.stderr, 'stderr')
  95         if trace_stream is not None: do_copy(trace_stream, 'trace log')
  96
  97 class Errplumb:
  98  def __init__(ep, critical=False):
  99         to_stderr = critical or opts.debug
 100         count = to_stderr + (trace_stream is not None)
 101         if count == 0:
 102                 ep.stream = open('/dev/null','w')
 103                 ep._sp = None
 104         elif count == 1:
 105                 if to_stderr: ep.stream = sys.stderr
 106                 else: ep.stream = trace_stream
 107                 ep._sp = None
 108         else:
 109                 ep._sp = subprocess.Popen(['tee','/dev/stderr'],
 110                         stdin=subprocess.PIPE, stdout=trace_stream,
 111                         close_fds=True)
 112                 ep.stream = ep._sp.stdin
 113  def wait(ep):
 114         if ep._sp is None: return
 115         ep._sp.stdin.close()
 116         rc = ep._sp.wait()
 117         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 118         ep._sp = None
 119
 120 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 121         if dbg is not None:
 122                 if isinstance(dbg,tuple): (what,script) = dbg
 123                 else: (what,script) = (dbg,None)
 124                 debug_subprocess(what, cmdl, script=script)
 125         ep = Errplumb(critical)
 126         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 127         output = running.communicate()[0]
 128         rc = running.wait()
 129         ep.wait()
 130         return (rc, output)
 131
 132 def preport(m):
 133         print m
 134         if trace_stream is not None: print >>trace_stream, m
 135
 136 def report(tname, result):
 137         preport('%-20s %s' % (tname, result))
 138
 139 #---------- errors we define
 140
 141 class Quit:
 142         def __init__(q,ec,m): q.ec = ec; q.m = m
 143
 144 def bomb(m):
 145         raise Quit(20, "unexpected error: %s" % m)
 146
 147 def badpkg(m):
 148         preport('blame: ' + ' '.join(testbed.blamed))
 149         preport('badpkg: ' + m)
 150         raise Quit(12, "erroneous package: %s" % m)
 151
 152 class Unsupported:
 153  def __init__(u, lno, m):
 154         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 155         else: u.m = m
 156  def report(u, tname):
 157         global errorcode
 158         errorcode != 2
 159         report(tname, 'SKIP %s' % u.m)
 160
 161 #---------- convenience function
 162
 163 def mkdir_okexist(pathname, mode=02755):
 164         try:
 165                 os.mkdir(pathname, mode)
 166         except OSError, oe:
 167                 if oe.errno != errno.EEXIST: raise
 168
 169 def rmtree(what, pathname):
 170         debug('/ %s rmtree %s' % (what, pathname))
 171         shutil.rmtree(pathname)
 172
 173 def debug_subprocess(what, cmdl=None, script=None):
 174         o = '$ '+what+':'
 175         if cmdl is not None:
 176                 ol = []
 177                 for x in cmdl:
 178                         if x is script: x = '<SCRIPT>'
 179                         ol.append(x.    replace('\\','\\\\').
 180                                         replace(' ','\\ ')      )
 181                 o += ' '+ ' '.join(ol)
 182         if script is not None:
 183                 o += '\n'
 184                 for l in script.rstrip('\n').split('\n'):
 185                         o += '$     '+l+'\n'
 186         debug(o)
 187
 188 def flatten(l):
 189         return reduce((lambda a,b: a + b), l, [])
 190
 191 #---------- fancy automatic file-copying class
 192
 193 class AutoFile:
 194         # p.what
 195         # p.path[tb]    None or path    not None => path known
 196         # p.file[tb]    None or path    not None => file exists
 197         # p.spec        string or None
 198         # p.spec_tbp    True or False, or not set if spec is None
 199         # p.dir         '' or '/'
 200
 201  def __init__(p, what):
 202         p.what = what
 203         p.path = [None,None]
 204         p.file = [None,None]
 205         p.spec = None
 206         p.dir = ''
 207
 208  def __repr__(p):
 209         return "<AF@%s>" % p.__str__()
 210  def __str__(p):
 211         def ptbp(tbp):
 212                 if p.path[tbp] is None: return '-'+p.dir
 213                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 214                 else: return p.path[tbp]+p.dir+'!'
 215         out = p.what
 216         if p.spec is not None:
 217                 if p.spec_tbp: out += '#'
 218                 else: out += '='
 219                 out += p.spec
 220         out += ':'
 221         out += ptbp(False)
 222         out += '|'
 223         out += ptbp(True)
 224         return out
 225
 226  def _wrong(p, how):
 227         xtra = ''
 228         if p.spec is not None: xtra = ' spec[%s]=%s' % (p.spec, p.spec_tb)
 229         raise ("internal error: %s (%s)" % (how, str(p)))
 230
 231  def _ensure_path(p, tbp):
 232         if p.path[tbp] is None:
 233                 if '/' in p.what:
 234                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 235                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 236                 if not tbp:
 237                         p.path[tbp] = tmpdir+'/'+p.what
 238                 else:
 239                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 240
 241  def write(p, tbp=False):
 242         p._debug('write %s...' % 'HT'[tbp])
 243         p._ensure_path(tbp)
 244
 245         if p.dir and not p.file[tbp]:
 246                 if not tbp:
 247                         p._debug('mkdir H')
 248                         mkdir_okexist(p.path[tbp])
 249                 else:
 250                         cmdl = ['sh','-ec',
 251                                 'test -d "$1" || mkdir -p "$1"',
 252                                 'x', p.path[tbp]]
 253                         tf_what = urllib.quote(p.what).replace('/','%2F')
 254                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 255                         if rc: bomb('failed to create directory %s' %
 256                                 p.path[tbp])
 257
 258         p.file[tbp] = p.path[tbp]
 259         return p.path[tbp]
 260
 261  def read(p, tbp=False):
 262         p._debug('read %s...' % 'HT'[tbp])
 263         p._ensure_path(tbp)
 264
 265         if p.file[tbp] is None:
 266                 if p.file[not tbp] is None:
 267                         p._wrong("requesting read but nonexistent")
 268                 cud = ['copyup','copydown'][tbp]
 269                 src = p.file[not tbp] + p.dir
 270                 dst = p.path[tbp] + p.dir
 271                 testbed.command(cud, (src, dst))
 272                 p.file[tbp] = p.path[tbp]
 273
 274         return p.file[tbp] + p.dir
 275
 276  def invalidate(p, tbp=False):
 277         p.file[tbp] = None
 278         p._debug('invalidated %s' % 'HT'[tbp])
 279
 280  def _debug(p, m):
 281         debug('/ %s#%x: %s' % (p.what, id(p), m))
 282
 283  def _constructed(p):
 284         p._debug('constructed: '+str(p))
 285         p._check()
 286
 287  def _check(p):
 288         for tbp in [False,True]:
 289          for pf in [p.path, p.file]:
 290                 if pf[tbp] is None: continue
 291                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 292                 if p.dir and pf[tbp].endswith('/'):
 293                         pf[tbp] = pf[tbp].rstrip('/')
 294                         if not pf[tbp]: pf[tbp] = '/'
 295                 if not p.dir and pf[tbp].endswith('/'):
 296                         bomb("directory `%s' specified for "
 297                              "non-directory %s" % (pf[tbp], p.what))
 298
 299  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 300         AutoFile.__init__(p,what)
 301         sh_on = ''; sh_sibl = ''
 302         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 303         if sibling: sh_sibl=' (sibling)'
 304         parent._debug('using as base: %s: %s%s%s' %
 305                         (str(parent), leaf, sh_on, sh_sibl))
 306         if not sibling and not parent.dir:
 307                 parent._wrong('asked for non-sibling relative path of non-dir')
 308         if sibling: trim = os.path.dirname
 309         else: trim = lambda x: x
 310         for tbp in [False,True]:
 311                 if parent.path[tbp] is None: continue
 312                 trimmed = trim(parent.path[tbp])
 313                 if trimmed: trimmed += '/'
 314                 p.path[tbp] = trimmed + leaf
 315                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 316                         p.file[tbp] = p.path[tbp]
 317
 318 class InputFile(AutoFile):
 319  def _init(p, what, spec, spec_tbp=False):
 320         AutoFile.__init__(p, what)
 321         p.spec = spec
 322         p.spec_tbp = spec_tbp
 323         p.path[spec_tbp] = spec
 324         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 325  def __init__(p, what, spec, spec_tbp=False):
 326         p._init(what,spec,spec_tbp)
 327         p._constructed()
 328
 329 class InputDir(InputFile):
 330  def __init__(p, what, spec, spec_tbp=False):
 331         InputFile._init(p,what,spec,spec_tbp)
 332         p.dir = '/'
 333         p._constructed()
 334
 335 class OutputFile(AutoFile):
 336  def _init(p, what, spec, spec_tbp=False):
 337         AutoFile.__init__(p, what)
 338         p.spec = spec
 339         p.spec_tbp = spec_tbp
 340         p.path[spec_tbp] = spec
 341  def __init__(p, what, spec, spec_tbp=False):
 342         p._init(what,spec,spec_tbp)
 343         p._constructed()
 344
 345 class OutputDir(OutputFile):
 346  def __init__(p, what, spec, spec_tbp=False):
 347         OutputFile._init(p,what,spec,spec_tbp)
 348         p.dir = '/'
 349         p._constructed()
 350
 351 class RelativeInputFile(AutoFile):
 352  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 353         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 354         p._constructed()
 355
 356 class RelativeOutputFile(AutoFile):
 357  def __init__(p, what, parent, leaf, sibling=False):
 358         p._relative_init(what, parent, leaf, None, False, sibling)
 359         p._constructed()
 360
 361 class TemporaryFile(AutoFile):
 362  def __init__(p, what):
 363         AutoFile.__init__(p, what)
 364         p._constructed()
 365
 366 class TemporaryDir(AutoFile):
 367  def __init__(p, what):
 368         AutoFile.__init__(p, what)
 369         p.dir = '/'
 370         p._constructed()
 371
 372 #---------- parsing and representation of the arguments
 373
 374 class Action:
 375  def __init__(a, kind, af, arghandling, what):
 376         # extra attributes get added during processing
 377         a.kind = kind
 378         a.af = af
 379         a.ah = arghandling
 380         a.what = what
 381  def __repr__(a):
 382         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 383
 384 def parse_args():
 385         global opts
 386         usage = "%prog <options> --- <virt-server>..."
 387         parser = OptionParser(usage=usage)
 388         pa = parser.add_option
 389         pe = parser.add_option
 390
 391         arghandling = {
 392                 'dsc_tests': True,
 393                 'dsc_filter': '*',
 394                 'deb_forbuilds': 'auto',
 395                 'deb_fortests': 'auto',
 396                 'tb': False,
 397                 'override_control': None
 398         }
 399         initial_arghandling = arghandling.copy()
 400         n_non_actions = 0
 401
 402         #----------
 403         # actions (ie, test sets to run, sources to build, binaries to use):
 404
 405         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 406                 parser.largs.append((value,kindpath))
 407                 n_non_actions += not(is_act)
 408
 409         def pa_action(long, metavar, kindpath, help, is_act=True):
 410                 pa('','--'+long, action='callback', callback=cb_action,
 411                         nargs=1, type='string',
 412                         callback_args=(long,kindpath,is_act), help=help)
 413
 414         pa_action('built-tree',         'TREE', '@/',
 415                 help='run tests from build tree TREE')
 416
 417         pa_action('unbuilt-tree',       'TREE', '@//',
 418                 help='run tests from build tree TREE')
 419
 420         pa_action('source',             'DSC', '@.dsc',
 421                 help='build DSC and use its tests and/or'
 422                     ' generated binary packages')
 423
 424         pa_action('binary',             'DEB', '@.deb',
 425                help='use binary package DEB according'
 426                     ' to most recent --binaries-* settings')
 427
 428         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 429                help='run tests from control file CONTROL instead,'
 430                     ' (applies to next test suite only)')
 431
 432         #----------
 433         # argument handling settings (what ways to use action
 434         #  arguments, and pathname processing):
 435
 436         def cb_setah(option, opt_str, value, parser, toset,setval):
 437                 if type(setval) == list:
 438                         if not value in setval:
 439                                 parser.error('value for %s option (%s) is not '
 440                                  'one of the permitted values (%s)' %
 441                                  (value, opt_str, setval.join(' ')))
 442                 elif setval is not None:
 443                         value = setval
 444                 for v in toset:
 445                         arghandling[v] = value
 446                 parser.largs.append(arghandling.copy())
 447
 448         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 449                 type = metavar
 450                 if type is not None: type = 'string'
 451                 pa('',long, action='callback', callback=cb_setah,
 452                    callback_args=(affected,effect), **kwargs)
 453
 454         #---- paths: host or testbed:
 455         #
 456         pa_setah('--paths-testbed', ['tb'],True,
 457                 help='subsequent path specifications refer to the testbed')
 458         pa_setah('--paths-host', ['tb'],False,
 459                 help='subsequent path specifications refer to the host')
 460
 461         #---- source processing settings:
 462
 463         pa_setah('--sources-tests', ['dsc_tests'],True,
 464                 help='run tests from builds of subsequent sources')
 465         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 466                 help='do not run tests from builds of subsequent sources')
 467
 468         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 469                 type='string', metavar='PATTERN-LIST',
 470                 help='from subsequent sources, use binaries matching'
 471                      ' PATTERN-LIST (comma-separated glob patterns)'
 472                      ' according to most recent --binaries-* settings')
 473         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 474                 help='from subsequent sources, do not use any binaries')
 475
 476         #---- binary package processing settings:
 477
 478         def pa_setahbins(long,toset,how):
 479          pa_setah(long, toset,['ignore','auto','install'],
 480                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 481                 help=how+' ignore binaries, install them as needed'
 482                         ' for dependencies, or unconditionally install'
 483                         ' them, respectively')
 484         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 485         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 486         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 487
 488         #----------
 489         # general options:
 490
 491         def cb_vserv(op,optstr,value,parser):
 492                 parser.values.vserver = list(parser.rargs)
 493                 del parser.rargs[:]
 494
 495         def cb_path(op,optstr,value,parser, constructor,long,dir):
 496                 name = long.replace('-','_')
 497                 af = constructor(arghandling['tb'], value, long, dir)
 498                 setattr(parser.values, name, af)
 499
 500         def pa_path(long, constructor, help, dir=False):
 501                 pa('','--'+long, action='callback', callback=cb_path,
 502                         callback_args=(constructor,long,dir),
 503                         nargs=1, type='string',
 504                         help=help, metavar='PATH')
 505
 506         pa_path('output-dir', OutputDir, dir=True,
 507                 help='write stderr/out files in PATH')
 508
 509         pa('','--tmp-dir',              type='string', dest='tmpdir',
 510                 help='write temporary files to TMPDIR, emptying it'
 511                      ' beforehand and leaving it behind at the end')
 512         pa('','--log-file',             type='string', dest='logfile',
 513                 help='write the log LOGFILE, emptying it beforehand,'
 514                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 515
 516         pa('','--user',                 type='string', dest='user',
 517                 help='run tests as USER (needs root on testbed)')
 518         pa('','--gain-root',            type='string', dest='gainroot',
 519                 help='prefix debian/rules binary with GAINROOT')
 520         pa('-q', '--quiet', action='store_false', dest='debug', default=True);
 521         pa('','--gnupg-home',           type='string', dest='gnupghome',
 522                 default='~/.autopkgtest/gpg',
 523                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 524                         " signing private apt archive);"
 525                         " `fresh' means generate new key each time.")
 526
 527         #----------
 528         # actual meat:
 529
 530         class SpecialOption(optparse.Option): pass
 531         vs_op = SpecialOption('','--VSERVER-DUMMY')
 532         vs_op.action = 'callback'
 533         vs_op.type = None
 534         vs_op.default = None
 535         vs_op.nargs = 0
 536         vs_op.callback = cb_vserv
 537         vs_op.callback_args = ( )
 538         vs_op.callback_kwargs = { }
 539         vs_op.help = 'introduces virtualisation server and args'
 540         vs_op._short_opts = []
 541         vs_op._long_opts = ['---']
 542
 543         pa(vs_op)
 544
 545         (opts,args) = parser.parse_args()
 546         if not hasattr(opts,'vserver'):
 547                 parser.error('you must specifiy --- <virt-server>...')
 548         if n_non_actions >= len(parser.largs):
 549                 parser.error('nothing to do specified')
 550
 551         arghandling = initial_arghandling
 552         opts.actions = []
 553         ix = 0
 554         for act in args:
 555                 if type(act) == dict:
 556                         arghandling = act
 557                         continue
 558                 elif type(act) == tuple:
 559                         pass
 560                 elif type(act) == str:
 561                         act = (act,act)
 562                 else:
 563                         raise ("unknown action in list `%s' having"
 564                               " type `%s'" % (act, type(act)))
 565                 (pathstr, kindpath) = act
 566
 567                 constructor = InputFile
 568                 if type(kindpath) is tuple:             kind = kindpath[0]
 569                 elif kindpath.endswith('.deb'):         kind = 'deb'
 570                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 571                 elif kindpath.endswith('//'):
 572                         kind = 'ubtree'
 573                         constructor = InputDir
 574                 elif kindpath.endswith('/'):
 575                         kind = 'tree'
 576                         constructor = InputDir
 577                 else: parser.error("do not know how to handle filename \`%s';"
 578                         " specify --source --binary or --build-tree")
 579
 580                 what = '%s%s' % (kind,ix); ix += 1
 581
 582                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 583                 else: fwhatx = '-'+kind
 584
 585                 af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 586                 opts.actions.append(Action(kind, af, arghandling, what))
 587
 588 def setup_trace():
 589         global trace_stream, tmpdir
 590
 591         if opts.tmpdir is not None:
 592                 rmtree('tmpdir(specified)',opts.tmpdir)
 593                 mkdir_okexist(opts.tmpdir, 0700)
 594                 tmpdir = opts.tmpdir
 595         else:
 596                 assert(tmpdir is None)
 597                 tmpdir = tempfile.mkdtemp()
 598
 599         if opts.logfile is None:
 600                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 601                         opts.logfile = opts.output_dir.spec + '/log'
 602                 elif opts.tmpdir is not None:
 603                         opts.logfile = opts.tmpdir + '/log'
 604         if opts.logfile is not None:
 605                 trace_stream = open(opts.logfile, 'w', 0)
 606
 607         debug('#options: '+`opts`)
 608
 609 def finalise_options():
 610         global opts, tb
 611
 612         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 613                 opts.user = ''
 614
 615         if opts.user is None:
 616                 su = 'suggested-normal-user='
 617                 ul = [
 618                         e[len(su):]
 619                         for e in testbed.caps
 620                         if e.startswith(su)
 621                         ]
 622                 if ul:
 623                         opts.user = ul[0]
 624                 else:
 625                         opts.user = ''
 626
 627         if opts.user:
 628                 if 'root-on-testbed' not in testbed.caps:
 629                         pstderr("warning: virtualisation"
 630                                 " system does not offer root on testbed,"
 631                                 " but --user option specified: failure likely")
 632                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 633         else:
 634                 opts.user_wrap = lambda x: x
 635
 636         if opts.gainroot is None:
 637                 opts.gainroot = ''
 638                 if (opts.user or
 639                     'root-on-testbed' not in testbed.caps):
 640                         opts.gainroot = 'fakeroot'
 641
 642         if opts.gnupghome.startswith('~/'):
 643                 try: home = os.environ['HOME']
 644                 except KeyError:
 645                         parser.error("HOME environment variable"
 646                                 " not set, needed for --gnupghome=`%s"
 647                                 % opts.gnupghome)
 648                 opts.gnupghome = home + opts.gnupghome[1:]
 649         elif opts.gnupghome == 'fresh':
 650                 opts.gnupghome = None
 651
 652 #---------- testbed management - the Testbed class
 653
 654 class Testbed:
 655  def __init__(tb):
 656         tb.sp = None
 657         tb.lastsend = None
 658         tb.scratch = None
 659         tb.modified = False
 660         tb.blamed = []
 661         tb._ephemeral = []
 662         tb._debug('init')
 663         tb._need_reset_apt = False
 664  def _debug(tb, m):
 665         debug('** '+m)
 666  def start(tb):
 667         tb._debug('start')
 668         p = subprocess.PIPE
 669         debug_subprocess('vserver', opts.vserver)
 670         tb._errplumb = Errplumb(True)
 671         tb.sp = subprocess.Popen(opts.vserver,
 672                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 673         tb.expect('ok')
 674         tb.caps = tb.commandr('capabilities')
 675  def stop(tb):
 676         tb._debug('stop')
 677         tb.close()
 678         if tb.sp is None: return
 679         ec = tb.sp.returncode
 680         if ec is None:
 681                 tb.sp.stdout.close()
 682                 tb.send('quit')
 683                 tb.sp.stdin.close()
 684                 ec = tb.sp.wait()
 685         if ec:
 686                 tb.bomb('testbed gave exit status %d after quit' % ec)
 687         tb._errplumb.wait()
 688  def open(tb):
 689         tb._debug('open, scratch=%s' % tb.scratch)
 690         if tb.scratch is not None: return
 691         pl = tb.commandr('open')
 692         tb.scratch = InputDir('tb-scratch', pl[0], True)
 693         tb.deps_processed = []
 694  def mungeing_apt(tb):
 695         if not 'revert' in tb.caps:
 696                 tb._need_reset_apt = True
 697  def reset_apt(tb):
 698         if not tb._need_reset_apt: return
 699         what = 'aptget-update-reset'
 700         cmdl = ['apt-get','-qy','update']
 701         rc = tb.execute(what, cmdl)
 702         if rc:
 703                 pstderr("\n" "warning: failed to restore"
 704                         " testbed apt cache, exit code %d" % rc)
 705         tb._need_reset_apt = False
 706  def close(tb):
 707         tb._debug('close, scratch=%s' % tb.scratch)
 708         if tb.scratch is None: return
 709         tb.scratch = None
 710         if tb.sp is None: return
 711         tb.command('close')
 712  def prepare(tb, deps_new):
 713         tb._debug('prepare, modified=%s, deps_processed=%s, deps_new=%s' %
 714                 (tb.modified, tb.deps_processed, deps_new))
 715         if 'revert' in tb.caps and (tb.modified or
 716             [d for d in tb.deps_processed if d not in deps_new]):
 717                 tb._debug('reset **')
 718                 tb.command('revert')
 719                 tb.blamed = []
 720                 for af in tb._ephemeral: af.invalidate(True)
 721         binaries.publish()
 722         tb.modified = False
 723         tb._install_deps(deps_new)
 724  def register_ephemeral(tb, af):
 725         if not getattr(af,'spec_tbp',False): tb._ephemeral.append(af)
 726  def _install_deps(tb, deps_new):
 727         tb._debug(' installing dependencies '+`deps_new`)
 728         tb.deps_processed = deps_new
 729         if not deps_new: return
 730         dstr = ', '.join(deps_new)
 731         script = binaries.apt_pkg_gdebi_script(
 732                 dstr, [[
 733                 'from GDebi.DebPackage import DebPackage',
 734                 'd = DebPackage(cache)',
 735                 'res = d.satisfyDependsStr(arg)',
 736                 ]])
 737         cmdl = ['python','-c',script]
 738         what = 'install-deps'
 739         rc = testbed.execute(what, cmdl, script=script)
 740         if rc: badpkg('dependency install failed, exit code %d' % rc)
 741  def needs_reset(tb):
 742         tb._debug('needs_reset, previously=%s' % tb.modified)
 743         tb.modified = True
 744  def blame(tb, m):
 745         tb._debug('blame += %s' % m)
 746         tb.blamed.append(m)
 747  def bomb(tb, m):
 748         tb._debug('bomb %s' % m)
 749         if tb.sp is not None:
 750                 tb.sp.stdout.close()
 751                 tb.sp.stdin.close()
 752                 ec = tb.sp.wait()
 753                 if ec: pstderr('adt-run: testbed failing,'
 754                                 ' exit status %d' % ec)
 755         tb.sp = None
 756         raise Quit(16, 'testbed failed: %s' % m)
 757  def send(tb, string):
 758         tb.sp.stdin
 759         try:
 760                 debug('>> '+string)
 761                 print >>tb.sp.stdin, string
 762                 tb.sp.stdin.flush()
 763                 tb.lastsend = string
 764         except:
 765                 (type, value, dummy) = sys.exc_info()
 766                 tb.bomb('cannot send to testbed: %s' % traceback.
 767                         format_exception_only(type, value))
 768  def expect(tb, keyword, nresults=None):
 769         l = tb.sp.stdout.readline()
 770         if not l: tb.bomb('unexpected eof from the testbed')
 771         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 772         l = l.rstrip('\n')
 773         debug('<< '+l)
 774         ll = l.split()
 775         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 776         if ll[0] != keyword:
 777                 if tb.lastsend is None:
 778                         tb.bomb("got banner `%s', expected `%s...'" %
 779                                 (l, keyword))
 780                 else:
 781                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 782                                 (tb.lastsend, l, keyword))
 783         ll = ll[1:]
 784         if nresults is not None and len(ll) != nresults:
 785                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 786                         " expected %d result parameters" %
 787                         (string, l, len(ll), nresults))
 788         return ll
 789  def commandr(tb, cmd, args=(), nresults=None):
 790         # pass args=[None,...] or =(None,...) to avoid more url quoting
 791         if type(cmd) is str: cmd = [cmd]
 792         if len(args) and args[0] is None: args = args[1:]
 793         else: args = map(urllib.quote, args)
 794         al = cmd + args
 795         tb.send(string.join(al))
 796         ll = tb.expect('ok', nresults)
 797         rl = map(urllib.unquote, ll)
 798         return rl
 799  def command(tb, cmd, args=()):
 800         tb.commandr(cmd, args, 0)
 801  def commandr1(tb, cmd, args=()):
 802         rl = tb.commandr(cmd, args, 1)
 803         return rl[0]
 804  def execute(tb, what, cmdl,
 805                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 806                 script=False):
 807         # Options for script:
 808         #   False - do not call debug_subprocess, no synch. reporting required
 809         #   None or string - call debug_subprocess with that value,
 810         #                       plumb stderr through synchronously if possible
 811         # Options for se:
 812         #   None - usual Errplumb (output is of kind 2c)
 813         #   string - path on testbed (output is of kind 2d)
 814
 815         if script is not False: debug_subprocess(what, cmdl, script=script)
 816         if cwd is None: cwd = tb.scratch.write(True)
 817
 818         xdump = None
 819         if se is None:
 820                 ep = Errplumb()
 821                 se_catch = TemporaryFile(what+'-xerr')
 822                 se_use = se_catch.write(True)
 823                 if opts.debug: xdump = 'debug=2-2'
 824                 elif trace_stream is not None:
 825                         xdump = 'debug=2-%d' % trace_stream.fileno()
 826         else:
 827                 ep = None
 828                 se_catch = None
 829                 se_use = se
 830
 831         cmdl = [None,
 832                 ','.join(map(urllib.quote, cmdl)),
 833                 si, so, se_use, cwd]
 834         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 835         rc = tb.commandr1('execute', cmdl)
 836         try: rc = int(rc)
 837         except ValueError: bomb("execute for %s gave invalid response `%s'"
 838                                         % (what,rc))
 839
 840         if se_catch is not None:
 841                 debug_file(se_catch.read())
 842         if ep is not None:
 843                 ep.wait()
 844
 845         return rc
 846
 847 #---------- representation of test control files: Field*, Test, etc.
 848
 849 class FieldBase:
 850  def __init__(f, fname, stz, base, tnames, vl):
 851         assert(vl)
 852         f.stz = stz
 853         f.base = base
 854         f.tnames = tnames
 855         f.vl = vl
 856  def words(f):
 857         def distribute(vle):
 858                 (lno, v) = vle
 859                 r = v.split()
 860                 r = map((lambda w: (lno, w)), r)
 861                 return r
 862         return flatten(map(distribute, f.vl))
 863  def atmostone(f):
 864         if len(vl) == 1:
 865                 (f.lno, f.v) = vl[0]
 866         else:
 867                 raise Unsupported(f.vl[1][0],
 868                         'only one %s field allowed' % fn)
 869         return f.v
 870
 871 class FieldIgnore(FieldBase):
 872  def parse(f): pass
 873
 874 class Restriction:
 875  def __init__(r,rname,base): pass
 876
 877 class Restriction_rw_build_tree(Restriction): pass
 878 class Restriction_rw_tests_tree(Restriction): pass
 879 class Restriction_breaks_testbed(Restriction):
 880  def __init__(r, rname, base):
 881         if 'revert' not in testbed.caps:
 882                 raise Unsupported(f.lno,
 883                         'Test breaks testbed but testbed cannot revert')
 884 class Restriction_needs_root(Restriction):
 885  def __init__(r, rname, base):
 886         if 'root-on-testbed' not in testbed.caps:
 887                 raise Unsupported(f.lno,
 888                         'Test needs root on testbed which is not available')
 889
 890 class Field_Restrictions(FieldBase):
 891  def parse(f):
 892         for wle in f.words():
 893                 (lno, rname) = wle
 894                 nrname = rname.replace('-','_')
 895                 try: rclass = globals()['Restriction_'+nrname]
 896                 except KeyError: raise Unsupported(lno,
 897                         'unknown restriction %s' % rname)
 898                 r = rclass(nrname, f.base)
 899                 f.base['restrictions'].append(r)
 900
 901 class Field_Tests(FieldIgnore): pass
 902
 903 class Field_Depends(FieldBase):
 904  def parse(f):
 905         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
 906         dl = map(lambda x: x.strip(),
 907                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
 908         re = regexp.compile('[^-.+:~0-9a-z()<>=*]')
 909         for d in dl:
 910                 if re.search(d):
 911                         badpkg("Test Depends field contains dependency"
 912                                " `%s' with invalid characters" % d)
 913         f.base['depends'] = dl
 914
 915 class Field_Tests_directory(FieldBase):
 916  def parse(f):
 917         td = atmostone(f)
 918         if td.startswith('/'): raise Unspported(f.lno,
 919                 'Tests-Directory may not be absolute')
 920         f.base['testsdir'] = td
 921
 922 def run_tests(stanzas, tree):
 923         global errorcode, testbed
 924         for stanza in stanzas:
 925                 tests = stanza[' tests']
 926                 if not tests:
 927                         report('*', 'SKIP no tests in this package')
 928                         errorcode |= 8
 929                 for t in tests:
 930                         t.prepare()
 931                         t.run(tree)
 932                         if 'breaks-testbed' in t.restrictions:
 933                                 testbed.needs_reset()
 934                 testbed.needs_reset()
 935
 936 class Test:
 937  def __init__(t, tname, base, act):
 938         if '/' in tname: raise Unsupported(base[' lno'],
 939                 'test name may not contain / character')
 940         for k in base: setattr(t,k,base[k])
 941         t.tname = tname
 942         t.act = act
 943         t.what = act.what+'t-'+tname
 944         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
 945         else: t.path = tname
 946         t._debug('constructed; path=%s' % t.path)
 947         t._debug(' .depends=%s' % t.depends)
 948  def _debug(t, m):
 949         debug('& %s: %s' % (t.what, m))
 950  def report(t, m):
 951         report(t.what, m)
 952  def reportfail(t, m):
 953         global errorcode
 954         errorcode |= 4
 955         report(t.what, 'FAIL ' + m)
 956  def prepare(t):
 957         t._debug('preparing')
 958         dn = []
 959         for d in t.depends:
 960                 t._debug(' processing dependency '+d)
 961                 if not '_' in d:
 962                         t._debug('  literal dependency '+d)
 963                         dn.append(d)
 964                 else:
 965                         for (pkg,bin) in t.act.binaries:
 966                                 d = d.replace('_',pkg)
 967                                 t._debug('  synthesised dependency '+d)
 968                                 dn.append(d)
 969         testbed.prepare(dn)
 970  def run(t, tree):
 971         t._debug('[----------------------------------------')
 972         def stdouterr(oe):
 973                 idstr = t.what + '-' + oe
 974                 if opts.output_dir is not None and opts.output_dir.tb:
 975                         use_dir = opts.output_dir
 976                 else:
 977                         use_dir = testbed.scratch
 978                 return RelativeOutputFile(idstr, use_dir, idstr)
 979
 980         t.act.work.write(True)
 981
 982         af = RelativeInputFile(t.what, tree, t.path)
 983         so = stdouterr('stdout')
 984         se = stdouterr('stderr')
 985         tf = af.read(True)
 986         if 'needs-root' not in t.restrictions:
 987                 tf = opts.user_wrap(tf)
 988         rc = testbed.execute('test-'+t.what, [tf],
 989                 so=so.write(True), se=se.write(True), cwd=tree.read(True))
 990
 991         so_read = so.read()
 992         se_read = se.read()
 993
 994         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
 995         stab = os.stat(se_read)
 996         if stab.st_size != 0:
 997                 l = open(se_read).readline()
 998                 l = l.rstrip('\n \t\r')
 999                 if len(l) > 40: l = l[:40] + '...'
1000                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1001                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1002                 debug_file(se_read)
1003         elif rc != 0:
1004                 t.reportfail('non-zero exit status %d' % rc)
1005         else:
1006                 t.report('PASS')
1007
1008         stab = os.stat(so_read)
1009         if stab.st_size != 0:
1010                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1011                 debug_file(so_read)
1012
1013         t._debug('----------------------------------------]')
1014
1015 def read_control(act, tree, control_override):
1016         stanzas = [ ]
1017
1018         if control_override is not None:
1019                 control_af = control_override
1020                 testbed.blame('arg:'+control_override.spec)
1021         else:
1022                 control_af = RelativeInputFile(act.what+'-testcontrol',
1023                         tree, 'debian/tests/control')
1024         try:
1025                 control = open(control_af.read(), 'r')
1026         except OSError, oe:
1027                 if oe[0] != errno.ENOENT: raise
1028                 return []
1029
1030         lno = 0
1031         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1032         stz = None      # stz[field_name][index] = (lno, value)
1033                         # special field names:
1034                         # stz[' lno'] = number
1035                         # stz[' tests'] = list of Test objects
1036         def end_stanza(stz):
1037                 if stz is None: return
1038                 stz[' errs'] = 0
1039                 stanzas.append(stz)
1040                 stz = None
1041                 hcurrent = None
1042
1043         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1044         while 1:
1045                 l = control.readline()
1046                 if not l: break
1047                 lno += 1
1048                 if not l.endswith('\n'): badctrl('unterminated line')
1049                 if regexp.compile('\s*\#').match(l): continue
1050                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1051                 initmat = initre.match(l)
1052                 if initmat:
1053                         (fname, l) = initmat.groups()
1054                         fname = string.capwords(fname)
1055                         if stz is None:
1056                                 stz = { ' lno': lno, ' tests': [] }
1057                         if not stz.has_key(fname): stz[fname] = [ ]
1058                         hcurrent = stz[fname]
1059                 elif regexp.compile('\s').match(l):
1060                         if not hcurrent: badctrl('unexpected continuation')
1061                 else:
1062                         badctrl('syntax error')
1063                 hcurrent.append((lno, l))
1064         end_stanza(stz)
1065
1066         def testbadctrl(stz, lno, m):
1067                 report_badctrl(lno, m)
1068                 stz[' errs'] += 1
1069
1070         for stz in stanzas:
1071                 try:
1072                         try: tnames = stz['Tests']
1073                         except KeyError:
1074                                 tnames = ['*']
1075                                 raise Unsupported(stz[' lno'],
1076                                         'no Tests field')
1077                         tnames = map((lambda lt: lt[1]), tnames)
1078                         tnames = string.join(tnames).split()
1079                         base = {
1080                                 'restrictions': [],
1081                                 'testsdir': 'debian/tests',
1082                                 'depends' : '_'
1083                         }
1084                         for fname in stz.keys():
1085                                 if fname.startswith(' '): continue
1086                                 vl = stz[fname]
1087                                 try: fclass = globals()['Field_'+
1088                                         fname.replace('-','_')]
1089                                 except KeyError: raise Unsupported(vl[0][0],
1090                                         'unknown metadata field %s' % fname)
1091                                 f = fclass(stz, fname, base, tnames, vl)
1092                                 f.parse()
1093                         for tname in tnames:
1094                                 t = Test(tname, base, act)
1095                                 stz[' tests'].append(t)
1096                 except Unsupported, u:
1097                         for tname in tnames: u.report(tname)
1098                         continue
1099
1100         return stanzas
1101
1102 def print_exception(ei, msgprefix=''):
1103         if msgprefix: pstderr(msgprefix)
1104         (et, q, tb) = ei
1105         if et is Quit:
1106                 pstderr('adt-run: ' + q.m)
1107                 return q.ec
1108         else:
1109                 pstderr("adt-run: unexpected, exceptional, error:")
1110                 traceback.print_exc(None, sys.stderr)
1111                 if trace_stream is not None:
1112                         traceback.print_exc(None, trace_stream)
1113                 return 20
1114
1115 def cleanup():
1116         global trace_stream
1117         try:
1118                 rm_ec = 0
1119                 if opts.tmpdir is None and tmpdir is not None:
1120                         rmtree('tmpdir', tmpdir)
1121                 if testbed is not None:
1122                         testbed.reset_apt()
1123                         testbed.stop()
1124                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1125                 if trace_stream is not None:
1126                         trace_stream.close()
1127                         trace_stream = None
1128         except:
1129                 print_exception(sys.exc_info(),
1130                         '\nadt-run: error cleaning up:\n')
1131                 os._exit(20)
1132
1133 #---------- registration, installation etc. of .deb's: Binaries
1134
1135 def determine_package(act):
1136         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1137         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1138         if rc: badpkg('failed to parse binary package, code %d' % rc)
1139         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1140         act.pkg = None
1141         for l in output.split('\n'):
1142                 m = re.match(l)
1143                 if not m: continue
1144                 if act.pkg: badpkg('two Package: lines in control file')
1145                 act.pkg = m.groups()[0]
1146         if not act.pkg: badpkg('no good Package: line in control file')
1147
1148 class Binaries:
1149  def __init__(b):
1150         b.dir = TemporaryDir('binaries')
1151         b.dir.write()
1152         ok = False
1153
1154         if opts.gnupghome is None:
1155                 opts.gnupghome = tmpdir+'/gnupg'
1156
1157         b._debug('initialising')
1158         try:
1159                 for x in ['pubring','secring']:
1160                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1161                 ok = True
1162         except OSError, oe:
1163                 if oe.errno != errno.ENOENT: raise
1164
1165         if ok: b._debug('no key generation needed')
1166         else: b.genkey()
1167
1168  def _debug(b, s):
1169         debug('* '+s)
1170
1171  def genkey(b):
1172         b._debug('preparing for key generation')
1173
1174         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1175         mkdir_okexist(opts.gnupghome, 0700)
1176
1177         script = '''
1178   exec >&2
1179   cd "$1"
1180   cat <<"END" >key-gen-params
1181 Key-Type: DSA
1182 Key-Length: 1024
1183 Key-Usage: sign
1184 Name-Real: autopkgtest per-run key
1185 Name-Comment: do not trust this key
1186 Name-Email: autopkgtest@example.com
1187 END
1188   set -x
1189   gpg --homedir="$1" --batch --gen-key key-gen-params
1190                 '''
1191         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1192         rc = subprocess_cooked(cmdl, dbg=(genkey,script))[0]
1193         if rc:
1194                 try:
1195                         f = open(opts.gnupghome+'/key-gen-log')
1196                         tp = file.read()
1197                 except OSError, e: tp = e
1198                 pstderr(tp)
1199                 bomb('key generation failed, code %d' % rc)
1200
1201  def apt_configs(b):
1202         return {
1203                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1204         }
1205
1206  def apt_pkg_gdebi_script(b, arg, middle):
1207         script = [
1208                 'import apt_pkg',
1209                 'import urllib',
1210                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1211                 ]
1212         for (k,v) in b.apt_configs().iteritems():
1213                 v = urllib.quote(v)
1214                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1215                                 % (k, v))
1216         script += [
1217                 'from GDebi.Cache import Cache',
1218                 'cache = Cache()',
1219                 ]
1220         for m in middle:
1221                 script += m + [
1222                 'print res',
1223                 'print d.missingDeps',
1224                 'print d.requiredChanges',
1225                 'assert(res)',
1226                 'cache.commit()',
1227                 ''
1228                 ]
1229         return '\n'.join(script)
1230  def apt_get(b):
1231         ag = ['apt-get','-qy']
1232         for kv in b.apt_configs().iteritems():
1233                 ag += ['-o', '%s=%s' % kv]
1234         return ag
1235
1236  def reset(b):
1237         b._debug('reset')
1238         rmtree('binaries', b.dir.read())
1239         b.dir.invalidate()
1240         b.dir.write()
1241         b.install = []
1242         b.blamed = []
1243         b.registered = set()
1244
1245  def register(b, act, pkg, af, forwhat, blamed):
1246         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1247                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1248
1249         if act.ah['deb_'+forwhat] == 'ignore': return
1250
1251         b.blamed += testbed.blamed
1252
1253         leafname = pkg+'.deb'
1254         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1255
1256         try: os.remove(dest.write())
1257         except OSError, oe:
1258                 if oe.errno != errno.ENOENT: raise e
1259
1260         try: os.link(af.read(), dest.write())
1261         except OSError, oe:
1262                 if oe.errno != errno.EXDEV: raise e
1263                 shutil.copy(af.read(), dest)
1264
1265         if act.ah['deb_'+forwhat] == 'install':
1266                 b.install.append(pkg)
1267
1268         b.registered.add(pkg)
1269
1270  def publish(b):
1271         b._debug('publish')
1272
1273         script = '''
1274   exec >&2
1275   cd "$1"
1276   apt-ftparchive packages . >Packages
1277   gzip <Packages >Packages.gz
1278   apt-ftparchive release . >Release
1279   rm -f Release.gpg
1280   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1281   gpg --homedir="$2" --batch --export >archive-key.pgp
1282                 '''
1283         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1284         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1285         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1286
1287         b.dir.invalidate(True)
1288         apt_source = b.dir.read(True)
1289
1290         so = TemporaryFile('vlds')
1291         script = '''
1292   exec 3>&1 >&2
1293   apt-key add archive-key.pgp
1294   echo "deb file://'''+apt_source+''' /" >sources.list
1295   cat /etc/apt/sources.list >>sources.list
1296   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1297     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1298   fi
1299   '''+ ' '.join(b.apt_get()) +''' update >&2
1300   cat /var/lib/dpkg/status >&3
1301                 '''
1302         testbed.mungeing_apt()
1303         rc = testbed.execute('apt-key', ['sh','-ec',script],
1304                                 so=so.write(True), cwd=b.dir.write(True),
1305                                 script=script)
1306         if rc: bomb('apt setup failed with exit code %d' % rc)
1307
1308         testbed.blamed += b.blamed
1309
1310         b._debug('publish reinstall checking...')
1311         pkgs_reinstall = set()
1312         pkg = None
1313         for l in open(so.read()):
1314                 if l.startswith('Package: '):
1315                         pkg = l[9:].rstrip()
1316                 elif l.startswith('Status: install '):
1317                         if pkg in b.registered:
1318                                 pkgs_reinstall.add(pkg)
1319                                 b._debug(' publish reinstall needs '+pkg)
1320
1321         if pkgs_reinstall:
1322                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1323                 what = 'apt-get-reinstall'
1324                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1325                         [pkg for pkg in pkgs_reinstall])
1326                 rc = testbed.execute(what, cmdl, script=None)
1327                 if rc: badpkg("installation of basic binarries failed,"
1328                                 " exit code %d" % rc)
1329
1330         b._debug('publish install...')
1331         for pkg in b.install:
1332                 what = 'apt-get-install-%s' % pkg
1333                 testbed.blame(pkg)
1334                 cmdl = b.apt_get() + ['install',pkg]
1335                 rc = testbed.execute(what, cmdl, script=None)
1336                 if rc: badpkg("installation of %s failed, exit code %d"
1337                                 % (pkg, rc))
1338
1339         b._debug('publish done')
1340
1341 #---------- processing of sources (building)
1342
1343 def source_rules_command(act,script,what,which,work,cwd,
1344                                 results_lines=0,xargs=[]):
1345         script = [      "exec 3>&1 >&2",
1346                         "set -x"        ] + script
1347         script = '\n'.join(script)
1348         so = TemporaryFile('%s-%s-results' % (what,which))
1349         rc = testbed.execute('%s-%s' % (what,which),
1350                         ['sh','-ec',script]+xargs, script=script,
1351                         so=so.write(True), cwd=cwd)
1352         results = open(so.read()).read().rstrip('\n').split("\n")
1353         if rc: badpkg("%s failed with exit code %d" % (which,rc))
1354         if results_lines is not None and len(results) != results_lines:
1355                 badpkg("got %d lines of results from %s where %d expected"
1356                         % (len(results), which, results_lines))
1357         if results_lines==1: return results[0]
1358         return results
1359
1360 def build_source(act):
1361         act.blame = 'arg:'+act.af.spec
1362         testbed.blame(act.blame)
1363         testbed.needs_reset()
1364
1365         what = act.what
1366         basename = act.af.spec
1367
1368         if act.kind == 'dsc':
1369                 dsc = act.af
1370                 dsc_file = open(dsc.read())
1371                 in_files = False
1372                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1373                 for l in dsc_file:
1374                         l = l.rstrip('\n')
1375                         if l.startswith('Files:'): in_files = True; continue
1376                         elif l.startswith('#'): pass
1377                         elif not l.startswith(' '):
1378                                 in_files = False
1379                                 if l.startswith('Source:'):
1380                                         act.blame = 'dsc:'+l[7:].strip()
1381                                         testbed.blame(act.blame)
1382                         if not in_files: continue
1383
1384                         m = fre.match(l)
1385                         if not m: badpkg(".dsc contains unparseable line"
1386                                         " in Files: `%s'" % l)
1387                         leaf = m.groups(0)[0]
1388                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1389                                         sibling=True)
1390                         subfile.read(True)
1391                 dsc.read(True)
1392
1393         if act.kind == 'ubtree':
1394                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1395                         act.af, 'debian/control')
1396                 dsc = TemporaryFile(what+'-fakedsc')
1397                 dsc_w = open(dsc.write(), 'w')
1398                 for l in open(debiancontrol.read()):
1399                         l = l.rstrip('\n')
1400                         if not len(l): break
1401                         print >>dsc_w, l
1402                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1403                 dsc_w.close()
1404
1405         script = binaries.apt_pkg_gdebi_script(
1406                 dsc.read(True), [[
1407                 'from GDebi.DscSrcPackage import DscSrcPackage',
1408                 'd = DscSrcPackage(cache, arg)',
1409                 'res = d.checkDeb()',
1410                  ],[
1411                 'from GDebi.DebPackage import DebPackage',
1412                 'd = DebPackage(cache)',
1413                 'res = d.satisfyDependsStr("build-essential")',
1414                 ]])
1415
1416         cmdl = ['python','-c',script]
1417         whatp = what+'-builddeps'
1418         rc = testbed.execute(what, cmdl, script=script)
1419         if rc: badpkg('build-depends install failed, exit code %d' % rc)
1420
1421         work = TemporaryDir(what+'-build')
1422
1423         script = [
1424                 'arg="$1"',
1425                 'origpwd=`pwd`',
1426                 'cd '+work.write(True)
1427         ]
1428
1429         if act.kind == 'ubtree':
1430                 spec = '%s/real-tree' % work.write(True)
1431                 create_command = '''
1432                         rm -rf "$spec"
1433                         mkdir "$spec"
1434                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1435                         '''
1436                 initcwd = act.af.read(True)
1437
1438         if act.kind == 'dsc':
1439                 spec = dsc.read(True)
1440                 create_command = '''
1441                         dpkg-source -x $spec
1442                         '''
1443                 initcwd = work.write(True)
1444
1445         if opts.user: script += [
1446                         'chown '+opts.user+' .',
1447                         'spec="$arg" '+opts.user_wrap(create_command)
1448                         ]
1449         else: script += [
1450                         'spec="$arg"',
1451                         create_command
1452                 ]
1453
1454         script += [
1455                         'cd */.',
1456                         'dpkg-checkbuilddeps',
1457                         'pwd >&3',
1458                         opts.user_wrap('debian/rules build'),
1459                 ]
1460         result_pwd = source_rules_command(act,script,what,'build',work,
1461                         cwd=initcwd, results_lines=1, xargs=['x',spec])
1462
1463         if os.path.dirname(result_pwd)+'/' != work.read(True):
1464                 badpkg("results dir `%s' is not in expected parent dir `%s'"
1465                         % (result_pwd, work.read(True)))
1466
1467         act.work = work
1468         act.tests_tree = InputDir(what+'-tests-tree',
1469                                 work.read(True)+os.path.basename(result_pwd),
1470                                 True)
1471         if act.ah['dsc_tests']:
1472                 act.tests_tree.read()
1473                 testbed.register_ephemeral(act.work)
1474                 testbed.register_ephemeral(act.tests_tree)
1475
1476         act.blamed = copy.copy(testbed.blamed)
1477
1478         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1479         act.binaries = []
1480         filter = act.ah['dsc_filter']
1481         debug_b('filter=%s' % filter)
1482         if filter != '_':
1483                 script = [
1484                         'cd '+work.write(True)+'/*/.',
1485                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1486                         'cd ..',
1487                         'echo *.deb >&3',
1488                         ]
1489                 result_debs = source_rules_command(act,script,what,
1490                                 'binary',work,work.write(True),results_lines=1)
1491                 if result_debs == '*': debs = []
1492                 else: debs = result_debs.split(' ')
1493                 debug_b('debs='+`debs`)
1494                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1495                 for deb in debs:
1496                         m = re.match(deb)
1497                         if not m: badpkg("badly-named binary `%s'" % deb)
1498                         pkg = m.groups()[0]
1499                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1500                         for pat in filter.split(','):
1501                                 debug_b('  pat=%s' % pat)
1502                                 if not fnmatch.fnmatchcase(pkg,pat):
1503                                         debug_b('   no match')
1504                                         continue
1505                                 deb_what = pkg+'_'+what+'.deb'
1506                                 bin = RelativeInputFile(deb_what,work,deb,True)
1507                                 debug_b('  deb_what=%s, bin=%s' %
1508                                         (deb_what, str(bin)))
1509                                 binaries.register(act,pkg,bin,
1510                                         'forbuilds',testbed.blamed)
1511                                 act.binaries.append((pkg,bin))
1512                                 break
1513                 debug_b('all done.')
1514
1515 #---------- main processing loop and main program
1516
1517 def process_actions():
1518         global binaries
1519
1520         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1521         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1522         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1523
1524         debug_a1('starting')
1525         testbed.open()
1526         binaries = Binaries()
1527
1528         for act in opts.actions:
1529                 testbed.register_ephemeral(act.af)
1530
1531         binaries.reset()
1532
1533         debug_a1('builds ...')
1534         for act in opts.actions:
1535                 debug_a2('%s %s' %
1536                         (act.kind, act.what))
1537
1538                 testbed.prepare([])
1539                 if act.kind == 'deb':
1540                         testbed.blame('arg:'+act.af.spec)
1541                         determine_package(act)
1542                         testbed.blame('deb:'+act.pkg)
1543                         binaries.register(act,act.pkg,act.af,
1544                                 'forbuilds',testbed.blamed)
1545                 if act.kind == 'dsc' or act.kind == 'ubtree':
1546                         build_source(act)
1547                 if act.kind == 'tree':
1548                         act.binaries = []
1549
1550         debug_a1('builds done.')
1551
1552         binaries.reset()
1553         control_override = None
1554
1555         debug_a1('tests ...')
1556         for act in opts.actions:
1557                 debug_a2('test %s %s' % (act.kind, act.what))
1558
1559                 testbed.needs_reset()
1560                 if act.kind == 'control':
1561                         control_override = act.af
1562                 if act.kind == 'deb':
1563                         binaries.register(act,act.pkg,act.af,'fortests',
1564                                 ['deb:'+act.pkg])
1565                 if act.kind == 'dsc' or act.kind == 'ubtree':
1566                         for (pkg,bin) in act.binaries:
1567                                 binaries.register(act,pkg,bin,'fortests',
1568                                         act.blamed)
1569                 if act.kind == 'dsc':
1570                         if act.ah['dsc_tests']:
1571                                 debug_a3('read control ...')
1572                                 stanzas = read_control(act, act.tests_tree,
1573                                                 control_override)
1574                                 testbed.blamed += act.blamed
1575                                 debug_a3('run_tests ...')
1576                                 run_tests(stanzas, act.tests_tree)
1577                         control_override = None
1578                 if act.kind == 'tree' or act.kind == 'ubtree':
1579                         testbed.blame('arg:'+act.af.spec)
1580                         stanzas = read_control(act, act.af, control_override)
1581                         debug_a3('run_tests ...')
1582                         run_tests(stanzas, act.af)
1583                         control_override = None
1584         debug_a1('tests done.')
1585
1586 def main():
1587         global testbed
1588         global tmpdir
1589         try:
1590                 parse_args()
1591         except SystemExit, se:
1592                 os._exit(20)
1593         try:
1594                 setup_trace()
1595                 testbed = Testbed()
1596                 testbed.start()
1597                 finalise_options()
1598                 process_actions()
1599         except:
1600                 ec = print_exception(sys.exc_info(), '')
1601                 cleanup()
1602                 os._exit(ec)
1603         cleanup()
1604         os._exit(errorcode)
1605
1606 main()