runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 binaries = None         # Binaries (.debs we have registered)
  49
  50 #---------- output handling
  51 #
  52 #  There are at least the following kinds of output:
  53 #
  54 #   stderr_stream
  55 #   1. stderr output which consists of
  56 #   1a. our errors
  57 #   1b. stuff printed to stderr by our direct subprocesses
  58 #         including the virtualisation server and testbed
  59 #         commands we don't expect to fail
  60 #
  61 #   trace_stream
  62 #   2. trace information, which consists of
  63 #   2a. our trace information from calls to debug()
  64 #   2b. progress information and stderr output from scripts
  65 #        we run on the host
  66 #   2c. progress information and stderr output from scripts
  67 #        we run on the testbed including tests and builds
  68 #
  69 #   xxxx
  70 #   3. actual test results (printed to our stdout)
  71 #
  72 # Capture of stderr output 1b and 2b is achieved by replumbing our
  73 # stderr if necessary.
  74
  75 stderr_stream = sys.stderr
  76 trace_stream = None
  77
  78 def pstderr(m):
  79         print >>stderr_stream, m
  80
  81 def debug(m):
  82         if trace_stream is None: return
  83         for l in m.rstrip('\n').split('\n'):
  84                 s = 'atd-run: debug: ' + l
  85                 print >>trace_stream, s
  86
  87 def report(tname, result):
  88         xxxx   xxxx
  89         m = '%-20s %s' % (tname, result)
  90         print m
  91         if trace_stream: print >>trace_stream, m
  92
  93 def debug_trace_shfrag(tmpdir, what, fd=None):
  94         if not opts.debug: return ''
  95         trace = "%s/%s-trace" % (tmpdir,what)
  96         if fd is None: fd = 2
  97         script = '''
  98                         rm -f -- "TRACE"
  99                         mkfifo -m600 "TRACE"
 100                         tee <"TRACE" /dev/stderr >&FD &
 101                         exec 2>"TRACE"
 102                 '''
 103         script = script.replace('TRACE',trace).replace('FD',`fd`)
 104         return script
 105
 106 #---------- errors we define
 107
 108 class Quit:
 109         def __init__(q,ec,m): q.ec = ec; q.m = m
 110
 111 def bomb(m, se=''):
 112         pstderr(se)
 113         raise Quit(20, "unexpected error: %s" % m)
 114
 115 def badpkg(m, se=''):
 116         pstderr(se)
 117         print 'blame: ', ' '.join(testbed.blamed)
 118         raise Quit(12, "erroneous package: %s" % m)
 119
 120 class Unsupported:
 121  def __init__(u, lno, m):
 122         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 123         else: u.m = m
 124  def report(u, tname):
 125         global errorcode
 126         errorcode != 2
 127         report(tname, 'SKIP %s' % u.m)
 128
 129 #---------- convenience function
 130
 131 def mkdir_okexist(pathname, mode=02755):
 132         try:
 133                 os.mkdir(pathname, mode)
 134         except OSError, oe:
 135                 if oe.errno != errno.EEXIST: raise
 136
 137 def rmtree(what, pathname):
 138         debug('/ %s rmtree %s' % (what, pathname))
 139         shutil.rmtree(pathname)
 140
 141 def debug_subprocess(what, cmdl=None, script=None):
 142         o = '$ '+what+':'
 143         if cmdl is not None:
 144                 ol = []
 145                 for x in cmdl:
 146                         if x is script: x = '<SCRIPT>'
 147                         ol.append(x.    replace('\\','\\\\').
 148                                         replace(' ','\\ ')      )
 149                 o += ' '+ ' '.join(ol)
 150         if script is not None:
 151                 o += '\n'
 152                 for l in script.rstrip('\n').split('\n'):
 153                         o += '$     '+l+'\n'
 154         debug(o)
 155
 156 def flatten(l):
 157         return reduce((lambda a,b: a + b), l, [])
 158
 159 #---------- fancy automatic file-copying class
 160
 161 class AutoFile:
 162         # p.what
 163         # p.path[tb]    None or path    not None => path known
 164         # p.file[tb]    None or path    not None => file exists
 165         # p.spec        string or None
 166         # p.spec_tbp    True or False, or not set if spec is None
 167         # p.dir         '' or '/'
 168
 169  def __init__(p, what):
 170         p.what = what
 171         p.path = [None,None]
 172         p.file = [None,None]
 173         p.spec = None
 174         p.dir = ''
 175
 176  def __str__(p):
 177         def ptbp(tbp):
 178                 if p.path[tbp] is None: return '-'+p.dir
 179                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 180                 else: return p.path[tbp]+p.dir+'!'
 181         out = p.what
 182         if p.spec is not None:
 183                 if p.spec_tbp: out += '#'
 184                 else: out += '='
 185                 out += p.spec
 186         out += ':'
 187         out += ptbp(False)
 188         out += '|'
 189         out += ptbp(True)
 190         return out
 191
 192  def _wrong(p, how):
 193         xtra = ''
 194         if p.spec is not None: xtra = ' spec[%s]=%s' % (p.spec, p.spec_tb)
 195         raise ("internal error: %s (%s)" % (how, str(p)))
 196
 197  def _ensure_path(p, tbp):
 198         if p.path[tbp] is None:
 199                 if '/' in p.what:
 200                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 201                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 202                 if not tbp:
 203                         p.path[tbp] = tmpdir+'/'+p.what
 204                 else:
 205                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 206
 207  def write(p, tbp=False):
 208         p._debug('write %s...' % 'HT'[tbp])
 209         p._ensure_path(tbp)
 210
 211         if p.dir and not p.file[tbp]:
 212                 if not tbp:
 213                         p._debug('mkdir H')
 214                         mkdir_okexist(p.path[tbp])
 215                 else:
 216                         cmdl = ['sh','-ec',
 217                                 'test -d "$1" || mkdir -p "$1"',
 218                                 'x', p.path[tbp]]
 219                         tf_what = urllib.quote(p.what).replace('/','%2F')
 220                         (rc,se) = testbed.execute('mkdir-'+tf_what, cmdl)
 221                         if rc: bomb('failed to create directory %s' %
 222                                 p.path[tbp], se)
 223
 224         p.file[tbp] = p.path[tbp]
 225         return p.path[tbp]
 226
 227  def read(p, tbp=False):
 228         p._debug('read %s...' % 'HT'[tbp])
 229         p._ensure_path(tbp)
 230
 231         if p.file[tbp] is None:
 232                 if p.file[not tbp] is None:
 233                         p._wrong("requesting read but nonexistent")
 234                 cud = ['copyup','copydown'][tbp]
 235                 src = p.file[not tbp] + p.dir
 236                 dst = p.path[tbp] + p.dir
 237                 testbed.command(cud, (src, dst))
 238                 p.file[tbp] = p.path[tbp]
 239
 240         return p.file[tbp] + p.dir
 241
 242  def invalidate(p, tbp=False):
 243         p.file[tbp] = None
 244         p._debug('invalidated %s' % 'HT'[tbp])
 245
 246  def _debug(p, m):
 247         debug('/ %s#%x: %s' % (p.what, id(p), m))
 248
 249  def _constructed(p):
 250         p._debug('constructed: '+str(p))
 251         p._check()
 252
 253  def _check(p):
 254         for tbp in [False,True]:
 255          for pf in [p.path, p.file]:
 256                 if pf[tbp] is None: continue
 257                 if not pf[tbp]: bomb('empty path specified for '+p.what)
 258                 if p.dir and pf[tbp].endswith('/'):
 259                         pf[tbp] = pf[tbp].rstrip('/')
 260                         if not pf[tbp]: pf[tbp] = '/'
 261                 if not p.dir and pf[tbp].endswith('/'):
 262                         bomb("directory `%s' specified for "
 263                              "non-directory %s" % (pf[tbp], p.what))
 264
 265  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 266         AutoFile.__init__(p,what)
 267         sh_on = ''; sh_sibl = ''
 268         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 269         if sibling: sh_sibl=' (sibling)'
 270         parent._debug('using as base: %s: %s%s%s' %
 271                         (str(parent), leaf, sh_on, sh_sibl))
 272         if not sibling and not parent.dir:
 273                 parent._wrong('asked for non-sibling relative path of non-dir')
 274         if sibling: trim = os.path.dirname
 275         else: trim = lambda x: x
 276         for tbp in [False,True]:
 277                 if parent.path[tbp] is None: continue
 278                 trimmed = trim(parent.path[tbp])
 279                 if trimmed: trimmed += '/'
 280                 p.path[tbp] = trimmed + leaf
 281                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 282                         p.file[tbp] = p.path[tbp]
 283
 284 class InputFile(AutoFile):
 285  def _init(p, what, spec, spec_tbp=False):
 286         AutoFile.__init__(p, what)
 287         p.spec = spec
 288         p.spec_tbp = spec_tbp
 289         p.path[spec_tbp] = spec
 290         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 291  def __init__(p, what, spec, spec_tbp=False):
 292         p._init(what,spec,spec_tbp)
 293         p._constructed()
 294
 295 class InputDir(InputFile):
 296  def __init__(p, what, spec, spec_tbp=False):
 297         InputFile._init(p,what,spec,spec_tbp)
 298         p.dir = '/'
 299         p._constructed()
 300
 301 class OutputFile(AutoFile):
 302  def _init(p, what, spec, spec_tbp=False):
 303         AutoFile.__init__(p, what)
 304         p.spec = spec
 305         p.spec_tbp = spec_tbp
 306         p.path[spec_tbp] = spec
 307  def __init__(p, what, spec, spec_tbp=False):
 308         p._init(what,spec,spec_tbp)
 309         p._constructed()
 310
 311 class OutputDir(OutputFile):
 312  def __init__(p, what, spec, spec_tbp=False):
 313         OutputFile._init(p,what,spec,spec_tbp)
 314         p.dir = '/'
 315         p._constructed()
 316
 317 class RelativeInputFile(AutoFile):
 318  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 319         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 320         p._constructed()
 321
 322 class RelativeOutputFile(AutoFile):
 323  def __init__(p, what, parent, leaf, sibling=False):
 324         p._relative_init(what, parent, leaf, None, False, sibling)
 325         p._constructed()
 326
 327 class TemporaryFile(AutoFile):
 328  def __init__(p, what):
 329         AutoFile.__init__(p, what)
 330         p._constructed()
 331
 332 class TemporaryDir(AutoFile):
 333  def __init__(p, what):
 334         AutoFile.__init__(p, what)
 335         p.dir = '/'
 336         p._constructed()
 337
 338 #---------- parsing and representation of the arguments
 339
 340 class Action:
 341  def __init__(a, kind, af, arghandling, what):
 342         # extra attributes get added during processing
 343         a.kind = kind
 344         a.af = af
 345         a.ah = arghandling
 346         a.what = what
 347
 348 def parse_args():
 349         global opts
 350         usage = "%prog <options> --- <virt-server>..."
 351         parser = OptionParser(usage=usage)
 352         pa = parser.add_option
 353         pe = parser.add_option
 354
 355         arghandling = {
 356                 'dsc_tests': True,
 357                 'dsc_filter': '*',
 358                 'deb_forbuilds': 'auto',
 359                 'deb_fortests': 'auto',
 360                 'tb': False,
 361                 'override_control': None
 362         }
 363         initial_arghandling = arghandling.copy()
 364         n_non_actions = 0
 365
 366         #----------
 367         # actions (ie, test sets to run, sources to build, binaries to use):
 368
 369         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 370                 parser.largs.append((value,kindpath))
 371                 n_non_actions += not(is_act)
 372
 373         def pa_action(long, metavar, kindpath, help, is_act=True):
 374                 pa('','--'+long, action='callback', callback=cb_action,
 375                         nargs=1, type='string',
 376                         callback_args=(long,kindpath,is_act), help=help)
 377
 378         pa_action('build-tree',         'TREE', '@/',
 379                 help='run tests from build tree TREE')
 380
 381         pa_action('source',             'DSC', '@.dsc',
 382                 help='build DSC and use its tests and/or'
 383                     ' generated binary packages')
 384
 385         pa_action('binary',             'DEB', '@.deb',
 386                help='use binary package DEB according'
 387                     ' to most recent --binaries-* settings')
 388
 389         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 390                help='run tests from control file CONTROL instead,'
 391                     ' (applies to next test suite only)')
 392
 393         #----------
 394         # argument handling settings (what ways to use action
 395         #  arguments, and pathname processing):
 396
 397         def cb_setah(option, opt_str, value, parser, toset,setval):
 398                 if type(setval) == list:
 399                         if not value in setval:
 400                                 parser.error('value for %s option (%s) is not '
 401                                  'one of the permitted values (%s)' %
 402                                  (value, opt_str, setval.join(' ')))
 403                 elif setval is not None:
 404                         value = setval
 405                 for v in toset:
 406                         arghandling[v] = value
 407                 parser.largs.append(arghandling.copy())
 408
 409         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 410                 type = metavar
 411                 if type is not None: type = 'string'
 412                 pa('',long, action='callback', callback=cb_setah,
 413                    callback_args=(affected,effect), **kwargs)
 414
 415         #---- paths: host or testbed:
 416         #
 417         pa_setah('--paths-testbed', ['tb'],True,
 418                 help='subsequent path specifications refer to the testbed')
 419         pa_setah('--paths-host', ['tb'],False,
 420                 help='subsequent path specifications refer to the host')
 421
 422         #---- source processing settings:
 423
 424         pa_setah('--sources-tests', ['dsc_tests'],True,
 425                 help='run tests from builds of subsequent sources')
 426         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 427                 help='do not run tests from builds of subsequent sources')
 428
 429         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 430                 type='string', metavar='PATTERN-LIST',
 431                 help='from subsequent sources, use binaries matching'
 432                      ' PATTERN-LIST (comma-separated glob patterns)'
 433                      ' according to most recent --binaries-* settings')
 434         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 435                 help='from subsequent sources, do not use any binaries')
 436
 437         #---- binary package processing settings:
 438
 439         def pa_setahbins(long,toset,how):
 440          pa_setah(long, toset,['ignore','auto','install'],
 441                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 442                 help=how+' ignore binaries, install them as needed'
 443                         ' for dependencies, or unconditionally install'
 444                         ' them, respectively')
 445         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 446         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 447         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 448
 449         #----------
 450         # general options:
 451
 452         def cb_vserv(op,optstr,value,parser):
 453                 parser.values.vserver = list(parser.rargs)
 454                 del parser.rargs[:]
 455
 456         def cb_path(op,optstr,value,parser, constructor,long,dir):
 457                 name = long.replace('-','_')
 458                 af = constructor(arghandling['tb'], value, long, dir)
 459                 setattr(parser.values, name, af)
 460
 461         def pa_path(long, constructor, help, dir=False):
 462                 pa('','--'+long, action='callback', callback=cb_path,
 463                         callback_args=(constructor,long,dir),
 464                         nargs=1, type='string',
 465                         help=help, metavar='PATH')
 466
 467         pa_path('output-dir', OutputDir, dir=True,
 468                 help='write stderr/out files in PATH')
 469
 470         pa('','--tmp-dir',              type='string', dest='tmpdir',
 471                 help='write temporary files to TMPDIR, emptying it'
 472                      ' beforehand and leaving it behind at the end')
 473
 474         pa('','--user',                 type='string', dest='user',
 475                 help='run tests as USER (needs root on testbed)')
 476         pa('','--gain-root',            type='string', dest='gainroot',
 477                 help='prefix debian/rules binary with GAINROOT')
 478         pa('-q', '--quiet', action='store_false', dest='debug', default=True);
 479         pa('','--gnupg-home',           type='string', dest='gnupghome',
 480                 default='~/.autopkgtest/gpg',
 481                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 482                         " signing private apt archive);"
 483                         " `fresh' means generate new key each time.")
 484
 485         #----------
 486         # actual meat:
 487
 488         class SpecialOption(optparse.Option): pass
 489         vs_op = SpecialOption('','--VSERVER-DUMMY')
 490         vs_op.action = 'callback'
 491         vs_op.type = None
 492         vs_op.default = None
 493         vs_op.nargs = 0
 494         vs_op.callback = cb_vserv
 495         vs_op.callback_args = ( )
 496         vs_op.callback_kwargs = { }
 497         vs_op.help = 'introduces virtualisation server and args'
 498         vs_op._short_opts = []
 499         vs_op._long_opts = ['---']
 500
 501         pa(vs_op)
 502
 503         (opts,args) = parser.parse_args()
 504         if not hasattr(opts,'vserver'):
 505                 parser.error('you must specifiy --- <virt-server>...')
 506         if n_non_actions >= len(parser.largs):
 507                 parser.error('nothing to do specified')
 508
 509         arghandling = initial_arghandling
 510         opts.actions = []
 511         ix = 0
 512         for act in args:
 513                 if type(act) == dict:
 514                         arghandling = act
 515                         continue
 516                 elif type(act) == tuple:
 517                         pass
 518                 elif type(act) == str:
 519                         act = (act,act)
 520                 else:
 521                         raise ("unknown action in list `%s' having"
 522                               " type `%s'" % (act, type(act)))
 523                 (pathstr, kindpath) = act
 524
 525                 constructor = InputFile
 526                 if type(kindpath) is tuple:             kind = kindpath[0]
 527                 elif kindpath.endswith('.deb'):         kind = 'deb'
 528                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 529                 elif kindpath.endswith('/'):
 530                         kind = 'tree'
 531                         constructor = InputDir
 532                 else: parser.error("do not know how to handle filename \`%s';"
 533                         " specify --source --binary or --build-tree")
 534
 535                 what = '%s%s' % (kind,ix); ix += 1
 536
 537                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 538                 else: fwhatx = '-'+kind
 539
 540                 af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 541                 opts.actions.append(Action(kind, af, arghandling, what))
 542
 543 def setup_trace():
 544         global debug_stream, tmpdir
 545
 546         if opts.tmpdir is not None:
 547                 rmtree('tmpdir(specified)',opts.tmpdir)
 548                 mkdir_okexist(opts.tmpdir, 0700)
 549                 tmpdir = opts.tmpdir
 550         else:
 551                 assert(tmpdir is None)
 552                 tmpdir = tempfile.mkdtemp()
 553
 554         if opts.debug:
 555                 debug_stream = sys.stderr
 556         elif opts.tmpdir is not None:
 557                 debug_stream = open(opts.tmpdir+'/trace', 'a', 0)
 558
 559 def finalise_options():
 560         global opts, tb
 561
 562         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 563                 opts.user = ''
 564
 565         if opts.user is None:
 566                 su = 'suggested-normal-user='
 567                 ul = [
 568                         e[len(su):]
 569                         for e in testbed.caps
 570                         if e.startswith(su)
 571                         ]
 572                 if ul:
 573                         opts.user = ul[0]
 574                 else:
 575                         opts.user = ''
 576
 577         if opts.user:
 578                 if 'root-on-testbed' not in testbed.caps:
 579                         pstderr("warning: virtualisation"
 580                                 " system does not offer root on testbed,"
 581                                 " but --user option specified: failure likely")
 582                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 583         else:
 584                 opts.user_wrap = lambda x: x
 585
 586         if opts.gainroot is None:
 587                 opts.gainroot = ''
 588                 if (opts.user or
 589                     'root-on-testbed' not in testbed.caps):
 590                         opts.gainroot = 'fakeroot'
 591
 592         if opts.gnupghome.startswith('~/'):
 593                 try: home = os.environ['HOME']
 594                 except KeyError:
 595                         parser.error("HOME environment variable"
 596                                 " not set, needed for --gnupghome=`%s"
 597                                 % opts.gnupghome)
 598                 opts.gnupghome = home + opts.gnupghome[1:]
 599         elif opts.gnupghome == 'fresh':
 600                 opts.gnupghome = None
 601
 602 #---------- testbed management - the Testbed class
 603
 604 class Testbed:
 605  def __init__(tb):
 606         tb.sp = None
 607         tb.lastsend = None
 608         tb.scratch = None
 609         tb.modified = False
 610         tb.blamed = []
 611         tb._ephemeral = []
 612         tb._debug('init')
 613  def _debug(tb, m):
 614         debug('** '+m)
 615  def start(tb):
 616         tb._debug('start')
 617         p = subprocess.PIPE
 618         debug_subprocess('vserver', opts.vserver)
 619         tb.sp = subprocess.Popen(opts.vserver,
 620                 stdin=p, stdout=p, stderr=None)
 621         tb.expect('ok')
 622         tb.caps = tb.commandr('capabilities')
 623         tb._need_reset_apt = False
 624  def stop(tb):
 625         tb._debug('stop')
 626         tb.close()
 627         if tb.sp is None: return
 628         ec = tb.sp.returncode
 629         if ec is None:
 630                 tb.sp.stdout.close()
 631                 tb.send('quit')
 632                 tb.sp.stdin.close()
 633                 ec = tb.sp.wait()
 634         if ec:
 635                 tb.bomb('testbed gave exit status %d after quit' % ec)
 636  def open(tb):
 637         tb._debug('open, scratch=%s' % tb.scratch)
 638         if tb.scratch is not None: return
 639         pl = tb.commandr('open')
 640         tb.scratch = InputDir('tb-scratch', pl[0], True)
 641         tb.deps_processed = []
 642  def mungeing_apt(tb):
 643         if not 'revert' in tb.caps:
 644                 tb._need_reset_apt = True
 645  def reset_apt(tb):
 646         if not tb._need_reset_apt: return
 647         what = 'aptget-update-reset'
 648         cmdl = ['apt-get','-qy','update']
 649         (rc,se) = tb.execute(what, cmdl)
 650         if rc:
 651                 pstderr("\n" "warning: failed to restore"
 652                         " testbed apt cache, exit code %d" % rc)
 653         tb._need_reset_apt = False
 654  def close(tb):
 655         tb._debug('close, scratch=%s' % tb.scratch)
 656         if tb.scratch is None: return
 657         tb.scratch = None
 658         if tb.sp is None: return
 659         tb.command('close')
 660  def prepare(tb, deps_new):
 661         tb._debug('prepare, modified=%s, deps_processed=%s, deps_new=%s' %
 662                 (tb.modified, tb.deps_processed, deps_new))
 663         if 'revert' in tb.caps and (tb.modified or
 664             [d for d in tb.deps_processed if d not in deps_new]):
 665                 tb._debug('reset **')
 666                 tb.command('revert')
 667                 tb.blamed = []
 668                 for af in tb._ephemeral: af.invalidate(True)
 669         binaries.publish()
 670         tb.modified = False
 671         tb._install_deps(deps_new)
 672  def register_ephemeral(tb, af):
 673         tb._ephemeral.append(af)
 674  def _install_deps(tb, deps_new):
 675         tb._debug(' installing dependencies '+`deps_new`)
 676         tb.deps_processed = deps_new
 677         if not deps_new: return
 678         dstr = ', '.join(deps_new)
 679         script = binaries.apt_pkg_gdebi_script(
 680                 dstr, [[
 681                 'from GDebi.DebPackage import DebPackage',
 682                 'd = DebPackage(cache)',
 683                 'res = d.satisfyDependsStr(arg)',
 684                 ]])
 685         cmdl = ['python','-c',script]
 686         what = 'install-deps'
 687         debug_subprocess(what, cmdl, script=script)
 688         (rc,se) = testbed.execute(what, cmdl)
 689         if rc: badpkg('dependency install failed, exit code %d' % rc, se)
 690  def needs_reset(tb):
 691         tb._debug('needs_reset, previously=%s' % tb.modified)
 692         tb.modified = True
 693  def blame(tb, m):
 694         tb._debug('blame += %s' % m)
 695         tb.blamed.append(m)
 696  def bomb(tb, m):
 697         tb._debug('bomb %s' % m)
 698         if tb.sp is not None:
 699                 tb.sp.stdout.close()
 700                 tb.sp.stdin.close()
 701                 ec = tb.sp.wait()
 702                 if ec: pstderr('adt-run: testbed failing,'
 703                                 ' exit status %d' % ec)
 704         tb.sp = None
 705         raise Quit(16, 'testbed failed: %s' % m)
 706  def send(tb, string):
 707         tb.sp.stdin
 708         try:
 709                 debug('>> '+string)
 710                 print >>tb.sp.stdin, string
 711                 tb.sp.stdin.flush()
 712                 tb.lastsend = string
 713         except:
 714                 (type, value, dummy) = sys.exc_info()
 715                 tb.bomb('cannot send to testbed: %s' % traceback.
 716                         format_exception_only(type, value))
 717  def expect(tb, keyword, nresults=None):
 718         l = tb.sp.stdout.readline()
 719         if not l: tb.bomb('unexpected eof from the testbed')
 720         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 721         l = l.rstrip('\n')
 722         debug('<< '+l)
 723         ll = l.split()
 724         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 725         if ll[0] != keyword:
 726                 if tb.lastsend is None:
 727                         tb.bomb("got banner `%s', expected `%s...'" %
 728                                 (l, keyword))
 729                 else:
 730                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 731                                 (tb.lastsend, l, keyword))
 732         ll = ll[1:]
 733         if nresults is not None and len(ll) != nresults:
 734                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 735                         " expected %d result parameters" %
 736                         (string, l, len(ll), nresults))
 737         return ll
 738  def commandr(tb, cmd, args=(), nresults=None):
 739         # pass args=[None,...] or =(None,...) to avoid more url quoting
 740         if type(cmd) is str: cmd = [cmd]
 741         if len(args) and args[0] is None: args = args[1:]
 742         else: args = map(urllib.quote, args)
 743         al = cmd + args
 744         tb.send(string.join(al))
 745         ll = tb.expect('ok', nresults)
 746         rl = map(urllib.unquote, ll)
 747         return rl
 748  def command(tb, cmd, args=()):
 749         tb.commandr(cmd, args, 0)
 750  def commandr1(tb, cmd, args=()):
 751         rl = tb.commandr(cmd, args, 1)
 752         return rl[0]
 753  def execute(tb, what, cmdargs,
 754                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 755                 dump_fd=None):
 756         xxx make this right
 757         if cwd is None: cwd = tb.scratch.write(True)
 758         se_use = se
 759         if se_use is None:
 760                 se_af = TemporaryFile('xerr-'+what)
 761                 se_use = se_af.write(True)
 762         cmdl = [None,
 763                 ','.join(map(urllib.quote, cmdargs)),
 764                 si, so, se_use, cwd]
 765         if debug_stream and dump_fd is not None:
 766                 cmdl += ['debug=%d-%d' % (dump_fd,debug_stream.fileno())]
 767         rc = tb.commandr1('execute', cmdl)
 768         try: rc = int(rc)
 769         except ValueError: bomb("execute for %s gave invalid response `%s'"
 770                                         % (what,rc))
 771         if se is not None: return rc
 772         return (rc, open(se_af.read()).read())
 773
 774 #---------- representation of test control files: Field*, Test, etc.
 775
 776 class FieldBase:
 777  def __init__(f, fname, stz, base, tnames, vl):
 778         assert(vl)
 779         f.stz = stz
 780         f.base = base
 781         f.tnames = tnames
 782         f.vl = vl
 783  def words(f):
 784         def distribute(vle):
 785                 (lno, v) = vle
 786                 r = v.split()
 787                 r = map((lambda w: (lno, w)), r)
 788                 return r
 789         return flatten(map(distribute, f.vl))
 790  def atmostone(f):
 791         if len(vl) == 1:
 792                 (f.lno, f.v) = vl[0]
 793         else:
 794                 raise Unsupported(f.vl[1][0],
 795                         'only one %s field allowed' % fn)
 796         return f.v
 797
 798 class FieldIgnore(FieldBase):
 799  def parse(f): pass
 800
 801 class Restriction:
 802  def __init__(r,rname,base): pass
 803
 804 class Restriction_rw_build_tree(Restriction): pass
 805 class Restriction_rw_tests_tree(Restriction): pass
 806 class Restriction_breaks_testbed(Restriction):
 807  def __init__(r, rname, base):
 808         if 'revert' not in testbed.caps:
 809                 raise Unsupported(f.lno,
 810                         'Test breaks testbed but testbed cannot revert')
 811
 812 class Field_Restrictions(FieldBase):
 813  def parse(f):
 814         for wle in f.words():
 815                 (lno, rname) = wle
 816                 rname = rname.replace('-','_')
 817                 try: rclass = globals()['Restriction_'+rname]
 818                 except KeyError: raise Unsupported(lno,
 819                         'unknown restriction %s' % rname)
 820                 r = rclass(rname, f.base)
 821                 f.base['restrictions'].append(r)
 822
 823 class Field_Tests(FieldIgnore): pass
 824
 825 class Field_Depends(FieldBase):
 826  def parse(f):
 827         dl = map(lambda x: x.strip(),
 828                 flatten(map(lambda x: x.split(','), f.vl)))
 829         re = regexp.compile('[^-.+:~]')
 830         for d in dl:
 831                 if re.search(d):
 832                         badpkg("Test Depends field contains dependency"
 833                                " `%s' with invalid characters" % d)
 834         f.base['depends'] = dl
 835
 836 class Field_Tests_directory(FieldBase):
 837  def parse(f):
 838         td = atmostone(f)
 839         if td.startswith('/'): raise Unspported(f.lno,
 840                 'Tests-Directory may not be absolute')
 841         f.base['testsdir'] = td
 842
 843 def run_tests(stanzas, tree):
 844         global errorcode, testbed
 845         for stanza in stanzas:
 846                 tests = stanza[' tests']
 847                 if not tests:
 848                         report('*', 'SKIP no tests in this package')
 849                         errorcode |= 8
 850                 for t in tests:
 851                         t.prepare()
 852                         t.run(tree)
 853                         if 'breaks-testbed' in t.restrictions:
 854                                 testbed.needs_reset()
 855                 testbed.needs_reset()
 856
 857 class Test:
 858  def __init__(t, tname, base, act):
 859         if '/' in tname: raise Unsupported(base[' lno'],
 860                 'test name may not contain / character')
 861         for k in base: setattr(t,k,base[k])
 862         t.tname = tname
 863         t.act = act
 864         t.what = act.what+'t-'+tname
 865         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
 866         else: t.path = tname
 867         t._debug('constructed; path=%s' % t.path)
 868         t._debug(' .depends=%s' % t.depends)
 869  def _debug(t, m):
 870         debug('& %s: %s' % (t.what, m))
 871  def report(t, m):
 872         report(t.what, m)
 873  def reportfail(t, m):
 874         global errorcode
 875         errorcode |= 4
 876         report(t.what, 'FAIL ' + m)
 877  def prepare(t):
 878         t._debug('preparing')
 879         dn = []
 880         for d in t.depends:
 881                 t._debug(' processing dependency '+d)
 882                 if not '*' in d:
 883                         t._debug('  literal dependency '+d)
 884                         dn.append(d)
 885                 else:
 886                         for (pkg,bin) in t.act.binaries:
 887                                 d = d.replace('*',pkg)
 888                                 t._debug('  synthesised dependency '+d)
 889                                 dn.append(d)
 890         testbed.prepare(dn)
 891  def run(t, tree):
 892         t._debug('running')
 893         def stdouterr(oe):
 894                 idstr = t.what + '-' + oe
 895                 if opts.output_dir is not None and opts.output_dir.tb:
 896                         use_dir = opts.output_dir
 897                 else:
 898                         use_dir = testbed.scratch
 899                 return RelativeOutputFile(idstr, use_dir, idstr)
 900
 901         t.act.work.write(True)
 902
 903         af = RelativeInputFile(t.what, tree, t.path)
 904         so = stdouterr('stdout')
 905         se = stdouterr('stderr')
 906         rc = testbed.execute('test-'+t.what,
 907                 [opts.user_wrap(af.read(True))],
 908                 so=so.write(True), se=se.write(True), cwd=tree.read(True))
 909
 910         stab = os.stat(se.read())
 911         if stab.st_size != 0:
 912                 l = open(se.read()).readline()
 913                 l = l.rstrip('\n \t\r')
 914                 if len(l) > 40: l = l[:40] + '...'
 915                 t.reportfail('status: %d, stderr: %s' % (rc, l))
 916         elif rc != 0:
 917                 t.reportfail('non-zero exit status %d' % rc)
 918         else:
 919                 t.report('PASS')
 920
 921 def read_control(act, tree, control_override):
 922         stanzas = [ ]
 923
 924         if control_override is not None:
 925                 control_af = control_override
 926                 testbed.blame('arg:'+control_override.spec)
 927         else:
 928                 control_af = RelativeInputFile(act.what+'-testcontrol',
 929                         tree, 'debian/tests/control')
 930         try:
 931                 control = open(control_af.read(), 'r')
 932         except OSError, oe:
 933                 if oe[0] != errno.ENOENT: raise
 934                 return []
 935
 936         lno = 0
 937         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
 938         stz = None      # stz[field_name][index] = (lno, value)
 939                         # special field names:
 940                         # stz[' lno'] = number
 941                         # stz[' tests'] = list of Test objects
 942         def end_stanza(stz):
 943                 if stz is None: return
 944                 stz[' errs'] = 0
 945                 stanzas.append(stz)
 946                 stz = None
 947                 hcurrent = None
 948
 949         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
 950         while 1:
 951                 l = control.readline()
 952                 if not l: break
 953                 lno += 1
 954                 if not l.endswith('\n'): badctrl('unterminated line')
 955                 if regexp.compile('\s*\#').match(l): continue
 956                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
 957                 initmat = initre.match(l)
 958                 if initmat:
 959                         (fname, l) = initmat.groups()
 960                         fname = string.capwords(fname)
 961                         if stz is None:
 962                                 stz = { ' lno': lno, ' tests': [] }
 963                         if not stz.has_key(fname): stz[fname] = [ ]
 964                         hcurrent = stz[fname]
 965                 elif regexp.compile('\s').match(l):
 966                         if not hcurrent: badctrl('unexpected continuation')
 967                 else:
 968                         badctrl('syntax error')
 969                 hcurrent.append((lno, l))
 970         end_stanza(stz)
 971
 972         def testbadctrl(stz, lno, m):
 973                 report_badctrl(lno, m)
 974                 stz[' errs'] += 1
 975
 976         for stz in stanzas:
 977                 try:
 978                         try: tnames = stz['Tests']
 979                         except KeyError:
 980                                 tnames = ['*']
 981                                 raise Unsupported(stz[' lno'],
 982                                         'no Tests field')
 983                         tnames = map((lambda lt: lt[1]), tnames)
 984                         tnames = string.join(tnames).split()
 985                         base = {
 986                                 'restrictions': [],
 987                                 'testsdir': 'debian/tests',
 988                                 'depends' : '*'
 989                         }
 990                         for fname in stz.keys():
 991                                 if fname.startswith(' '): continue
 992                                 vl = stz[fname]
 993                                 try: fclass = globals()['Field_'+
 994                                         fname.replace('-','_')]
 995                                 except KeyError: raise Unsupported(vl[0][0],
 996                                         'unknown metadata field %s' % fname)
 997                                 f = fclass(stz, fname, base, tnames, vl)
 998                                 f.parse()
 999                         for tname in tnames:
1000                                 t = Test(tname, base, act)
1001                                 stz[' tests'].append(t)
1002                 except Unsupported, u:
1003                         for tname in tnames: u.report(tname)
1004                         continue
1005
1006         return stanzas
1007
1008 def print_exception(ei, msgprefix=''):
1009         if msgprefix: pstderr(msgprefix)
1010         (et, q, tb) = ei
1011         if et is Quit:
1012                 pstderr('adt-run: ' + q.m)
1013                 return q.ec
1014         else:
1015                 pstderr("adt-run: unexpected, exceptional, error:")
1016                 traceback.print_exc(None, sys.stderr)
1017                 if debug_stream: traceback.print_exc(None, debug_stream)
1018                 return 20
1019
1020 def cleanup():
1021         global debug_stream
1022         try:
1023                 rm_ec = 0
1024                 if opts.tmpdir is None and tmpdir is not None:
1025                         rmtree('tmpdir', tmpdir)
1026                 if testbed is not None:
1027                         testbed.reset_apt()
1028                         testbed.stop()
1029                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1030                 if debug_stream and debug_stream is not sys.stderr:
1031                         debug_stream.close()
1032                         debug_stream = None
1033         except:
1034                 print_exception(sys.exc_info(),
1035                         '\nadt-run: error cleaning up:\n')
1036                 os._exit(20)
1037
1038 #---------- registration, installation etc. of .deb's: Binaries
1039
1040 def determine_package(act):
1041         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1042         running = subprocess.Popen(cmd, stdout=subprocess.PIPE)
1043         output = running.communicate()[0]
1044         rc = running.wait()
1045         if rc: badpkg('failed to parse binary package, code %d' % rc)
1046         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1047         act.pkg = None
1048         for l in output.split('\n'):
1049                 m = re.match(l)
1050                 if not m: continue
1051                 if act.pkg: badpkg('two Package: lines in control file')
1052                 act.pkg = m.groups()[0]
1053         if not act.pkg: badpkg('no good Package: line in control file')
1054
1055 class Binaries:
1056  def __init__(b):
1057         b.dir = TemporaryDir('binaries')
1058         b.dir.write()
1059         ok = False
1060
1061         if opts.gnupghome is None:
1062                 opts.gnupghome = tmpdir+'/gnupg'
1063
1064         b._debug('initialising')
1065         try:
1066                 for x in ['pubring','secring']:
1067                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1068                 ok = True
1069         except OSError, oe:
1070                 if oe.errno != errno.ENOENT: raise
1071
1072         if ok: b._debug('no key generation needed')
1073         else: b.genkey()
1074
1075  def _debug(b, s):
1076         debug('* '+s)
1077
1078  def genkey(b):
1079         b._debug('preparing for key generation')
1080
1081         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1082         mkdir_okexist(opts.gnupghome, 0700)
1083
1084         script = debug_trace_shfrag(tmpdir, 'genkey') + '''
1085   exec >&2
1086   cd "$1"
1087   cat <<"END" >key-gen-params
1088 Key-Type: DSA
1089 Key-Length: 1024
1090 Key-Usage: sign
1091 Name-Real: autopkgtest per-run key
1092 Name-Comment: do not trust this key
1093 Name-Email: autopkgtest@example.com
1094 END
1095   set -x
1096   gpg --homedir="$1" --batch --gen-key key-gen-params
1097                 '''
1098         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1099         debug_subprocess('genkey', cmdl, script=script)
1100         rc = subprocess.call(cmdl)
1101         if rc:
1102                 try:
1103                         f = open(opts.gnupghome+'/key-gen-log')
1104                         tp = file.read()
1105                 except OSError, e: tp = e
1106                 pstderr(tp)
1107                 bomb('key generation failed, code %d' % rc)
1108
1109  def apt_configs(b):
1110         return {
1111                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1112         }
1113
1114  def apt_pkg_gdebi_script(b, arg, middle):
1115         script = [
1116                 'import apt_pkg',
1117                 'import urllib',
1118                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1119                 ]
1120         for (k,v) in b.apt_configs().iteritems():
1121                 v = urllib.quote(v)
1122                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1123                                 % (k, v))
1124         script += [
1125                 'from GDebi.Cache import Cache',
1126                 'cache = Cache()',
1127                 ]
1128         for m in middle:
1129                 script += m + [
1130                 'print res',
1131                 'print d.missingDeps',
1132                 'print d.requiredChanges',
1133                 'assert(res)',
1134                 'cache.commit()',
1135                 ''
1136                 ]
1137         return '\n'.join(script)
1138  def apt_get(b):
1139         ag = ['apt-get','-qy']
1140         for kv in b.apt_configs().iteritems():
1141                 ag += ['-o', '%s=%s' % kv]
1142         return ag
1143
1144  def reset(b):
1145         b._debug('reset')
1146         rmtree('binaries', b.dir.read())
1147         b.dir.invalidate()
1148         b.dir.write()
1149         b.install = []
1150         b.blamed = []
1151         b.registered = set()
1152
1153  def register(b, act, pkg, af, forwhat, blamed):
1154         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1155                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1156
1157         if act.ah['deb_'+forwhat] == 'ignore': return
1158
1159         b.blamed += testbed.blamed
1160
1161         leafname = pkg+'.deb'
1162         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1163
1164         try: os.remove(dest.write())
1165         except OSError, oe:
1166                 if oe.errno != errno.ENOENT: raise e
1167
1168         try: os.link(af.read(), dest.write())
1169         except OSError, oe:
1170                 if oe.errno != errno.EXDEV: raise e
1171                 shutil.copy(af.read(), dest)
1172
1173         if act.ah['deb_'+forwhat] == 'install':
1174                 b.install.append(pkg)
1175
1176         b.registered.add(pkg)
1177
1178  def publish(b):
1179         b._debug('publish')
1180
1181         script = debug_trace_shfrag(tmpdir, 'ftparchive') + '''
1182   exec >&2
1183   cd "$1"
1184   apt-ftparchive packages . >Packages
1185   gzip <Packages >Packages.gz
1186   apt-ftparchive release . >Release
1187   rm -f Release.gpg
1188   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1189   gpg --homedir="$2" --batch --export >archive-key.pgp
1190                 '''
1191         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1192         debug_subprocess('ftparchive', cmdl, script)
1193         rc = subprocess.call(cmdl)
1194         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1195
1196         b.dir.invalidate(True)
1197         apt_source = b.dir.read(True)
1198
1199         so = TemporaryFile('vlds')
1200         script = debug_trace_shfrag(testbed.scratch.write(True),
1201                         'apt-key', 4) + '''
1202   exec 3>&1 >&2
1203   apt-key add archive-key.pgp
1204   echo "deb file://'''+apt_source+''' /" >sources.list
1205   cat /etc/apt/sources.list >>sources.list
1206   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1207     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1208   fi
1209   '''+ ' '.join(b.apt_get()) +''' update >&2
1210   cat /var/lib/dpkg/status >&3
1211                 '''
1212         testbed.mungeing_apt()
1213         debug_subprocess('apt-key', script=script)
1214         (rc,se) = testbed.execute('apt-key',
1215                                 ['sh','-ec',script], dump_fd=4,
1216                                 so=so.write(True), cwd=b.dir.write(True))
1217         if rc: bomb('apt setup failed with exit code %d' % rc, se)
1218
1219         testbed.blamed += b.blamed
1220
1221         b._debug('publish reinstall checking...')
1222         pkgs_reinstall = set()
1223         pkg = None
1224         for l in open(so.read()):
1225                 if l.startswith('Package: '):
1226                         pkg = l[9:].rstrip()
1227                 elif l.startswith('Status: install '):
1228                         if pkg in b.registered:
1229                                 pkgs_reinstall.add(pkg)
1230                                 b._debug(' publish reinstall needs '+pkg)
1231
1232         if pkgs_reinstall:
1233                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1234                 what = 'apt-get-reinstall'
1235                 cmdl = (b.apt_get() + ['--reinstall','install'] +
1236                         [pkg for pkg in pkgs_reinstall])
1237                 debug_subprocess(what, cmdl)
1238                 (rc,se) = testbed.execute(what, cmdl)
1239                 if rc: badpkg("installation of basic binarries failed,"
1240                                 " exit code %d" % rc, se)
1241
1242         b._debug('publish install...')
1243         for pkg in b.install:
1244                 what = 'apt-get-install-%s' % pkg
1245                 testbed.blame(pkg)
1246                 cmdl = b.apt_get() + ['install',pkg]
1247                 debug_subprocess(what, cmdl)
1248                 (rc,se) = testbed.execute(what, cmdl)
1249                 if rc: badpkg("installation of %s failed, exit code %d"
1250                                 % (pkg, rc), se)
1251
1252         b._debug('publish done')
1253
1254 #---------- processing of sources (building)
1255
1256 def source_rules_command(act,script,what,which,work,results_lines=0):
1257         script = [      "exec 3>&1 >&2",
1258                         "set -x"        ] + script
1259         script = debug_trace_shfrag('.', '%s-%s' % (what,which),
1260                         4) + '\n'.join(script)
1261         so = TemporaryFile('%s-%s-results' % (what,which))
1262         se = TemporaryFile('%s-%s-log' % (what,which))
1263         debug_subprocess('source-rules-command/%s/%s' % (act.what, which),
1264                         script=script)
1265         rc = testbed.execute('%s-%s' % (what,which),
1266                         ['sh','-xec',script],
1267                         so=so.write(True), se=se.write(True),
1268                         cwd= work.write(True), dump_fd=4)
1269         results = open(so.read()).read().rstrip('\n').split("\n")
1270         se = open(se.read()).read()
1271         if rc: badpkg("%s failed with exit code %d" % (which,rc), se)
1272         if results_lines is not None and len(results) != results_lines:
1273                 badpkg("got %d lines of results from %s where %d expected"
1274                         % (len(results), which, results_lines), se)
1275         if results_lines==1: return results[0]
1276         return results
1277
1278 def build_source(act):
1279         act.blame = 'arg:'+act.af.spec
1280         testbed.blame(act.blame)
1281         testbed.needs_reset()
1282
1283         what = act.what
1284         dsc = act.af
1285         basename = dsc.spec
1286
1287         dsc_file = open(dsc.read())
1288         in_files = False
1289         fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1290         for l in dsc_file:
1291                 l = l.rstrip('\n')
1292                 if l.startswith('Files:'): in_files = True; continue
1293                 elif l.startswith('#'): pass
1294                 elif not l.startswith(' '):
1295                         in_files = False
1296                         if l.startswith('Source:'):
1297                                 act.blame = 'dsc:'+l[7:].strip()
1298                                 testbed.blame(act.blame)
1299                 if not in_files: continue
1300
1301                 m = fre.match(l)
1302                 if not m: badpkg(".dsc contains unparseable line"
1303                                 " in Files: `%s'" % l)
1304                 leaf = m.groups(0)[0]
1305                 subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1306                                 sibling=True)
1307                 subfile.read(True)
1308         dsc.read(True)
1309
1310         script = binaries.apt_pkg_gdebi_script(
1311                 dsc.read(True), [[
1312                 'from GDebi.DscSrcPackage import DscSrcPackage',
1313                 'd = DscSrcPackage(cache, arg)',
1314                 'res = d.checkDeb()',
1315                  ],[
1316                 'from GDebi.DebPackage import DebPackage',
1317                 'd = DebPackage(cache)',
1318                 'res = d.satisfyDependsStr("build-essential")',
1319                 ]])
1320         cmdl = ['python','-c',script]
1321         whatp = what+'-builddeps'
1322         debug_subprocess(whatp, cmdl, script=script)
1323         (rc,se) = testbed.execute(what, cmdl)
1324         if rc: badpkg('build-depends install failed, exit code %d' % rc, se)
1325
1326         work = TemporaryDir(what+'-build')
1327
1328         script = [
1329                         'cd '+work.write(True),
1330                 ]
1331         if opts.user: script += [
1332                         'chown '+opts.user+' .',
1333                         'dsc=dsc.read(True) '+
1334                                 opts.user_wrap('dpkg-source -x $dsc')
1335                 ]
1336         else: script += [
1337                         'dpkg-source -x '+dsc.read(True),
1338                 ]
1339         script += [
1340                         'cd */.',
1341                         'dpkg-checkbuilddeps',
1342                         'pwd >&3',
1343                         opts.user_wrap('debian/rules build'),
1344                 ]
1345         result_pwd = source_rules_command(act,script,what,'build',work,1)
1346
1347         if os.path.dirname(result_pwd)+'/' != work.read(True):
1348                 badpkg("results dir `%s' is not in expected parent dir `%s'"
1349                         % (result_pwd, work.read(True)))
1350
1351         act.work = work
1352         act.tests_tree = InputDir(what+'-tests-tree',
1353                                 work.read(True)+os.path.basename(result_pwd),
1354                                 True)
1355         if act.ah['dsc_tests']:
1356                 act.tests_tree.read()
1357                 testbed.register_ephemeral(act.work)
1358                 testbed.register_ephemeral(act.tests_tree)
1359
1360         act.blamed = copy.copy(testbed.blamed)
1361
1362         def debug_b(m): debug('* <dsc:%s> %s' % (act.what, m))
1363         act.binaries = []
1364         filter = act.ah['dsc_filter']
1365         debug_b('filter=%s' % filter)
1366         if filter != '_':
1367                 script = [
1368                         'cd '+work.write(True)+'/*/.',
1369                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1370                         'cd ..',
1371                         'echo *.deb >&3',
1372                         ]
1373                 result_debs = source_rules_command(act,script,what,
1374                                 'binary',work,1)
1375                 if result_debs == '*': debs = []
1376                 else: debs = result_debs.split(' ')
1377                 debug_b('debs='+`debs`)
1378                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1379                 for deb in debs:
1380                         m = re.match(deb)
1381                         if not m: badpkg("badly-named binary `%s'" % deb)
1382                         pkg = m.groups()[0]
1383                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1384                         for pat in filter.split(','):
1385                                 debug_b('  pat=%s' % pat)
1386                                 if not fnmatch.fnmatchcase(pkg,pat):
1387                                         debug_b('   no match')
1388                                         continue
1389                                 deb_what = pkg+'_'+what+'.deb'
1390                                 bin = RelativeInputFile(deb_what,work,deb,True)
1391                                 debug_b('  deb_what=%s, bin=%s' %
1392                                         (deb_what, str(bin)))
1393                                 binaries.register(act,pkg,bin,
1394                                         'forbuilds',testbed.blamed)
1395                                 act.binaries.append((pkg,bin))
1396                                 break
1397                 debug_b('all done.')
1398
1399 #---------- main processing loop and main program
1400
1401 def process_actions():
1402         global binaries
1403
1404         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1405         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1406         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1407
1408         debug_a1('starting')
1409         testbed.open()
1410         binaries = Binaries()
1411
1412         binaries.reset()
1413
1414         debug_a1('builds ...')
1415         for act in opts.actions:
1416                 debug_a2('%s %s' %
1417                         (act.kind, act.what))
1418
1419                 testbed.prepare([])
1420                 if act.kind == 'deb':
1421                         testbed.blame('arg:'+act.af.spec)
1422                         determine_package(act)
1423                         testbed.blame('deb:'+act.pkg)
1424                         binaries.register(act,act.pkg,act.af,
1425                                 'forbuilds',testbed.blamed)
1426                 if act.kind == 'dsc':
1427                         build_source(act)
1428                 if act.kind == 'tree':
1429                         act.binaries = []
1430
1431         debug_a1('builds done.')
1432
1433         binaries.reset()
1434         control_override = None
1435
1436         debug_a1('tests ...')
1437         for act in opts.actions:
1438                 debug_a2('test %s %s' % (act.kind, act.what))
1439
1440                 testbed.needs_reset()
1441                 if act.kind == 'control':
1442                         control_override = act.af
1443                 if act.kind == 'deb':
1444                         binaries.register(act,act.pkg,act.af,'fortests',
1445                                 ['deb:'+act.pkg])
1446                 if act.kind == 'dsc':
1447                         for (pkg,bin) in act.binaries:
1448                                 binaries.register(act,pkg,bin,'fortests',
1449                                         act.blamed)
1450                         if act.ah['dsc_tests']:
1451                                 debug_a3('read control ...')
1452                                 stanzas = read_control(act, act.tests_tree,
1453                                                 control_override)
1454                                 testbed.blamed += act.blamed
1455                                 debug_a3('run_tests ...')
1456                                 run_tests(stanzas, act.tests_tree)
1457                         control_override = None
1458                 if act.kind == 'tree':
1459                         testbed.blame('arg:'+act.af.spec)
1460                         stanzas = read_control(act, act.af, control_override)
1461                         debug_a3('run_tests ...')
1462                         run_tests(stanzas, act.af)
1463                         control_override = None
1464         debug_a1('tests done.')
1465
1466 def main():
1467         global testbed
1468         global tmpdir
1469         try:
1470                 parse_args()
1471         except SystemExit, se:
1472                 os._exit(20)
1473         try:
1474                 setup_trace()
1475                 testbed = Testbed()
1476                 testbed.start()
1477                 finalise_options()
1478                 process_actions()
1479         except:
1480                 ec = print_exception(sys.exc_info(), '')
1481                 cleanup()
1482                 os._exit(ec)
1483         cleanup()
1484         os._exit(errorcode)
1485
1486 main()