runner/adt-run

   1 #!/usr/bin/python2.4
   2 #
   3 # adt-run is part of autopkgtest
   4 # autopkgtest is a tool for testing Debian binary packages
   5 #
   6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21 #
  22 # See the file CREDITS for a full list of credits information (often
  23 # installed as /usr/share/doc/autopkgtest/CREDITS).
  24
  25 import signal
  26 import optparse
  27 import tempfile
  28 import sys
  29 import subprocess
  30 import traceback
  31 import urllib
  32 import string
  33 import re as regexp
  34 import os
  35 import errno
  36 import fnmatch
  37 import shutil
  38 import copy
  39
  40 from optparse import OptionParser
  41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
  42
  43 #---------- global variables
  44
  45 tmpdir = None           # pathstring on host
  46 testbed = None          # Testbed
  47 errorcode = 0           # exit status that we are going to use
  48 timeouts = { 'short':100, 'install':3000, 'test':10000, 'build':100000 }
  49 binaries = None         # Binaries (.debs we have registered)
  50 build_essential = ["build-essential"]
  51 paths = []
  52
  53 #---------- output handling
  54 #
  55 #  There are at least the following kinds of output:
  56 #
  57 #   1. stderr output which consists of
  58 #   1a. our errors
  59 #   1b. stuff printed to stderr by the virtualisation server
  60 #   1c. stuff printed to stderr by our short-lived subprocesseses
  61 #           which we don't expect to fail
  62 #
  63 #   2. trace information, which consists of
  64 #   2a. our trace information from calls to debug()
  65 #   2b. progress information and stderr output from
  66 #        general scripts we run on the host
  67 #   2c. progress information and stderr output from things
  68 #        we run on the testbed including builds
  69 #   2d. stderr and stdout output from actual tests
  70 #
  71 #   xxxx
  72 #   3. actual test results (printed to our stdout)
  73 #
  74 # Cloning of 1a and 2a, where necessary, is done by us writing
  75 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
  76 # is done by forking off a copy of ourselves to do plumbing,
  77 # which copy we wait for at the appropriate point.
  78
  79 class DummyOpts:
  80  def __init__(do): do.debuglevel = 0
  81
  82 opts = DummyOpts()
  83 trace_stream = None
  84 summary_stream = None
  85
  86 def pstderr(m):
  87         print >>sys.stderr, m
  88         if trace_stream is not None: print >>trace_stream, m
  89
  90 def debug(m, minlevel=0):
  91         if opts.debuglevel < minlevel: return
  92         if opts.quiet and trace_stream is None: return
  93         p = 'adt-run: trace'
  94         if minlevel: p += `minlevel`
  95         p += ': '
  96         for l in m.rstrip('\n').split('\n'):
  97                 s = p + l
  98                 if not opts.quiet: print >>sys.stderr, s
  99                 if trace_stream is not None: print >>trace_stream, s
 100
 101 def debug_file(hp, minlevel=0):
 102         if opts.debuglevel < minlevel: return
 103         def do_copy(stream, what):
 104                 rc = subprocess.call(['cat',hp], stdout=stream)
 105                 if rc: bomb('cat failed copying data from %s'
 106                             ' to %s, exit code %d' % (hp, what, rc))
 107         if not opts.quiet: do_copy(sys.stderr, 'stderr')
 108         if trace_stream is not None: do_copy(trace_stream, 'trace log')
 109
 110 class Errplumb:
 111  def __init__(ep, critical=False):
 112         to_stderr = critical or not opts.quiet
 113         count = to_stderr + (trace_stream is not None)
 114         if count == 0:
 115                 ep.stream = open('/dev/null','w')
 116                 ep._sp = None
 117         elif count == 1:
 118                 if to_stderr: ep.stream = os.dup(2)
 119                 else: ep.stream = trace_stream
 120                 ep._sp = None
 121         else:
 122                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
 123                         stdin=subprocess.PIPE, stdout=trace_stream,
 124                         close_fds=True)
 125                 ep.stream = ep._sp.stdin
 126  def wait(ep):
 127         if ep._sp is None: return
 128         if type(ep.stream) == type(2):
 129                 os.close(ep.stream)
 130                 ep.stream = ()
 131         ep._sp.stdin.close()
 132         rc = ep._sp.wait()
 133         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
 134         ep._sp = None
 135
 136 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
 137         if dbg is not None:
 138                 if isinstance(dbg,tuple): (what,script) = dbg
 139                 else: (what,script) = (dbg,None)
 140                 debug_subprocess(what, cmdl, script=script)
 141         ep = Errplumb(critical)
 142         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
 143         output = running.communicate()[0]
 144         rc = running.wait()
 145         ep.wait()
 146         return (rc, output)
 147
 148 def psummary(m):
 149         if summary_stream is not None: print >>summary_stream, m
 150
 151 def preport(m):
 152         print m
 153         sys.stdout.flush()
 154         if trace_stream is not None: print >>trace_stream, m
 155         psummary(m)
 156
 157 def report(tname, result):
 158         preport('%-20s %s' % (tname, result))
 159
 160 #---------- errors we define
 161
 162 class Quit:
 163         def __init__(q,ec,m): q.ec = ec; q.m = m
 164
 165 def bomb(m):
 166         raise Quit(20, "unexpected error: %s" % m)
 167
 168 def badpkg(m):
 169         preport('blame: ' + ' '.join(testbed.blamed))
 170         preport('badpkg: ' + m)
 171         raise Quit(12, "erroneous package: %s" % m)
 172
 173 class Unsupported:
 174  def __init__(u, lno, m):
 175         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
 176         else: u.m = m
 177  def report(u, tname):
 178         global errorcode
 179         errorcode != 2
 180         report(tname, 'SKIP %s' % u.m)
 181
 182 #---------- convenience function
 183
 184 def mkdir_okexist(pathname, mode=02755):
 185         try:
 186                 os.mkdir(pathname, mode)
 187         except (IOError,OSError), oe:
 188                 if oe.errno != errno.EEXIST: raise
 189
 190 def rmtree(what, pathname):
 191         debug('/ %s rmtree %s' % (what, pathname), 2)
 192         try: shutil.rmtree(pathname)
 193         except (IOError,OSError), oe:
 194                 if oe.errno != errno.EEXIST: raise
 195
 196 def debug_subprocess(what, cmdl=None, script=None):
 197         o = '$ '+what+':'
 198         if cmdl is not None:
 199                 ol = []
 200                 for x in cmdl:
 201                         if x is script: x = '<SCRIPT>'
 202                         ol.append(x.    replace('\\','\\\\').
 203                                         replace(' ','\\ ')      )
 204                 o += ' '+ ' '.join(ol)
 205         debug(o)
 206         if script is not None and opts.debuglevel >= 1:
 207                 o = ''
 208                 for l in script.rstrip('\n').split('\n'):
 209                         o += '$     '+l+'\n'
 210                 debug(o,1)
 211
 212 def flatten(l):
 213         return reduce((lambda a,b: a + b), l, [])
 214
 215 #---------- fancy automatic file-copying class
 216
 217 class AutoFile:
 218         # p.what
 219         # p.path[tb]    None or path    not None => path known
 220         # p.file[tb]    None or path    not None => file exists
 221         # p.ephem[tb]   boolean         True => destroyed by tb reset
 222         # p.spec        string or None
 223         # p.spec_tbp    True or False, or not set if spec is None
 224         # p.dir         '' or '/'
 225
 226  def __init__(p, what):
 227         p.what = what
 228         p.path = [None,None]
 229         p.file = [None,None]
 230         p.ephem = [False,False]
 231         p.spec = None
 232         p.spec_tbp = None
 233         p.dir = ''
 234
 235  def __repr__(p):
 236         return "<AF@%s>" % p.__str__()
 237  def __str__(p):
 238         def ptbp(tbp):
 239                 if p.path[tbp] is None: return '-'+p.dir
 240                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
 241                 else: return p.path[tbp]+p.dir+'!'
 242         out = p.what
 243         if p.spec is not None:
 244                 if not hasattr(p,'spec_tb'): out += '~'
 245                 elif p.spec_tbp: out += '#'
 246                 else: out += '='
 247                 out += p.spec
 248         out += ':'
 249         out += ptbp(False)
 250         out += '|'
 251         out += ptbp(True)
 252         return out
 253
 254  def _wrong(p, how):
 255         xtra = ''
 256         if p.spec is not None:
 257                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
 258         raise ("internal error: %s (%s)" % (how, str(p)))
 259
 260  def _ensure_path(p, tbp):
 261         if p.path[tbp] is None:
 262                 if '/' in p.what:
 263                         p._debug('tmp-parent %s...' % 'HT'[tbp])
 264                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
 265                 if not tbp:
 266                         p.path[tbp] = tmpdir+'/'+p.what
 267                 else:
 268                         p.ephem[tbp] = True
 269                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
 270
 271  def write(p, tbp=False):
 272         p._debug('write %s...' % 'HT'[tbp])
 273         p._ensure_path(tbp)
 274
 275         if p.dir and not p.file[tbp]:
 276                 if not tbp:
 277                         p._debug('mkdir H')
 278                         mkdir_okexist(p.path[tbp])
 279                 else:
 280                         cmdl = ['sh','-ec',
 281                                 'test -d "$1" || mkdir -p "$1"',
 282                                 'x', p.path[tbp]]
 283                         tf_what = urllib.quote(p.what).replace('/','%2F')
 284                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
 285                         if rc: bomb('failed to create directory %s' %
 286                                 p.path[tbp])
 287
 288         p.file[tbp] = p.path[tbp]
 289         return p.path[tbp]
 290
 291  def read(p, tbp=False):
 292         p._debug('read %s...' % 'HT'[tbp])
 293         p._ensure_path(tbp)
 294
 295         if p.file[tbp] is None:
 296                 if p.file[not tbp] is None:
 297                         p._wrong("requesting read but nonexistent")
 298                 cud = ['copyup','copydown'][tbp]
 299                 src = p.file[not tbp] + p.dir
 300                 dst = p.path[tbp] + p.dir
 301                 testbed.command(cud, (src, dst))
 302                 p.file[tbp] = p.path[tbp]
 303
 304         return p.file[tbp] + p.dir
 305
 306  def invalidate(p, tbp=False):
 307         if p.path[tbp] is not None:
 308                 p._debug('invalidating %s' % 'HT'[tbp])
 309         p.file[tbp] = None
 310         if p.spec_tbp != tbp or p.spec is None:
 311                 p.path[tbp] = None
 312
 313  def invalidate_ephem(p, tbp=False):
 314         if p.ephem[tbp]:
 315                 p.invalidate(tbp)
 316
 317  def _debug(p, m):
 318         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
 319
 320  def _constructed(p):
 321         p._debug('constructed: '+str(p))
 322         p._check()
 323         paths.append(p)
 324
 325  def _check(p):
 326         for tbp in [False,True]:
 327          for pf in [p.path, p.file]:
 328                 if pf[tbp] is None: continue
 329                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
 330                 if p.dir and pf[tbp].endswith('/'):
 331                         pf[tbp] = pf[tbp].rstrip('/')
 332                         if not pf[tbp]: pf[tbp] = '/'
 333                 if not p.dir and pf[tbp].endswith('/'):
 334                         bomb("directory `%s' specified for "
 335                              "non-directory %s" % (pf[tbp], p.what))
 336
 337  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
 338         AutoFile.__init__(p,what)
 339         sh_on = ''; sh_sibl = ''
 340         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
 341         if sibling: sh_sibl=' (sibling)'
 342         parent._debug('using as base: %s: %s%s%s' %
 343                         (str(parent), leaf, sh_on, sh_sibl))
 344         if not sibling and not parent.dir:
 345                 parent._wrong('asked for non-sibling relative path of non-dir')
 346         if sibling: trim = os.path.dirname
 347         else: trim = lambda x: x
 348         for tbp in [False,True]:
 349                 p.ephem[tbp] = parent.ephem[tbp]
 350                 if parent.path[tbp] is None: continue
 351                 trimmed = trim(parent.path[tbp])
 352                 if trimmed: trimmed += '/'
 353                 p.path[tbp] = trimmed + leaf
 354                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
 355                         p.file[tbp] = p.path[tbp]
 356
 357 class InputFile(AutoFile):
 358  def _init(p, what, spec, spec_tbp=False):
 359         AutoFile.__init__(p, what)
 360         p.spec = spec
 361         p.spec_tbp = spec_tbp
 362         p.path[spec_tbp] = spec
 363         p.file[p.spec_tbp] = p.path[p.spec_tbp]
 364  def __init__(p, what, spec, spec_tbp=False):
 365         p._init(what,spec,spec_tbp)
 366         p._constructed()
 367
 368 class InputDir(InputFile):
 369  def __init__(p, what, spec, spec_tbp=False):
 370         InputFile._init(p,what,spec,spec_tbp)
 371         p.dir = '/'
 372         p._constructed()
 373
 374 class OutputFile(AutoFile):
 375  def _init(p, what, spec, spec_tbp=False):
 376         AutoFile.__init__(p, what)
 377         p.spec = spec
 378         p.spec_tbp = spec_tbp
 379         p.path[spec_tbp] = spec
 380  def __init__(p, what, spec, spec_tbp=False):
 381         p._init(what,spec,spec_tbp)
 382         p._constructed()
 383
 384 class OutputDir(OutputFile):
 385  def __init__(p, what, spec, spec_tbp=False):
 386         OutputFile._init(p,what,spec,spec_tbp)
 387         p.dir = '/'
 388         p._constructed()
 389
 390 class RelativeInputDir(AutoFile):
 391  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 392         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 393         p.dir = '/'
 394         p._constructed()
 395
 396 class RelativeInputFile(AutoFile):
 397  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
 398         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
 399         p._constructed()
 400
 401 class RelativeOutputFile(AutoFile):
 402  def __init__(p, what, parent, leaf, sibling=False):
 403         p._relative_init(what, parent, leaf, None, False, sibling)
 404         p._constructed()
 405
 406 class TemporaryFile(AutoFile):
 407  def __init__(p, what):
 408         AutoFile.__init__(p, what)
 409         p._constructed()
 410
 411 class TemporaryDir(AutoFile):
 412  def __init__(p, what):
 413         AutoFile.__init__(p, what)
 414         p.dir = '/'
 415         p._constructed()
 416
 417 #---------- parsing and representation of the arguments
 418
 419 class Action:
 420  def __init__(a, kind, af, arghandling, what):
 421         # extra attributes get added during processing
 422         a.kind = kind
 423         a.af = af
 424         a.ah = arghandling
 425         a.what = what
 426         a.missing_tests_control = False
 427  def __repr__(a):
 428         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
 429
 430 def parse_args():
 431         global opts, timeouts
 432         global n_non_actions # argh, stupid python scoping rules
 433         usage = "%prog <options> --- <virt-server>..."
 434         parser = OptionParser(usage=usage)
 435         pa = parser.add_option
 436         pe = parser.add_option
 437
 438         arghandling = {
 439                 'dsc_tests': True,
 440                 'dsc_filter': '*',
 441                 'deb_forbuilds': 'auto',
 442                 'deb_fortests': 'auto',
 443                 'tb': False,
 444                 'override_control': None,
 445                 'set_lang': 'C'
 446         }
 447         initial_arghandling = arghandling.copy()
 448         n_non_actions = 0
 449
 450         #----------
 451         # actions (ie, test sets to run, sources to build, binaries to use):
 452
 453         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
 454                 global n_non_actions
 455                 parser.largs.append((value,kindpath))
 456                 n_non_actions += not(is_act)
 457
 458         def pa_action(long, metavar, kindpath, help, is_act=True):
 459                 pa('','--'+long, action='callback', callback=cb_action,
 460                         nargs=1, type='string',
 461                         callback_args=(long,kindpath,is_act), help=help)
 462
 463         pa_action('built-tree',         'TREE', '@/',
 464                 help='run tests from build tree TREE')
 465
 466         pa_action('unbuilt-tree',       'TREE', '@//',
 467                 help='run tests from build tree TREE')
 468
 469         pa_action('source',             'DSC', '@.dsc',
 470                 help='build DSC and use its tests and/or'
 471                     ' generated binary packages')
 472
 473         pa_action('binary',             'DEB', '@.deb',
 474                help='use binary package DEB according'
 475                     ' to most recent --binaries-* settings')
 476
 477         def cb_actnoarg(op,optstr,value,parser, largsentry):
 478                 parser.largs.append(largsentry)
 479         pa('','--instantiate', action='callback', callback=cb_actnoarg,
 480                 callback_args=((None, ('instantiate',)),),
 481                 help='instantiate testbed now (during testing phase)'
 482                      ' and install packages'
 483                      ' selected for automatic installation, even'
 484                      ' if this might apparently not be required otherwise')
 485
 486         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
 487                help='run tests from control file CONTROL instead,'
 488                     ' (applies to next test suite only)')
 489
 490         #----------
 491         # argument handling settings (what ways to use action
 492         #  arguments, and pathname processing):
 493
 494         def cb_setah(option, opt_str, value, parser, toset,setval):
 495                 if type(setval) == list:
 496                         if not value in setval:
 497                                 parser.error('value for %s option (%s) is not '
 498                                  'one of the permitted values (%s)' %
 499                                  (value, opt_str, setval.join(' ')))
 500                 elif setval is not None:
 501                         value = setval
 502                 for v in toset:
 503                         arghandling[v] = value
 504                 parser.largs.append(arghandling.copy())
 505
 506         def pa_setah(long, affected,effect, metavar=None, **kwargs):
 507                 type = metavar
 508                 if type is not None: type = 'string'
 509                 pa('',long, action='callback', callback=cb_setah,
 510                    callback_args=(affected,effect), **kwargs)
 511
 512         #---- paths: host or testbed:
 513         #
 514         pa_setah('--paths-testbed', ['tb'],True,
 515                 help='subsequent path specifications refer to the testbed')
 516         pa_setah('--paths-host', ['tb'],False,
 517                 help='subsequent path specifications refer to the host')
 518
 519         #---- source processing settings:
 520
 521         pa_setah('--sources-tests', ['dsc_tests'],True,
 522                 help='run tests from builds of subsequent sources')
 523         pa_setah('--sources-no-tests', ['dsc_tests'],False,
 524                 help='do not run tests from builds of subsequent sources')
 525
 526         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
 527                 type='string', metavar='PATTERN-LIST',
 528                 help='from subsequent sources, use binaries matching'
 529                      ' PATTERN-LIST (comma-separated glob patterns)'
 530                      ' according to most recent --binaries-* settings')
 531         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
 532                 help='from subsequent sources, do not use any binaries')
 533         #---- binary package processing settings:
 534
 535         def pa_setahbins(long,toset,how):
 536          pa_setah(long, toset,['ignore','auto','install'],
 537                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
 538                 help=how+' ignore binaries, install them as needed'
 539                         ' for dependencies, or unconditionally install'
 540                         ' them, respectively')
 541         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
 542         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
 543         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
 544
 545         #----------
 546         # general options:
 547
 548         def cb_vserv(op,optstr,value,parser):
 549                 parser.values.vserver = list(parser.rargs)
 550                 del parser.rargs[:]
 551
 552         def cb_path(op,optstr,value,parser, constructor,long,dir):
 553                 name = long.replace('-','_')
 554                 af = constructor(long, value,arghandling['tb'])
 555                 setattr(parser.values, name, af)
 556
 557         def pa_path(long, constructor, help, dir=False):
 558                 pa('','--'+long, action='callback', callback=cb_path,
 559                         callback_args=(constructor,long,dir),
 560                         nargs=1, type='string',
 561                         help=help, metavar='PATH')
 562
 563         pa_path('output-dir', OutputDir, dir=True,
 564                 help='write stderr/out files in PATH')
 565
 566         pa('--leave-lang', dest='set_lang', action='store_false',
 567                 help="leave LANG on testbed set to testbed's default")
 568         pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
 569                 help='set LANG on testbed to LANGVAL', default='C')
 570
 571         pa('','--tmp-dir',              type='string', dest='tmpdir',
 572                 help='write temporary files to TMPDIR, emptying it'
 573                      ' beforehand and leaving it behind at the end')
 574         pa('','--log-file',             type='string', dest='logfile',
 575                 help='write the log LOGFILE, emptying it beforehand,'
 576                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
 577         pa('','--summary-file',         type='string', dest='summary',
 578                 help='write a summary report to SUMMARY,'
 579                      ' emptying it beforehand')
 580
 581         for k in timeouts.keys():
 582                 pa('','--timeout-'+k,   type='int', dest='timeout_'+k,
 583                         metavar='T', help='set %s timeout to T')
 584         pa('','--timeout-factor',       type='float', dest='timeout_factor',
 585                         metavar='FACTOR', default=1.0,
 586                         help='multiply all default timeouts by FACTOR')
 587
 588         pa('','--user',                 type='string', dest='user',
 589                 help='run tests as USER (needs root on testbed)')
 590         pa('','--gain-root',            type='string', dest='gainroot',
 591                 help='prefix debian/rules binary with GAINROOT')
 592         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
 593         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
 594         pa('','--gnupg-home',           type='string', dest='gnupghome',
 595                 default='~/.autopkgtest/gpg',
 596                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
 597                         " signing private apt archive);"
 598                         " `fresh' means generate new key each time.")
 599
 600         #----------
 601         # actual meat:
 602
 603         class SpecialOption(optparse.Option): pass
 604         vs_op = SpecialOption('','--VSERVER-DUMMY')
 605         vs_op.action = 'callback'
 606         vs_op.type = None
 607         vs_op.default = None
 608         vs_op.nargs = 0
 609         vs_op.callback = cb_vserv
 610         vs_op.callback_args = ( )
 611         vs_op.callback_kwargs = { }
 612         vs_op.help = 'introduces virtualisation server and args'
 613         vs_op._short_opts = []
 614         vs_op._long_opts = ['---']
 615
 616         pa(vs_op)
 617
 618         (opts,args) = parser.parse_args()
 619         if not hasattr(opts,'vserver'):
 620                 parser.error('you must specifiy --- <virt-server>...')
 621         if n_non_actions >= len(parser.largs):
 622                 parser.error('nothing to do specified')
 623
 624         for k in timeouts.keys():
 625                 t = getattr(opts,'timeout_'+k)
 626                 if t is None: t = timeouts[k] * opts.timeout_factor
 627                 timeouts[k] = int(t)
 628
 629         arghandling = initial_arghandling
 630         opts.actions = []
 631         ix = 0
 632         for act in args:
 633                 if type(act) == dict:
 634                         arghandling = act
 635                         continue
 636                 elif type(act) == tuple:
 637                         pass
 638                 elif type(act) == str:
 639                         act = (act,act)
 640                 else:
 641                         raise ("unknown action in list `%s' having"
 642                               " type `%s'" % (act, type(act)))
 643                 (pathstr, kindpath) = act
 644
 645                 constructor = InputFile
 646                 if type(kindpath) is tuple:             kind = kindpath[0]
 647                 elif kindpath.endswith('.deb'):         kind = 'deb'
 648                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
 649                 elif kindpath.endswith('//'):
 650                         kind = 'ubtree'
 651                         constructor = InputDir
 652                 elif kindpath.endswith('/'):
 653                         kind = 'tree'
 654                         constructor = InputDir
 655                 else: parser.error("do not know how to handle filename `%s';"
 656                         " specify --source --binary or --build-tree" %
 657                         kindpath)
 658
 659                 what = '%s%s' % (kind,ix); ix += 1
 660
 661                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
 662                 else: fwhatx = '-'+kind
 663
 664                 if pathstr is None: af = None
 665                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
 666                 opts.actions.append(Action(kind, af, arghandling, what))
 667
 668 def setup_trace():
 669         global trace_stream, tmpdir, summary_stream
 670
 671         if opts.tmpdir is not None:
 672                 rmtree('tmpdir(specified)',opts.tmpdir)
 673                 mkdir_okexist(opts.tmpdir, 0700)
 674                 tmpdir = opts.tmpdir
 675         else:
 676                 assert(tmpdir is None)
 677                 tmpdir = tempfile.mkdtemp()
 678
 679         if opts.logfile is None:
 680                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
 681                         opts.logfile = opts.output_dir.spec + '/log'
 682                 elif opts.tmpdir is not None:
 683                         opts.logfile = opts.tmpdir + '/log'
 684         if opts.logfile is not None:
 685                 trace_stream = open(opts.logfile, 'w', 0)
 686         if opts.summary is not None:
 687                 summary_stream = open(opts.summary, 'w', 0)
 688
 689         debug('options: '+`opts`+'; timeouts: '+`timeouts`, 1)
 690
 691 def finalise_options():
 692         global opts, tb, build_essential
 693
 694         if opts.user is None and 'root-on-testbed' not in testbed.caps:
 695                 opts.user = ''
 696
 697         if opts.user is None:
 698                 su = 'suggested-normal-user='
 699                 ul = [
 700                         e[len(su):]
 701                         for e in testbed.caps
 702                         if e.startswith(su)
 703                         ]
 704                 if ul:
 705                         opts.user = ul[0]
 706                 else:
 707                         opts.user = ''
 708
 709         if opts.user:
 710                 if 'root-on-testbed' not in testbed.caps:
 711                         pstderr("warning: virtualisation"
 712                                 " system does not offer root on testbed,"
 713                                 " but --user option specified: failure likely")
 714                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
 715         else:
 716                 opts.user_wrap = lambda x: x
 717
 718         if opts.gainroot is None:
 719                 opts.gainroot = ''
 720                 if (opts.user or
 721                     'root-on-testbed' not in testbed.caps):
 722                         opts.gainroot = 'fakeroot'
 723                         build_essential += ['fakeroot']
 724
 725         if opts.gnupghome.startswith('~/'):
 726                 try: home = os.environ['HOME']
 727                 except KeyError:
 728                         parser.error("HOME environment variable"
 729                                 " not set, needed for --gnupghome=`%s"
 730                                 % opts.gnupghome)
 731                 opts.gnupghome = home + opts.gnupghome[1:]
 732         elif opts.gnupghome == 'fresh':
 733                 opts.gnupghome = None
 734
 735 #---------- testbed management - the Testbed class
 736
 737 class Testbed:
 738  def __init__(tb):
 739         tb.sp = None
 740         tb.lastsend = None
 741         tb.scratch = None
 742         tb.modified = False
 743         tb.blamed = []
 744         tb._debug('init')
 745         tb._need_reset_apt = False
 746  def _debug(tb, m, minlevel=0):
 747         debug('** '+m, minlevel)
 748  def start(tb):
 749         tb._debug('start')
 750         p = subprocess.PIPE
 751         debug_subprocess('vserver', opts.vserver)
 752         tb._errplumb = Errplumb(True)
 753         tb.sp = subprocess.Popen(opts.vserver,
 754                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
 755         tb.expect('ok')
 756         tb.caps = tb.commandr('capabilities')
 757  def stop(tb):
 758         tb._debug('stop')
 759         tb.close()
 760         if tb.sp is None: return
 761         ec = tb.sp.returncode
 762         if ec is None:
 763                 tb.sp.stdout.close()
 764                 tb.send('quit')
 765                 tb.sp.stdin.close()
 766                 ec = tb.sp.wait()
 767         if ec:
 768                 tb.bomb('testbed gave exit status %d after quit' % ec)
 769         tb._errplumb.wait()
 770  def open(tb):
 771         tb._debug('open, scratch=%s' % tb.scratch)
 772         if tb.scratch is not None: return
 773         pl = tb.commandr('open')
 774         tb._opened(pl)
 775  def _opened(tb, pl):
 776         tb.scratch = InputDir('tb-scratch', pl[0], True)
 777         tb.deps_processed = []
 778         for af in paths: af.invalidate_ephem(True)
 779         tb._auxverbscript_make()
 780  def _auxverbscript_make(tb):
 781         pec = tb.commandr('print-auxverb-command')
 782         if len(pec) < 1: tb.bomb('too few results from print-execute-command')
 783         cmdl = map(urllib.unquote, pec[0].split(','))
 784
 785         shellquote_re = regexp.compile('"')
 786         def shellquote_arg(s): return "'" + shellquote_re.sub(r"'\''", s) + "'"
 787         def shellquote_cmdl(l): return ' '.join(map(shellquote_arg,l))
 788
 789         tb._debug('cmdl = %s' % (`cmdl`))
 790
 791         tb.ec_auxverbscript = TemporaryFile('satdep-auxverb')
 792         print >>open(tb.ec_auxverbscript.write(),'w'), (
 793 '''#!/bin/sh
 794 set -ex
 795 echo >&2 ": $*"
 796 if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
 797         # This is a pretty nasty hack.  Hopefully it can go away
 798         #  eventually.  See #635763.
 799         set -- dpkg --print-architecture
 800 fi
 801 exec '''+shellquote_cmdl(cmdl)+' "$@"'+"\n"
 802                 )
 803         os.chmod(tb.ec_auxverbscript.write(), 0755)
 804
 805  def mungeing_apt(tb):
 806         if not 'revert' in tb.caps:
 807                 tb._need_reset_apt = True
 808  def reset_apt(tb):
 809         if not tb._need_reset_apt: return
 810         what = 'aptget-update-reset'
 811         cmdl = ['sh','-c','apt-get -qy update 2>&1']
 812         rc = tb.execute(what, cmdl, kind='install')
 813         if rc:
 814                 pstderr("\n" "warning: failed to restore"
 815                         " testbed apt cache, exit code %d" % rc)
 816         tb._need_reset_apt = False
 817  def close(tb):
 818         tb._debug('close, scratch=%s' % tb.scratch)
 819         if tb.scratch is None: return
 820         tb.scratch = None
 821         if tb.sp is None: return
 822         tb.command('close')
 823  def prepare1(tb, deps_new):
 824         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
 825                 (tb.modified, tb.deps_processed, deps_new), 1)
 826         if 'revert' in tb.caps and (tb.modified or
 827             [d for d in tb.deps_processed if d not in deps_new]):
 828                 for af in paths: af.read(False)
 829                 tb._debug('reset **')
 830                 pl = tb.commandr('revert')
 831                 tb._opened(pl)
 832         tb.modified = False
 833  def prepare2(tb, deps_new):
 834         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
 835         binaries.publish()
 836         tb._install_deps(deps_new)
 837  def prepare(tb, deps_new):
 838         tb.prepare1(deps_new)
 839         tb.prepare2(deps_new)
 840  def _install_deps(tb, deps_new):
 841         tb._debug(' installing dependencies '+`deps_new`, 1)
 842         tb.deps_processed = deps_new
 843         if not deps_new: return
 844         tb.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')
 845  def needs_reset(tb):
 846         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
 847         tb.modified = True
 848  def blame(tb, m):
 849         tb._debug('blame += %s' % m, 1)
 850         tb.blamed.append(m)
 851  def bomb(tb, m):
 852         tb._debug('bomb %s' % m)
 853         if tb.sp is not None:
 854                 tb.sp.stdout.close()
 855                 tb.sp.stdin.close()
 856                 ec = tb.sp.wait()
 857                 if ec: pstderr('adt-run: testbed failing,'
 858                                 ' exit status %d' % ec)
 859         tb.sp = None
 860         raise Quit(16, 'testbed failed: %s' % m)
 861  def send(tb, string):
 862         tb.sp.stdin
 863         try:
 864                 debug('>> '+string, 2)
 865                 print >>tb.sp.stdin, string
 866                 tb.sp.stdin.flush()
 867                 tb.lastsend = string
 868         except:
 869                 (type, value, dummy) = sys.exc_info()
 870                 tb.bomb('cannot send to testbed: %s' % traceback.
 871                         format_exception_only(type, value))
 872  def expect(tb, keyword, nresults=None):
 873         l = tb.sp.stdout.readline()
 874         if not l: tb.bomb('unexpected eof from the testbed')
 875         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
 876         l = l.rstrip('\n')
 877         debug('<< '+l, 2)
 878         ll = l.split()
 879         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
 880         if ll[0] != keyword:
 881                 if tb.lastsend is None:
 882                         tb.bomb("got banner `%s', expected `%s...'" %
 883                                 (l, keyword))
 884                 else:
 885                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
 886                                 (tb.lastsend, l, keyword))
 887         ll = ll[1:]
 888         if nresults is not None and len(ll) != nresults:
 889                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
 890                         " expected %d result parameters" %
 891                         (tb.lastsend, l, len(ll), nresults))
 892         return ll
 893  def commandr(tb, cmd, args=(), nresults=None, unquote=True):
 894         # pass args=[None,...] or =(None,...) to avoid more url quoting
 895         if type(cmd) is str: cmd = [cmd]
 896         if len(args) and args[0] is None: args = args[1:]
 897         else: args = map(urllib.quote, args)
 898         al = cmd + args
 899         tb.send(string.join(al))
 900         ll = tb.expect('ok', nresults)
 901         if unquote: ll = map(urllib.unquote, ll)
 902         return ll
 903  def command(tb, cmd, args=()):
 904         tb.commandr(cmd, args, 0)
 905  def commandr1(tb, cmd, args=()):
 906         rl = tb.commandr(cmd, args, 1)
 907         return rl[0]
 908  def execute(tb, what, cmdl,
 909                 si='/dev/null', so='/dev/null', se=None, cwd=None,
 910                 script=False, tmpdir=None, kind='short'):
 911         # Options for script:
 912         #   False - do not call debug_subprocess, no synch. reporting required
 913         #   None or string - call debug_subprocess with that value,
 914         #                       plumb stderr through synchronously if possible
 915         # Options for se:
 916         #   None - usual Errplumb (output is of kind 2c)
 917         #   string - path on testbed (output is of kind 2d)
 918
 919         timeout = timeouts[kind]
 920
 921         if script is not False: debug_subprocess(what, cmdl, script=script)
 922         if cwd is None: cwd = tb.scratch.write(True)
 923
 924         xdump = None
 925         if se is None:
 926                 ep = Errplumb()
 927                 se_catch = TemporaryFile(what+'-xerr')
 928                 se_use = se_catch.write(True)
 929                 if not opts.quiet: xdump = 'debug=2-2'
 930                 elif trace_stream is not None:
 931                         xdump = 'debug=2-%d' % trace_stream.fileno()
 932         else:
 933                 ep = None
 934                 se_catch = None
 935                 se_use = se
 936
 937         cmdl = [None,
 938                 ','.join(map(urllib.quote, cmdl)),
 939                 si, so, se_use, cwd]
 940
 941         if timeout is not None and timeout > 0:
 942                 cmdl.append('timeout=%d' % timeout)
 943
 944         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
 945         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
 946         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
 947         if opts.set_lang is not False:
 948                 cmdl.append('env=LANG=%s' % opts.set_lang)
 949
 950         rc = tb.commandr1('execute', cmdl)
 951         try: rc = int(rc)
 952         except ValueError: bomb("execute for %s gave invalid response `%s'"
 953                                         % (what,rc))
 954
 955         if se_catch is not None:
 956                 debug_file(se_catch.read())
 957         if ep is not None:
 958                 ep.wait()
 959
 960         return rc
 961
 962  def satisfy_dependencies_string(tb, deps, what):
 963         # Must have called Binaries.configure_apt
 964         debug('dependencies: %s: satisfying %s' % (what,deps))
 965         dsc = TemporaryFile('deps.dsc')
 966         print >>open(dsc.write(),'w'), 'Build-Depends: ', deps, '\n\n'
 967         # pbuilder-satisfydepends has a bug where it ignores the
 968         #  Build-Depends if it's the last line in the dsc (#635696)
 969         tb.satisfy_dependencies_dsc(dsc, what)
 970
 971  def satisfy_dependencies_dsc(tb, dsc, what):
 972         # Must have called Binaries.configure_apt
 973         cmdl = [ '/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
 974                 '--binary-all', # --check-key
 975                 '--internal-chrootexec',tb.ec_auxverbscript.read(),
 976                 '-c',dsc.read()
 977         ]
 978         # The --internal-chrootexec option is really handy but
 979         #  perhaps we are not supposed to use it ?  See also #635697.
 980         debug('dependencies: %s: running %s' % (what,`cmdl`))
 981         rc = subprocess.call(cmdl, stdout=None, stderr=None)
 982         if rc: badpkg('dependency install failed, exit code %d' % rc)
 983
 984 #---------- representation of test control files: Field*, Test, etc.
 985
 986 class FieldBase:
 987  def __init__(f, fname, stz, base, tnames, vl):
 988         assert(vl)
 989         f.stz = stz
 990         f.base = base
 991         f.tnames = tnames
 992         f.vl = vl
 993  def words(f):
 994         def distribute(vle):
 995                 (lno, v) = vle
 996                 r = v.split()
 997                 r = map((lambda w: (lno, w)), r)
 998                 return r
 999         return flatten(map(distribute, f.vl))
1000  def atmostone(f):
1001         if len(vl) == 1:
1002                 (f.lno, f.v) = vl[0]
1003         else:
1004                 raise Unsupported(f.vl[1][0],
1005                         'only one %s field allowed' % fn)
1006         return f.v
1007
1008 class FieldIgnore(FieldBase):
1009  def parse(f): pass
1010
1011 class Restriction:
1012  def __init__(r,rname,base): pass
1013
1014 class Restriction_rw_build_tree(Restriction): pass
1015 class Restriction_breaks_testbed(Restriction):
1016  def __init__(r, rname, base):
1017         if 'revert' not in testbed.caps:
1018                 raise Unsupported(f.lno,
1019                         'Test breaks testbed but testbed cannot revert')
1020 class Restriction_needs_root(Restriction):
1021  def __init__(r, rname, base):
1022         if 'root-on-testbed' not in testbed.caps:
1023                 raise Unsupported(f.lno,
1024                         'Test needs root on testbed which is not available')
1025
1026 class Field_Restrictions(FieldBase):
1027  def parse(f):
1028         for wle in f.words():
1029                 (lno, rname) = wle
1030                 nrname = rname.replace('-','_')
1031                 try: rclass = globals()['Restriction_'+nrname]
1032                 except KeyError: raise Unsupported(lno,
1033                         'unknown restriction %s' % rname)
1034                 r = rclass(nrname, f.base)
1035                 f.base['restriction_names'].append(rname)
1036                 f.base['restrictions'].append(r)
1037
1038 class Field_Features(FieldIgnore):
1039  def parse(f):
1040         for wle in f.words():
1041                 (lno, fname) = wle
1042                 f.base['feature_names'].append(fname)
1043                 nfname = fname.replace('-','_')
1044                 try: fclass = globals()['Feature_'+nfname]
1045                 except KeyError: continue
1046                 ft = fclass(nfname, f.base)
1047                 f.base['features'].append(ft)
1048
1049 class Field_Tests(FieldIgnore): pass
1050
1051 class Field_Depends(FieldBase):
1052  def parse(f):
1053         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
1054         dl = map(lambda x: x.strip(),
1055                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
1056         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
1057         for d in dl:
1058                 if re.search(d):
1059                         badpkg("Test Depends field contains dependency"
1060                                " `%s' with invalid characters" % d)
1061         f.base['depends'] = dl
1062
1063 class Field_Tests_directory(FieldBase):
1064  def parse(f):
1065         td = atmostone(f)
1066         if td.startswith('/'): raise Unspported(f.lno,
1067                 'Tests-Directory may not be absolute')
1068         f.base['testsdir'] = td
1069
1070 def run_tests(stanzas, tree):
1071         global errorcode, testbed
1072         if stanzas == ():
1073                 report('*', 'SKIP no tests in this package')
1074                 errorcode |= 8
1075         for stanza in stanzas:
1076                 tests = stanza[' tests']
1077                 if not tests:
1078                         report('*', 'SKIP package has metadata but no tests')
1079                         errorcode |= 8
1080                 for t in tests:
1081                         t.prepare()
1082                         t.run(tree)
1083                         if 'breaks-testbed' in t.restriction_names:
1084                                 testbed.needs_reset()
1085                 testbed.needs_reset()
1086
1087 class Test:
1088  def __init__(t, tname, base, act):
1089         if '/' in tname: raise Unsupported(base[' lno'],
1090                 'test name may not contain / character')
1091         for k in base: setattr(t,k,base[k])
1092         t.tname = tname
1093         t.act = act
1094         t.what = act.what+'t-'+tname
1095         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1096         else: t.path = tname
1097         t._debug('constructed; path=%s' % t.path)
1098         t._debug(' .depends=%s' % t.depends)
1099  def _debug(t, m):
1100         debug('& %s: %s' % (t.what, m))
1101  def report(t, m):
1102         report(t.what, m)
1103  def reportfail(t, m):
1104         global errorcode
1105         errorcode |= 4
1106         report(t.what, 'FAIL ' + m)
1107  def prepare(t):
1108         t._debug('preparing')
1109         dn = []
1110         for d in t.depends:
1111                 t._debug(' processing dependency '+d)
1112                 if not '@' in d:
1113                         t._debug('  literal dependency '+d)
1114                         dn.append(d)
1115                 else:
1116                         for (pkg,bin) in t.act.binaries:
1117                                 d = d.replace('@',pkg)
1118                                 t._debug('  synthesised dependency '+d)
1119                                 dn.append(d)
1120         testbed.prepare(dn)
1121  def run(t, tree):
1122         t._debug('[----------------------------------------')
1123         def stdouterr(oe):
1124                 idstr = t.what + '-' + oe
1125                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1126                         use_dir = opts.output_dir
1127                 else:
1128                         use_dir = testbed.scratch
1129                 return RelativeOutputFile(idstr, use_dir, idstr)
1130
1131         if hasattr(t.act,'work'): t.act.work.write(True)
1132         tree.read(True)
1133
1134         af = RelativeInputFile(t.what, tree, t.path)
1135         so = stdouterr('stdout')
1136         se = stdouterr('stderr')
1137
1138         tf = af.read(True)
1139         tmpdir = None
1140
1141         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1142         if rc: bomb('failed to chmod +x %s' % tf)
1143
1144         if 'needs-root' not in t.restriction_names and opts.user is not None:
1145                 tfl = ['su',opts.user,'-c',tf]
1146                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1147                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1148                 if opts.user:
1149                         script += '; chown %s "$1"' % opts.user
1150                         if 'rw-build-tree' in t.restriction_names:
1151                                 script += '; chown -R %s "$2"' % opts.user
1152                 rc = testbed.execute('mktmpdir-'+t.what,
1153                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1154                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1155                                 % (tmpdir, rc))
1156         else:
1157                 tfl = [tf]
1158
1159         rc = testbed.execute('test-'+t.what, tfl,
1160                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1161                 tmpdir=tmpdir, kind='test')
1162
1163         so_read = so.read()
1164         se_read = se.read()
1165
1166         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1167         stab = os.stat(se_read)
1168         if stab.st_size != 0:
1169                 l = open(se_read).readline()
1170                 l = l.rstrip('\n \t\r')
1171                 if len(l) > 35: l = l[:35] + '...'
1172                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1173                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1174                 debug_file(se_read)
1175         elif rc != 0:
1176                 t.reportfail('non-zero exit status %d' % rc)
1177         else:
1178                 t.report('PASS')
1179
1180         stab = os.stat(so_read)
1181         if stab.st_size != 0:
1182                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1183                 debug_file(so_read)
1184
1185         t._debug('----------------------------------------]')
1186
1187 def read_control(act, tree, control_override):
1188         stanzas = [ ]
1189
1190         if control_override is not None:
1191                 control_af = control_override
1192                 testbed.blame('arg:'+control_override.spec)
1193         else:
1194                 if act.missing_tests_control:
1195                         return ()
1196                 control_af = RelativeInputFile(act.what+'-testcontrol',
1197                         tree, 'debian/tests/control')
1198         try:
1199                 control = open(control_af.read(), 'r')
1200         except (IOError,OSError), oe:
1201                 if oe[0] != errno.ENOENT: raise
1202                 return []
1203
1204         lno = 0
1205         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1206         stz = None      # stz[field_name][index] = (lno, value)
1207                         # special field names:
1208                         # stz[' lno'] = number
1209                         # stz[' tests'] = list of Test objects
1210         def end_stanza(stz):
1211                 if stz is None: return
1212                 stz[' errs'] = 0
1213                 stanzas.append(stz)
1214                 stz = None
1215                 hcurrent = None
1216
1217         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1218         while 1:
1219                 l = control.readline()
1220                 if not l: break
1221                 lno += 1
1222                 if not l.endswith('\n'): badctrl('unterminated line')
1223                 if regexp.compile('\s*\#').match(l): continue
1224                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1225                 initmat = initre.match(l)
1226                 if initmat:
1227                         (fname, l) = initmat.groups()
1228                         fname = string.capwords(fname)
1229                         if stz is None:
1230                                 stz = { ' lno': lno, ' tests': [] }
1231                         if not stz.has_key(fname): stz[fname] = [ ]
1232                         hcurrent = stz[fname]
1233                 elif regexp.compile('\s').match(l):
1234                         if not hcurrent: badctrl('unexpected continuation')
1235                 else:
1236                         badctrl('syntax error')
1237                 hcurrent.append((lno, l))
1238         end_stanza(stz)
1239
1240         def testbadctrl(stz, lno, m):
1241                 report_badctrl(lno, m)
1242                 stz[' errs'] += 1
1243
1244         for stz in stanzas:
1245                 try:
1246                         try: tnames = stz['Tests']
1247                         except KeyError:
1248                                 tnames = ['*']
1249                                 raise Unsupported(stz[' lno'],
1250                                         'no Tests field')
1251                         tnames = map((lambda lt: lt[1]), tnames)
1252                         tnames = string.join(tnames).split()
1253                         base = {
1254                                 'restriction_names': [],
1255                                 'restrictions': [],
1256                                 'feature_names': [],
1257                                 'features': [],
1258                                 'testsdir': 'debian/tests',
1259                                 'depends' : '@'
1260                         }
1261                         for fname in stz.keys():
1262                                 if fname.startswith(' '): continue
1263                                 vl = stz[fname]
1264                                 try: fclass = globals()['Field_'+
1265                                         fname.replace('-','_')]
1266                                 except KeyError: raise Unsupported(vl[0][0],
1267                                         'unknown metadata field %s' % fname)
1268                                 f = fclass(stz, fname, base, tnames, vl)
1269                                 f.parse()
1270                         for tname in tnames:
1271                                 t = Test(tname, base, act)
1272                                 stz[' tests'].append(t)
1273                 except Unsupported, u:
1274                         for tname in tnames: u.report(tname)
1275                         continue
1276
1277         return stanzas
1278
1279 def print_exception(ei, msgprefix=''):
1280         if msgprefix: pstderr(msgprefix)
1281         (et, q, tb) = ei
1282         if et is Quit:
1283                 pstderr('adt-run: ' + q.m)
1284                 psummary('quitting: '+q.m)
1285                 return q.ec
1286         else:
1287                 pstderr("adt-run: unexpected, exceptional, error:")
1288                 psummary('quitting: unexpected error, consult transcript')
1289                 traceback.print_exc(None, sys.stderr)
1290                 if trace_stream is not None:
1291                         traceback.print_exc(None, trace_stream)
1292                 return 20
1293
1294 def cleanup():
1295         global trace_stream
1296         try:
1297                 if testbed is not None:
1298                         testbed.reset_apt()
1299                         testbed.stop()
1300                 if opts.tmpdir is None and tmpdir is not None:
1301                         rmtree('tmpdir', tmpdir)
1302                 if trace_stream is not None:
1303                         trace_stream.close()
1304                         trace_stream = None
1305         except:
1306                 print_exception(sys.exc_info(),
1307                         '\nadt-run: error cleaning up:\n')
1308                 os._exit(20)
1309
1310 #---------- registration, installation etc. of .deb's: Binaries
1311
1312 def determine_package(act):
1313         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1314         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1315         if rc: badpkg('failed to parse binary package, code %d' % rc)
1316         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1317         act.pkg = None
1318         for l in output.split('\n'):
1319                 m = re.match(l)
1320                 if not m: continue
1321                 if act.pkg: badpkg('two Package: lines in control file')
1322                 act.pkg = m.groups()[0]
1323         if not act.pkg: badpkg('no good Package: line in control file')
1324
1325 class Binaries:
1326  def __init__(b, tb):
1327         b.dir = TemporaryDir('binaries')
1328         b.dir.write()
1329         ok = False
1330
1331         if opts.gnupghome is None:
1332                 opts.gnupghome = tmpdir+'/gnupg'
1333
1334         b._debug('initialising')
1335         try:
1336                 for x in ['pubring','secring']:
1337                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1338                 ok = True
1339         except (IOError,OSError), oe:
1340                 if oe.errno != errno.ENOENT: raise
1341
1342         if ok: b._debug('no key generation needed')
1343         else: b.genkey()
1344
1345  def _debug(b, s):
1346         debug('* '+s)
1347
1348  def genkey(b):
1349         b._debug('preparing for key generation')
1350
1351         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1352         mkdir_okexist(opts.gnupghome, 0700)
1353
1354         script = '''
1355   exec >&2
1356   cd "$1"
1357   cat <<"END" >key-gen-params
1358 Key-Type: DSA
1359 Key-Length: 1024
1360 Key-Usage: sign
1361 Name-Real: autopkgtest per-run key
1362 Name-Comment: do not trust this key
1363 Name-Email: autopkgtest@example.com
1364 END
1365   set -x
1366   gpg --homedir="$1" --batch --gen-key key-gen-params
1367                 '''
1368         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1369         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1370         if rc: bomb('key generation failed, code %d' % rc)
1371
1372  def apt_configs(b):
1373         return {
1374                 "Debug::pkgProblemResolver": "true",
1375                 "APT::Get::force-yes" : "true",
1376                 "APT::Get::Assume-Yes" : "true",
1377                 "quiet" : "true",
1378         }
1379
1380  def _configure_apt(b, tb):
1381         config = OutputFile('apt-config','/etc/apt/apt.conf.d/90autopkgtest',
1382                         True)
1383         f = open(config.write(),'w')
1384         for (k,v) in b.apt_configs().iteritems():
1385                 print >>f, '%s { "%s"; };' % (k, v)
1386         f.close()
1387         config.read(True)
1388
1389         prefs = OutputFile('apt-prefs','/etc/apt/preferences.d/90autopkgtest',
1390                         True)
1391         print >>open(prefs.write(),'w'), '''
1392 Package: *
1393 Pin: origin ""
1394 Pin-Priority: 1002
1395 '''
1396         prefs.read(True)
1397
1398  def _apt_get(b):
1399         ag = ['apt-get','-q']
1400         for kv in b.apt_configs().iteritems():
1401                 ag += ['-o', '%s=%s' % kv]
1402         return ' '.join(ag)
1403
1404  def reset(b):
1405         b._debug('reset')
1406         rmtree('binaries', b.dir.read())
1407         b.dir.invalidate()
1408         b.dir.write()
1409         b.install = []
1410         b.blamed = []
1411         b.registered = set()
1412
1413  def register(b, act, pkg, af, forwhat, blamed):
1414         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1415                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1416
1417         if act.ah['deb_'+forwhat] == 'ignore': return
1418
1419         b.blamed += testbed.blamed
1420
1421         leafname = pkg+'.deb'
1422         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1423
1424         try: os.remove(dest.write())
1425         except (IOError,OSError), oe:
1426                 if oe.errno != errno.ENOENT: raise e
1427
1428         try: os.link(af.read(), dest.write())
1429         except (IOError,OSError), oe:
1430                 if oe.errno != errno.EXDEV: raise e
1431                 shutil.copy(af.read(), dest)
1432
1433         if act.ah['deb_'+forwhat] == 'install':
1434                 b.install.append(pkg)
1435
1436         b.registered.add(pkg)
1437
1438  def publish(b):
1439         b._debug('publish')
1440
1441         b._configure_apt(testbed)
1442
1443         script = '''
1444   exec >&2
1445   cd "$1"
1446   apt-ftparchive packages . >Packages
1447   gzip <Packages >Packages.gz
1448   apt-ftparchive release . >Release
1449   rm -f Release.gpg
1450   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1451   gpg --homedir="$2" --batch --export >archive-key.pgp
1452                 '''
1453         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1454         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1455         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1456
1457         b.dir.invalidate(True)
1458         apt_source = b.dir.read(True)
1459
1460         so = TemporaryFile('vlds')
1461         script = '''
1462   exec 3>&1 >&2
1463   apt-key add archive-key.pgp
1464   echo "deb file://'''+apt_source+''' /" >/etc/apt/sources.list.d/autopkgtest.list
1465   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1466     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1467   fi
1468   '''+ b._apt_get() +''' update >&2
1469   cat /var/lib/dpkg/status >&3
1470                 '''
1471         testbed.mungeing_apt()
1472         rc = testbed.execute('apt-key', ['sh','-ec',script],
1473                                 so=so.write(True), cwd=b.dir.write(True),
1474                                 script=script, kind='install')
1475         if rc: bomb('apt setup failed with exit code %d' % rc)
1476
1477         testbed.blamed += b.blamed
1478
1479         b._debug('publish reinstall checking...')
1480         pkgs_reinstall = set()
1481         pkg = None
1482         for l in open(so.read()):
1483                 if l.startswith('Package: '):
1484                         pkg = l[9:].rstrip()
1485                 elif l.startswith('Status: install '):
1486                         if pkg in b.registered:
1487                                 pkgs_reinstall.add(pkg)
1488                                 b._debug(' publish reinstall needs '+pkg)
1489
1490         if pkgs_reinstall:
1491                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1492                 what = 'apt-get-reinstall'
1493                 cmdl = (b._apt_get() + ' --reinstall install '+
1494                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1495                 cmdl = ['sh','-c',cmdl]
1496                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1497                 if rc: badpkg("installation of basic binarries failed,"
1498                                 " exit code %d" % rc)
1499
1500         b._debug('publish install...')
1501         for pkg in b.install:
1502                 what = 'apt-get-install-%s' % pkg
1503                 testbed.blame(pkg)
1504                 cmdl = b._apt_get() + ' install ' + pkg + ' >&2'
1505                 cmdl = ['sh','-c',cmdl]
1506                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1507                 if rc: badpkg("installation of %s failed, exit code %d"
1508                                 % (pkg, rc))
1509
1510         b._debug('publish done')
1511
1512 #---------- processing of sources (building)
1513
1514 def source_rules_command(act,script,what,which,work,cwd,
1515                                 results_lines=0,xargs=[]):
1516         script = [      "exec 3>&1 >&2",
1517                         "set -x"        ] + script
1518         script = '\n'.join(script)
1519         so = TemporaryFile('%s-%s-results' % (what,which))
1520         rc = testbed.execute('%s-%s' % (what,which),
1521                         ['sh','-ec',script]+xargs, script=script,
1522                         so=so.write(True), cwd=cwd, kind='build')
1523         results = open(so.read()).read().rstrip('\n')
1524         if len(results): results = results.split("\n")
1525         else: results = []
1526         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1527         if results_lines is not None and len(results) != results_lines:
1528                 badpkg("got %d lines of results from %s where %d expected"
1529                         % (len(results), which, results_lines))
1530         if results_lines==1: return results[0]
1531         return results
1532
1533 def build_source(act, control_override):
1534         act.blame = 'arg:'+act.af.spec
1535         testbed.blame(act.blame)
1536         testbed.prepare1([])
1537         testbed.needs_reset()
1538
1539         what = act.what
1540         basename = act.af.spec
1541         debiancontrol = None
1542         act.binaries = []
1543
1544         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1545
1546         if act.kind == 'dsc':
1547                 dsc = act.af
1548                 dsc_file = open(dsc.read())
1549                 in_files = False
1550                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1551                 for l in dsc_file:
1552                         l = l.rstrip('\n')
1553                         if l.startswith('Files:'): in_files = True; continue
1554                         elif l.startswith('#'): pass
1555                         elif not l.startswith(' '):
1556                                 in_files = False
1557                                 if l.startswith('Source:'):
1558                                         act.blame = 'dsc:'+l[7:].strip()
1559                                         testbed.blame(act.blame)
1560                         if not in_files: continue
1561
1562                         m = fre.match(l)
1563                         if not m: badpkg(".dsc contains unparseable line"
1564                                         " in Files: `%s'" % l)
1565                         leaf = m.groups(0)[0]
1566                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1567                                         sibling=True)
1568                         subfile.read(True)
1569                 dsc.read(True)
1570
1571         if act.kind == 'ubtree':
1572                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1573                         act.af, 'debian/control')
1574                 dsc = TemporaryFile(what+'-fakedsc')
1575                 dsc_w = open(dsc.write(), 'w')
1576                 for l in open(debiancontrol.read()):
1577                         l = l.rstrip('\n')
1578                         if not len(l): break
1579                         print >>dsc_w, l
1580                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1581                 dsc_w.close()
1582
1583         if act.kind == 'dsc':
1584                 testbed.prepare2([])
1585                 testbed.satisfy_dependencies_string('dpkg-dev',
1586                                                 'install dpkg-dev')
1587
1588         work = TemporaryDir(what+'-build')
1589         act.work = work
1590
1591         tmpdir = work.write(True)+'/tmpdir'
1592         tmpdir_script = [
1593                         'TMPDIR="$1"',
1594                         'rm -rf -- "$TMPDIR"',
1595                         'export TMPDIR',
1596                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1597                 ]
1598
1599         if act.kind == 'ubtree':
1600                 spec = '%s/real-tree' % work.write(True)
1601                 create_command = '''
1602                         rm -rf "$spec"
1603                         mkdir "$spec"
1604                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1605                         '''
1606                 initcwd = act.af.read(True)
1607
1608         if act.kind == 'dsc':
1609                 spec = dsc.read(True)
1610                 create_command = '''
1611                         dpkg-source -x $spec
1612                         '''
1613                 initcwd = work.write(True)
1614
1615         script = [
1616                 'spec="$2"',
1617                 'origpwd=`pwd`',
1618                 'cd '+work.write(True)
1619         ]
1620
1621         if opts.user:
1622                 script += ([ 'chown '+opts.user+' .' ] +
1623                         tmpdir_script +
1624                         [ 'spec="$spec" origpwd="$origpwd" '
1625                                 +opts.user_wrap(create_command) ])
1626         else:
1627                 script += (tmpdir_script +
1628                         [ create_command ])
1629
1630         script += [
1631                         'cd [a-z0-9]*-*/.',
1632                         'pwd >&3',
1633                         'set +e; test -f debian/tests/control; echo $? >&3'
1634                 ]
1635         (result_pwd, control_test_rc) = source_rules_command(
1636                         act,script,what,'extract',work,
1637                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1638
1639         filter = act.ah['dsc_filter']
1640
1641         if control_test_rc == '1': act.missing_tests_control = True
1642
1643         # For optional builds:
1644         #
1645         # We might need to build the package because:
1646         #   - we want its binaries (filter isn't _ and at least one of the
1647         #       deb_... isn't ignore)
1648         #   - the test control file says so
1649         #       (assuming we have any tests)
1650
1651         class NeedBuildException: pass
1652         def build_needed(m):
1653                 debug_b('build needed for %s' % m)
1654                 raise NeedBuildException()
1655
1656         try:
1657                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1658                                       act.ah['deb_fortests'] != 'ignore'):
1659                         build_needed('binaries')
1660
1661                 result_pwd_af = InputDir(what+'-treeforcontrol',
1662                         result_pwd, True)
1663                 stanzas = read_control(act, result_pwd_af, control_override)
1664                 for stanza in stanzas:
1665                         for t in stanza[' tests']:
1666                                 if 'no-build-needed' not in t.feature_names:
1667                                         build_needed('test %s' % t.tname)
1668                                 for d in t.depends:
1669                                         if '@' in d:
1670                                                 build_needed('test %s '
1671                                                  'dependency %s' % (t.tname,d))
1672
1673                 debug_b('build not needed')
1674                 built = False
1675
1676         except NeedBuildException:
1677
1678                 if act.kind != 'dsc':
1679                         testbed.prepare2([])
1680
1681                 testbed.satisfy_dependencies_string('build-essential',
1682                                 'install build-essential')
1683                 testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')
1684
1685                 script = tmpdir_script + [
1686                         'cd "$2"',
1687                         'dpkg-checkbuilddeps',
1688                         opts.user_wrap('debian/rules build'),
1689                         ]
1690                 source_rules_command(act,script,what,'build',work,
1691                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1692
1693                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1694                         badpkg("results dir `%s' is not in expected parent"
1695                                " dir `%s'" % (result_pwd, work.read(True)))
1696
1697                 built = True
1698
1699         act.tests_tree = RelativeInputDir(what+'-tests-tree',
1700                                 work, os.path.basename(result_pwd),
1701                                 True)
1702
1703         if not built:
1704                 act.blamed = []
1705                 return
1706
1707         act.blamed = copy.copy(testbed.blamed)
1708
1709         debug_b('filter=%s' % filter)
1710         if filter != '_':
1711                 script = tmpdir_script + [
1712                         'cd '+work.write(True)+'/[a-z0-9]*-*/.',
1713                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1714                         'cd ..',
1715                         'echo *.deb >&3',
1716                         ]
1717                 result_debs = source_rules_command(act,script,what,
1718                                 'binary',work,work.write(True),
1719                                 results_lines=1, xargs=['x',tmpdir])
1720                 if result_debs == '*.deb': debs = []
1721                 else: debs = result_debs.split(' ')
1722                 debug_b('debs='+`debs`)
1723                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1724                 for deb in debs:
1725                         m = re.match(deb)
1726                         if not m: badpkg("badly-named binary `%s'" % deb)
1727                         pkg = m.groups()[0]
1728                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1729                         for pat in filter.split(','):
1730                                 debug_b('  pat=%s' % pat)
1731                                 if not fnmatch.fnmatchcase(pkg,pat):
1732                                         debug_b('   no match')
1733                                         continue
1734                                 deb_what = pkg+'_'+what+'.deb'
1735                                 bin = RelativeInputFile(deb_what,work,deb,True)
1736                                 debug_b('  deb_what=%s, bin=%s' %
1737                                         (deb_what, str(bin)))
1738                                 binaries.register(act,pkg,bin,
1739                                         'forbuilds',testbed.blamed)
1740                                 act.binaries.append((pkg,bin))
1741                                 break
1742                 debug_b('all done.')
1743
1744 #---------- main processing loop and main program
1745
1746 def process_actions():
1747         global binaries
1748
1749         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1750         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1751         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1752
1753         debug_a1('starting')
1754         testbed.open()
1755         binaries = Binaries(testbed)
1756
1757         binaries.reset()
1758         control_override = None
1759
1760         debug_a1('builds ...')
1761         for act in opts.actions:
1762                 debug_a2('%s %s' %
1763                         (act.kind, act.what))
1764
1765                 if act.kind == 'control':
1766                         control_override = act.af
1767                 if act.kind == 'deb':
1768                         testbed.blame('arg:'+act.af.spec)
1769                         determine_package(act)
1770                         testbed.blame('deb:'+act.pkg)
1771                         binaries.register(act,act.pkg,act.af,
1772                                 'forbuilds',testbed.blamed)
1773                 if act.kind == 'dsc' or act.kind == 'ubtree':
1774                         build_source(act, control_override)
1775                 if act.kind == 'tree':
1776                         act.binaries = []
1777                 if act.kind.endswith('tree') or act.kind == 'dsc':
1778                         control_override = None
1779                 if act.kind == 'instantiate':
1780                         pass
1781
1782         debug_a1('builds done.')
1783
1784         binaries.reset()
1785         control_override = None
1786
1787         debug_a1('tests ...')
1788         for act in opts.actions:
1789                 debug_a2('test %s %s' % (act.kind, act.what))
1790
1791                 testbed.needs_reset()
1792                 if act.kind == 'control':
1793                         control_override = act.af
1794                 if act.kind == 'deb':
1795                         binaries.register(act,act.pkg,act.af,'fortests',
1796                                 ['deb:'+act.pkg])
1797                 if act.kind == 'dsc' or act.kind == 'ubtree':
1798                         for (pkg,bin) in act.binaries:
1799                                 binaries.register(act,pkg,bin,'fortests',
1800                                         act.blamed)
1801                 if act.kind == 'dsc':
1802                         if act.ah['dsc_tests']:
1803                                 debug_a3('read control ...')
1804                                 stanzas = read_control(act, act.tests_tree,
1805                                                 control_override)
1806                                 testbed.blamed += act.blamed
1807                                 debug_a3('run_tests ...')
1808                                 run_tests(stanzas, act.tests_tree)
1809                         control_override = None
1810                 if act.kind == 'tree' or act.kind == 'ubtree':
1811                         testbed.blame('arg:'+act.af.spec)
1812                         stanzas = read_control(act, act.af, control_override)
1813                         debug_a3('run_tests ...')
1814                         run_tests(stanzas, act.af)
1815                         control_override = None
1816                 if act.kind == 'instantiate':
1817                         testbed.prepare([])
1818         debug_a1('tests done.')
1819
1820 def main():
1821         global testbed
1822         global tmpdir
1823         try:
1824                 parse_args()
1825         except SystemExit, se:
1826                 os._exit(20)
1827         try:
1828                 setup_trace()
1829                 testbed = Testbed()
1830                 testbed.start()
1831                 finalise_options()
1832                 process_actions()
1833         except:
1834                 ec = print_exception(sys.exc_info(), '')
1835                 cleanup()
1836                 os._exit(ec)
1837         cleanup()
1838         os._exit(errorcode)
1839
1840 main()