chiark / gitweb /
* Fix fd handling to work around Python's habit of closing files
[autopkgtest.git] / runner / adt-run
1 #!/usr/bin/python2.4
2 #
3 # adt-run is part of autopkgtest
4 # autopkgtest is a tool for testing Debian binary packages
5 #
6 # autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
7 #
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the GNU General Public License
19 # along with this program; if not, write to the Free Software
20 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 #
22 # See the file CREDITS for a full list of credits information (often
23 # installed as /usr/share/doc/autopkgtest/CREDITS).
24
25 import signal
26 import optparse
27 import tempfile
28 import sys
29 import subprocess
30 import traceback
31 import urllib
32 import string
33 import re as regexp
34 import os
35 import errno
36 import fnmatch
37 import shutil
38 import copy
39
40 from optparse import OptionParser
41 signal.signal(signal.SIGINT, signal.SIG_DFL) # undo stupid Python SIGINT thing
42
43 #---------- global variables
44
45 tmpdir = None           # pathstring on host
46 testbed = None          # Testbed
47 errorcode = 0           # exit status that we are going to use
48 timeouts = { 'short':10, 'install':900, 'test':600, 'build':3000 }
49 binaries = None         # Binaries (.debs we have registered)
50 build_essential = ["build-essential"]
51
52 #---------- output handling
53 #
54 #  There are at least the following kinds of output:
55 #
56 #   1. stderr output which consists of
57 #   1a. our errors
58 #   1b. stuff printed to stderr by the virtualisation server
59 #   1c. stuff printed to stderr by our short-lived subprocesseses
60 #           which we don't expect to fail
61 #
62 #   2. trace information, which consists of
63 #   2a. our trace information from calls to debug()
64 #   2b. progress information and stderr output from
65 #        general scripts we run on the host
66 #   2c. progress information and stderr output from things
67 #        we run on the testbed including builds
68 #   2d. stderr and stdout output from actual tests
69 #
70 #   xxxx
71 #   3. actual test results (printed to our stdout)
72 #
73 # Cloning of 1a and 2a, where necessary, is done by us writing
74 # the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
75 # is done by forking off a copy of ourselves to do plumbing,
76 # which copy we wait for at the appropriate point.
77
78 class DummyOpts:
79  def __init__(do): do.debuglevel = 0
80
81 opts = DummyOpts()
82 trace_stream = None
83 summary_stream = None
84
85 def pstderr(m):
86         print >>sys.stderr, m
87         if trace_stream is not None: print >>trace_stream, m
88
89 def debug(m, minlevel=0):
90         if opts.debuglevel < minlevel: return
91         if opts.quiet and trace_stream is None: return
92         p = 'adt-run: trace'
93         if minlevel: p += `minlevel`
94         p += ': '
95         for l in m.rstrip('\n').split('\n'):
96                 s = p + l
97                 if not opts.quiet: print >>sys.stderr, s
98                 if trace_stream is not None: print >>trace_stream, s
99
100 def debug_file(hp, minlevel=0):
101         if opts.debuglevel < minlevel: return
102         def do_copy(stream, what):
103                 rc = subprocess.call(['cat',hp], stdout=stream)
104                 if rc: bomb('cat failed copying data from %s'
105                             ' to %s, exit code %d' % (hp, what, rc))
106         if not opts.quiet: do_copy(sys.stderr, 'stderr')
107         if trace_stream is not None: do_copy(trace_stream, 'trace log')
108
109 class Errplumb:
110  def __init__(ep, critical=False):
111         to_stderr = critical or not opts.quiet
112         count = to_stderr + (trace_stream is not None)
113         if count == 0:
114                 ep.stream = open('/dev/null','w')
115                 ep._sp = None
116         elif count == 1:
117                 if to_stderr: ep.stream = os.dup(2)
118                 else: ep.stream = trace_stream
119                 ep._sp = None
120         else:
121                 ep._sp = subprocess.Popen(['tee','-a','/dev/stderr'],
122                         stdin=subprocess.PIPE, stdout=trace_stream,
123                         close_fds=True)
124                 ep.stream = ep._sp.stdin
125  def wait(ep):
126         if ep._sp is None: return
127         if type(ep.stream) == type(2):
128                 os.close(ep.stream)
129                 ep.stream = ()
130         ep._sp.stdin.close()
131         rc = ep._sp.wait()
132         if rc: bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
133         ep._sp = None
134
135 def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
136         if dbg is not None:
137                 if isinstance(dbg,tuple): (what,script) = dbg
138                 else: (what,script) = (dbg,None)
139                 debug_subprocess(what, cmdl, script=script)
140         ep = Errplumb(critical)
141         running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
142         output = running.communicate()[0]
143         rc = running.wait()
144         ep.wait()
145         return (rc, output)
146
147 def psummary(m):
148         if summary_stream is not None: print >>summary_stream, m
149
150 def preport(m):
151         print m
152         sys.stdout.flush()
153         if trace_stream is not None: print >>trace_stream, m
154         psummary(m)
155
156 def report(tname, result):
157         preport('%-20s %s' % (tname, result))
158
159 #---------- errors we define
160
161 class Quit:
162         def __init__(q,ec,m): q.ec = ec; q.m = m
163
164 def bomb(m):
165         raise Quit(20, "unexpected error: %s" % m)
166
167 def badpkg(m):
168         preport('blame: ' + ' '.join(testbed.blamed))
169         preport('badpkg: ' + m)
170         raise Quit(12, "erroneous package: %s" % m)
171
172 class Unsupported:
173  def __init__(u, lno, m):
174         if lno >= 0: u.m = '%s (control line %d)' % (m, lno)
175         else: u.m = m
176  def report(u, tname):
177         global errorcode
178         errorcode != 2
179         report(tname, 'SKIP %s' % u.m)
180
181 #---------- convenience function
182
183 def mkdir_okexist(pathname, mode=02755):
184         try:
185                 os.mkdir(pathname, mode)
186         except (IOError,OSError), oe:
187                 if oe.errno != errno.EEXIST: raise
188
189 def rmtree(what, pathname):
190         debug('/ %s rmtree %s' % (what, pathname), 2)
191         shutil.rmtree(pathname)
192
193 def debug_subprocess(what, cmdl=None, script=None):
194         o = '$ '+what+':'
195         if cmdl is not None:
196                 ol = []
197                 for x in cmdl:
198                         if x is script: x = '<SCRIPT>'
199                         ol.append(x.    replace('\\','\\\\').
200                                         replace(' ','\\ ')      )
201                 o += ' '+ ' '.join(ol)
202         debug(o)
203         if script is not None and opts.debuglevel >= 1:
204                 o = ''
205                 for l in script.rstrip('\n').split('\n'):
206                         o += '$     '+l+'\n'
207                 debug(o,1)
208
209 def flatten(l):
210         return reduce((lambda a,b: a + b), l, []) 
211
212 #---------- fancy automatic file-copying class
213
214 class AutoFile:
215         # p.what
216         # p.path[tb]    None or path    not None => path known
217         # p.file[tb]    None or path    not None => file exists
218         # p.spec        string or None
219         # p.spec_tbp    True or False, or not set if spec is None
220         # p.dir         '' or '/'
221
222  def __init__(p, what):
223         p.what = what
224         p.path = [None,None]
225         p.file = [None,None]
226         p.spec = None
227         p.dir = ''
228
229  def __repr__(p):
230         return "<AF@%s>" % p.__str__()
231  def __str__(p):
232         def ptbp(tbp):
233                 if p.path[tbp] is None: return '-'+p.dir
234                 elif p.file[tbp] is None: return p.path[tbp]+p.dir+'?'
235                 else: return p.path[tbp]+p.dir+'!'
236         out = p.what
237         if p.spec is not None:
238                 if not hasattr(p,'spec_tb'): out += '~'
239                 elif p.spec_tbp: out += '#'
240                 else: out += '='
241                 out += p.spec
242         out += ':'
243         out += ptbp(False)
244         out += '|'
245         out += ptbp(True)
246         return out
247
248  def _wrong(p, how):
249         xtra = ''
250         if p.spec is not None:
251                 xtra = ' spec[%s]=%s' % (p.spec, getattr(p,'spec_tb',None))
252         raise ("internal error: %s (%s)" % (how, str(p)))
253
254  def _ensure_path(p, tbp):
255         if p.path[tbp] is None:
256                 if '/' in p.what:
257                         p._debug('tmp-parent %s...' % 'HT'[tbp])
258                         TemporaryDir(os.path.dirname(p.what)).write(tbp)
259                 if not tbp:
260                         p.path[tbp] = tmpdir+'/'+p.what
261                 else:
262                         p.path[tbp] = testbed.scratch.path[True]+'/'+p.what
263
264  def write(p, tbp=False):
265         p._debug('write %s...' % 'HT'[tbp])
266         p._ensure_path(tbp)
267
268         if p.dir and not p.file[tbp]:
269                 if not tbp:
270                         p._debug('mkdir H')
271                         mkdir_okexist(p.path[tbp])
272                 else:
273                         cmdl = ['sh','-ec',
274                                 'test -d "$1" || mkdir -p "$1"',
275                                 'x', p.path[tbp]]
276                         tf_what = urllib.quote(p.what).replace('/','%2F')
277                         rc = testbed.execute('mkdir-'+tf_what, cmdl)
278                         if rc: bomb('failed to create directory %s' %
279                                 p.path[tbp])
280
281         p.file[tbp] = p.path[tbp]
282         return p.path[tbp]
283
284  def read(p, tbp=False):
285         p._debug('read %s...' % 'HT'[tbp])
286         p._ensure_path(tbp)
287
288         if p.file[tbp] is None:
289                 if p.file[not tbp] is None:
290                         p._wrong("requesting read but nonexistent")
291                 cud = ['copyup','copydown'][tbp]
292                 src = p.file[not tbp] + p.dir
293                 dst = p.path[tbp] + p.dir
294                 testbed.command(cud, (src, dst))
295                 p.file[tbp] = p.path[tbp]
296
297         return p.file[tbp] + p.dir
298
299  def invalidate(p, tbp=False):
300         p.file[tbp] = None
301         p._debug('invalidated %s' % 'HT'[tbp])
302
303  def _debug(p, m):
304         debug('/ %s#%x: %s' % (p.what, id(p), m), 3)
305
306  def _constructed(p):
307         p._debug('constructed: '+str(p))
308         p._check()
309
310  def _check(p):
311         for tbp in [False,True]:
312          for pf in [p.path, p.file]:
313                 if pf[tbp] is None: continue
314                 if not pf[tbp]: bomb('empty path specified for %s' % p.what)
315                 if p.dir and pf[tbp].endswith('/'):
316                         pf[tbp] = pf[tbp].rstrip('/')
317                         if not pf[tbp]: pf[tbp] = '/'
318                 if not p.dir and pf[tbp].endswith('/'):
319                         bomb("directory `%s' specified for "
320                              "non-directory %s" % (pf[tbp], p.what))
321
322  def _relative_init(p, what, parent, leaf, onlyon_tbp, setfiles, sibling):
323         AutoFile.__init__(p,what)
324         sh_on = ''; sh_sibl = ''
325         if onlyon_tbp is not None: sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
326         if sibling: sh_sibl=' (sibling)'
327         parent._debug('using as base: %s: %s%s%s' %
328                         (str(parent), leaf, sh_on, sh_sibl))
329         if not sibling and not parent.dir:
330                 parent._wrong('asked for non-sibling relative path of non-dir')
331         if sibling: trim = os.path.dirname
332         else: trim = lambda x: x
333         for tbp in [False,True]:
334                 if parent.path[tbp] is None: continue
335                 trimmed = trim(parent.path[tbp])
336                 if trimmed: trimmed += '/'
337                 p.path[tbp] = trimmed + leaf
338                 if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
339                         p.file[tbp] = p.path[tbp]
340
341 class InputFile(AutoFile):
342  def _init(p, what, spec, spec_tbp=False):
343         AutoFile.__init__(p, what)
344         p.spec = spec
345         p.spec_tbp = spec_tbp
346         p.path[spec_tbp] = spec
347         p.file[p.spec_tbp] = p.path[p.spec_tbp]
348  def __init__(p, what, spec, spec_tbp=False):
349         p._init(what,spec,spec_tbp)
350         p._constructed()
351
352 class InputDir(InputFile):
353  def __init__(p, what, spec, spec_tbp=False):
354         InputFile._init(p,what,spec,spec_tbp)
355         p.dir = '/'
356         p._constructed()
357
358 class OutputFile(AutoFile):
359  def _init(p, what, spec, spec_tbp=False):
360         AutoFile.__init__(p, what)
361         p.spec = spec
362         p.spec_tbp = spec_tbp
363         p.path[spec_tbp] = spec
364  def __init__(p, what, spec, spec_tbp=False):
365         p._init(what,spec,spec_tbp)
366         p._constructed()
367
368 class OutputDir(OutputFile):
369  def __init__(p, what, spec, spec_tbp=False):
370         OutputFile._init(p,what,spec,spec_tbp)
371         p.dir = '/'
372         p._constructed()
373
374 class RelativeInputFile(AutoFile):
375  def __init__(p, what, parent, leaf, onlyon_tbp=None, sibling=False):
376         p._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
377         p._constructed()
378
379 class RelativeOutputFile(AutoFile):
380  def __init__(p, what, parent, leaf, sibling=False):
381         p._relative_init(what, parent, leaf, None, False, sibling)
382         p._constructed()
383
384 class TemporaryFile(AutoFile):
385  def __init__(p, what):
386         AutoFile.__init__(p, what)
387         p._constructed()
388
389 class TemporaryDir(AutoFile):
390  def __init__(p, what):
391         AutoFile.__init__(p, what)
392         p.dir = '/'
393         p._constructed()
394
395 #---------- parsing and representation of the arguments
396
397 class Action:
398  def __init__(a, kind, af, arghandling, what):
399         # extra attributes get added during processing
400         a.kind = kind
401         a.af = af
402         a.ah = arghandling
403         a.what = what
404         a.missing_tests_control = False
405  def __repr__(a):
406         return "<Action %s %s %s>" % (a.kind, a.what, `a.af`)
407
408 def parse_args():
409         global opts
410         global n_non_actions # argh, stupid python scoping rules
411         usage = "%prog <options> --- <virt-server>..."
412         parser = OptionParser(usage=usage)
413         pa = parser.add_option
414         pe = parser.add_option
415
416         arghandling = {
417                 'dsc_tests': True,
418                 'dsc_filter': '*',
419                 'deb_forbuilds': 'auto',
420                 'deb_fortests': 'auto',
421                 'tb': False,
422                 'override_control': None
423         }
424         initial_arghandling = arghandling.copy()
425         n_non_actions = 0
426
427         #----------
428         # actions (ie, test sets to run, sources to build, binaries to use):
429
430         def cb_action(op,optstr,value,parser, long,kindpath,is_act):
431                 global n_non_actions
432                 parser.largs.append((value,kindpath))
433                 n_non_actions += not(is_act)
434
435         def pa_action(long, metavar, kindpath, help, is_act=True):
436                 pa('','--'+long, action='callback', callback=cb_action,
437                         nargs=1, type='string',
438                         callback_args=(long,kindpath,is_act), help=help)
439
440         pa_action('built-tree',         'TREE', '@/',
441                 help='run tests from build tree TREE')
442
443         pa_action('unbuilt-tree',       'TREE', '@//',
444                 help='run tests from build tree TREE')
445
446         pa_action('source',             'DSC', '@.dsc',
447                 help='build DSC and use its tests and/or'
448                     ' generated binary packages')
449
450         pa_action('binary',             'DEB', '@.deb',
451                help='use binary package DEB according'
452                     ' to most recent --binaries-* settings')
453
454         def cb_actnoarg(op,optstr,value,parser, largsentry):
455                 parser.largs.append(largsentry)
456         pa('','--instantiate', action='callback', callback=cb_actnoarg,
457                 callback_args=((None, ('instantiate',)),),
458                 help='instantiate testbed now (during testing phase)'
459                      ' and install packages'
460                      ' selected for automatic installation, even'
461                      ' if this might apparently not be required otherwise')
462
463         pa_action('override-control',   'CONTROL', ('control',), is_act=0,
464                help='run tests from control file CONTROL instead,'
465                     ' (applies to next test suite only)')
466
467         #----------
468         # argument handling settings (what ways to use action
469         #  arguments, and pathname processing):
470
471         def cb_setah(option, opt_str, value, parser, toset,setval):
472                 if type(setval) == list:
473                         if not value in setval:
474                                 parser.error('value for %s option (%s) is not '
475                                  'one of the permitted values (%s)' %
476                                  (value, opt_str, setval.join(' ')))
477                 elif setval is not None:
478                         value = setval
479                 for v in toset:
480                         arghandling[v] = value
481                 parser.largs.append(arghandling.copy())
482
483         def pa_setah(long, affected,effect, metavar=None, **kwargs):
484                 type = metavar
485                 if type is not None: type = 'string'
486                 pa('',long, action='callback', callback=cb_setah,
487                    callback_args=(affected,effect), **kwargs)
488
489         #---- paths: host or testbed:
490         #
491         pa_setah('--paths-testbed', ['tb'],True,
492                 help='subsequent path specifications refer to the testbed')
493         pa_setah('--paths-host', ['tb'],False,
494                 help='subsequent path specifications refer to the host')
495
496         #---- source processing settings:
497
498         pa_setah('--sources-tests', ['dsc_tests'],True,
499                 help='run tests from builds of subsequent sources')
500         pa_setah('--sources-no-tests', ['dsc_tests'],False,
501                 help='do not run tests from builds of subsequent sources')
502
503         pa_setah('--built-binaries-filter', ['dsc_filter'],None,
504                 type='string', metavar='PATTERN-LIST',
505                 help='from subsequent sources, use binaries matching'
506                      ' PATTERN-LIST (comma-separated glob patterns)'
507                      ' according to most recent --binaries-* settings')
508         pa_setah('--no-built-binaries', ['dsc_filter'], '_',
509                 help='from subsequent sources, do not use any binaries')
510
511         #---- binary package processing settings:
512
513         def pa_setahbins(long,toset,how):
514          pa_setah(long, toset,['ignore','auto','install'],
515                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
516                 help=how+' ignore binaries, install them as needed'
517                         ' for dependencies, or unconditionally install'
518                         ' them, respectively')
519         pa_setahbins('--binaries', ['deb_forbuilds','deb_fortests'], '')
520         pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
521         pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')
522
523         #----------
524         # general options:
525
526         def cb_vserv(op,optstr,value,parser):
527                 parser.values.vserver = list(parser.rargs)
528                 del parser.rargs[:]
529
530         def cb_path(op,optstr,value,parser, constructor,long,dir):
531                 name = long.replace('-','_')
532                 af = constructor(long, value,arghandling['tb'])
533                 setattr(parser.values, name, af)
534
535         def pa_path(long, constructor, help, dir=False):
536                 pa('','--'+long, action='callback', callback=cb_path,
537                         callback_args=(constructor,long,dir),
538                         nargs=1, type='string',
539                         help=help, metavar='PATH')
540
541         pa_path('output-dir', OutputDir, dir=True,
542                 help='write stderr/out files in PATH')
543
544         pa('','--tmp-dir',              type='string', dest='tmpdir',
545                 help='write temporary files to TMPDIR, emptying it'
546                      ' beforehand and leaving it behind at the end')
547         pa('','--log-file',             type='string', dest='logfile',
548                 help='write the log LOGFILE, emptying it beforehand,'
549                      ' instead of using OUTPUT-DIR/log or TMPDIR/log')
550         pa('','--summary-file',         type='string', dest='summary',
551                 help='write a summary report to SUMMARY,'
552                      ' emptying it beforehand')
553
554         pa('','--user',                 type='string', dest='user',
555                 help='run tests as USER (needs root on testbed)')
556         pa('','--gain-root',            type='string', dest='gainroot',
557                 help='prefix debian/rules binary with GAINROOT')
558         pa('-q', '--quiet', action='store_false', dest='quiet', default=False);
559         pa('-d', '--debug', action='count', dest='debuglevel', default=0);
560         pa('','--gnupg-home',           type='string', dest='gnupghome',
561                 default='~/.autopkgtest/gpg',
562                 help='use GNUPGHOME rather than ~/.autopkgtest (for'
563                         " signing private apt archive);"
564                         " `fresh' means generate new key each time.")
565
566         #----------
567         # actual meat:
568
569         class SpecialOption(optparse.Option): pass
570         vs_op = SpecialOption('','--VSERVER-DUMMY')
571         vs_op.action = 'callback'
572         vs_op.type = None
573         vs_op.default = None
574         vs_op.nargs = 0
575         vs_op.callback = cb_vserv
576         vs_op.callback_args = ( )
577         vs_op.callback_kwargs = { }
578         vs_op.help = 'introduces virtualisation server and args'
579         vs_op._short_opts = []
580         vs_op._long_opts = ['---']
581
582         pa(vs_op)
583
584         (opts,args) = parser.parse_args()
585         if not hasattr(opts,'vserver'):
586                 parser.error('you must specifiy --- <virt-server>...')
587         if n_non_actions >= len(parser.largs):
588                 parser.error('nothing to do specified')
589
590         arghandling = initial_arghandling
591         opts.actions = []
592         ix = 0
593         for act in args:
594                 if type(act) == dict:
595                         arghandling = act
596                         continue
597                 elif type(act) == tuple:
598                         pass
599                 elif type(act) == str:
600                         act = (act,act)
601                 else:
602                         raise ("unknown action in list `%s' having"
603                               " type `%s'" % (act, type(act)))
604                 (pathstr, kindpath) = act
605
606                 constructor = InputFile
607                 if type(kindpath) is tuple:             kind = kindpath[0]
608                 elif kindpath.endswith('.deb'):         kind = 'deb'
609                 elif kindpath.endswith('.dsc'):         kind = 'dsc'
610                 elif kindpath.endswith('//'):
611                         kind = 'ubtree'
612                         constructor = InputDir
613                 elif kindpath.endswith('/'):
614                         kind = 'tree'
615                         constructor = InputDir
616                 else: parser.error("do not know how to handle filename \`%s';"
617                         " specify --source --binary or --build-tree")
618
619                 what = '%s%s' % (kind,ix); ix += 1
620
621                 if kind == 'dsc': fwhatx = '/' + os.path.basename(pathstr)
622                 else: fwhatx = '-'+kind
623
624                 if pathstr is None: af = None
625                 else: af = constructor(what+fwhatx, pathstr, arghandling['tb'])
626                 opts.actions.append(Action(kind, af, arghandling, what))
627
628 def setup_trace():
629         global trace_stream, tmpdir, summary_stream
630
631         if opts.tmpdir is not None:
632                 rmtree('tmpdir(specified)',opts.tmpdir)
633                 mkdir_okexist(opts.tmpdir, 0700)
634                 tmpdir = opts.tmpdir
635         else:
636                 assert(tmpdir is None)
637                 tmpdir = tempfile.mkdtemp()
638
639         if opts.logfile is None:
640                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
641                         opts.logfile = opts.output_dir.spec + '/log'
642                 elif opts.tmpdir is not None:
643                         opts.logfile = opts.tmpdir + '/log'
644         if opts.logfile is not None:
645                 trace_stream = open(opts.logfile, 'w', 0)
646         if opts.summary is not None:
647                 summary_stream = open(opts.summary, 'w', 0)
648
649         debug('options: '+`opts`, 1)
650
651 def finalise_options():
652         global opts, tb, build_essential
653
654         if opts.user is None and 'root-on-testbed' not in testbed.caps:
655                 opts.user = ''
656
657         if opts.user is None:
658                 su = 'suggested-normal-user='
659                 ul = [
660                         e[len(su):]
661                         for e in testbed.caps
662                         if e.startswith(su)
663                         ]
664                 if ul:
665                         opts.user = ul[0]
666                 else:
667                         opts.user = ''
668
669         if opts.user:
670                 if 'root-on-testbed' not in testbed.caps:
671                         pstderr("warning: virtualisation"
672                                 " system does not offer root on testbed,"
673                                 " but --user option specified: failure likely")
674                 opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
675         else:
676                 opts.user_wrap = lambda x: x
677
678         if opts.gainroot is None:
679                 opts.gainroot = ''
680                 if (opts.user or
681                     'root-on-testbed' not in testbed.caps):
682                         opts.gainroot = 'fakeroot'
683                         build_essential += ['fakeroot']
684
685         if opts.gnupghome.startswith('~/'):
686                 try: home = os.environ['HOME']
687                 except KeyError:
688                         parser.error("HOME environment variable"
689                                 " not set, needed for --gnupghome=`%s"
690                                 % opts.gnupghome)
691                 opts.gnupghome = home + opts.gnupghome[1:]
692         elif opts.gnupghome == 'fresh':
693                 opts.gnupghome = None
694
695 #---------- testbed management - the Testbed class
696
697 class Testbed:
698  def __init__(tb):
699         tb.sp = None
700         tb.lastsend = None
701         tb.scratch = None
702         tb.modified = False
703         tb.blamed = []
704         tb._ephemeral = []
705         tb._debug('init')
706         tb._need_reset_apt = False
707  def _debug(tb, m, minlevel=0):
708         debug('** '+m, minlevel)
709  def start(tb):
710         tb._debug('start')
711         p = subprocess.PIPE
712         debug_subprocess('vserver', opts.vserver)
713         tb._errplumb = Errplumb(True)
714         tb.sp = subprocess.Popen(opts.vserver,
715                 stdin=p, stdout=p, stderr=tb._errplumb.stream)
716         tb.expect('ok')
717         tb.caps = tb.commandr('capabilities')
718  def stop(tb):
719         tb._debug('stop')
720         tb.close()
721         if tb.sp is None: return
722         ec = tb.sp.returncode
723         if ec is None:
724                 tb.sp.stdout.close()
725                 tb.send('quit')
726                 tb.sp.stdin.close()
727                 ec = tb.sp.wait()
728         if ec:
729                 tb.bomb('testbed gave exit status %d after quit' % ec)
730         tb._errplumb.wait()
731  def open(tb):
732         tb._debug('open, scratch=%s' % tb.scratch)
733         if tb.scratch is not None: return
734         pl = tb.commandr('open')
735         tb.scratch = InputDir('tb-scratch', pl[0], True)
736         tb.deps_processed = []
737  def mungeing_apt(tb):
738         if not 'revert' in tb.caps:
739                 tb._need_reset_apt = True
740  def reset_apt(tb):
741         if not tb._need_reset_apt: return
742         what = 'aptget-update-reset'
743         cmdl = ['sh','-c','apt-get -qy update 2>&1']
744         rc = tb.execute(what, cmdl, kind='install')
745         if rc:
746                 pstderr("\n" "warning: failed to restore"
747                         " testbed apt cache, exit code %d" % rc)
748         tb._need_reset_apt = False
749  def close(tb):
750         tb._debug('close, scratch=%s' % tb.scratch)
751         if tb.scratch is None: return
752         tb.scratch = None
753         if tb.sp is None: return
754         tb.command('close')
755  def prepare1(tb, deps_new):
756         tb._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
757                 (tb.modified, tb.deps_processed, deps_new), 1)
758         if 'revert' in tb.caps and (tb.modified or
759             [d for d in tb.deps_processed if d not in deps_new]):
760                 for af in tb._ephemeral: af.read(False)
761                 tb._debug('reset **')
762                 tb.command('revert')
763                 tb.blamed = []
764                 for af in tb._ephemeral: af.invalidate(True)
765         tb.modified = False
766  def prepare2(tb, deps_new):
767         tb._debug('prepare2, deps_new=%s' % deps_new, 1)
768         binaries.publish()
769         tb._install_deps(deps_new)
770  def prepare(tb, deps_new):
771         tb.prepare1(deps_new)
772         tb.prepare2(deps_new)
773  def register_ephemeral(tb, af):
774         tb._ephemeral.append(af)
775  def _install_deps(tb, deps_new):
776         tb._debug(' installing dependencies '+`deps_new`, 1)
777         tb.deps_processed = deps_new
778         if not deps_new: return
779         dstr = ', '.join(deps_new)
780         script = binaries.apt_pkg_gdebi_script(
781                 dstr, [[
782                 'from GDebi.DebPackage import DebPackage',
783                 'd = DebPackage(cache)',
784                 'res = d.satisfyDependsStr(arg)',
785                 ]])
786         cmdl = ['python','-c',script]
787         what = 'install-deps'
788         rc = testbed.execute(what+'-debinstall', cmdl, script=script,
789                                 kind='install')
790         if rc: badpkg('dependency install failed, exit code %d' % rc)
791  def needs_reset(tb):
792         tb._debug('needs_reset, previously=%s' % tb.modified, 1)
793         tb.modified = True
794  def blame(tb, m):
795         tb._debug('blame += %s' % m, 1)
796         tb.blamed.append(m)
797  def bomb(tb, m):
798         tb._debug('bomb %s' % m)
799         if tb.sp is not None:
800                 tb.sp.stdout.close()
801                 tb.sp.stdin.close()
802                 ec = tb.sp.wait()
803                 if ec: pstderr('adt-run: testbed failing,'
804                                 ' exit status %d' % ec)
805         tb.sp = None
806         raise Quit(16, 'testbed failed: %s' % m)
807  def send(tb, string):
808         tb.sp.stdin
809         try:
810                 debug('>> '+string, 2)
811                 print >>tb.sp.stdin, string
812                 tb.sp.stdin.flush()
813                 tb.lastsend = string
814         except:
815                 (type, value, dummy) = sys.exc_info()
816                 tb.bomb('cannot send to testbed: %s' % traceback.
817                         format_exception_only(type, value))
818  def expect(tb, keyword, nresults=None):
819         l = tb.sp.stdout.readline()
820         if not l: tb.bomb('unexpected eof from the testbed')
821         if not l.endswith('\n'): tb.bomb('unterminated line from the testbed')
822         l = l.rstrip('\n')
823         debug('<< '+l, 2)
824         ll = l.split()
825         if not ll: tb.bomb('unexpected whitespace-only line from the testbed')
826         if ll[0] != keyword:
827                 if tb.lastsend is None:
828                         tb.bomb("got banner `%s', expected `%s...'" %
829                                 (l, keyword))
830                 else:
831                         tb.bomb("sent `%s', got `%s', expected `%s...'" %
832                                 (tb.lastsend, l, keyword))
833         ll = ll[1:]
834         if nresults is not None and len(ll) != nresults:
835                 tb.bomb("sent `%s', got `%s' (%d result parameters),"
836                         " expected %d result parameters" %
837                         (string, l, len(ll), nresults))
838         return ll
839  def commandr(tb, cmd, args=(), nresults=None):
840         # pass args=[None,...] or =(None,...) to avoid more url quoting
841         if type(cmd) is str: cmd = [cmd]
842         if len(args) and args[0] is None: args = args[1:]
843         else: args = map(urllib.quote, args)
844         al = cmd + args
845         tb.send(string.join(al))
846         ll = tb.expect('ok', nresults)
847         rl = map(urllib.unquote, ll)
848         return rl
849  def command(tb, cmd, args=()):
850         tb.commandr(cmd, args, 0)
851  def commandr1(tb, cmd, args=()):
852         rl = tb.commandr(cmd, args, 1)
853         return rl[0]
854  def execute(tb, what, cmdl,
855                 si='/dev/null', so='/dev/null', se=None, cwd=None,
856                 script=False, tmpdir=None, kind='short'):
857         # Options for script:
858         #   False - do not call debug_subprocess, no synch. reporting required
859         #   None or string - call debug_subprocess with that value,
860         #                       plumb stderr through synchronously if possible
861         # Options for se:
862         #   None - usual Errplumb (output is of kind 2c)
863         #   string - path on testbed (output is of kind 2d)
864
865         timeout = timeouts[kind]
866
867         if script is not False: debug_subprocess(what, cmdl, script=script)
868         if cwd is None: cwd = tb.scratch.write(True)
869
870         xdump = None
871         if se is None:
872                 ep = Errplumb()
873                 se_catch = TemporaryFile(what+'-xerr')
874                 se_use = se_catch.write(True)
875                 if not opts.quiet: xdump = 'debug=2-2'
876                 elif trace_stream is not None:
877                         xdump = 'debug=2-%d' % trace_stream.fileno()
878         else:
879                 ep = None
880                 se_catch = None
881                 se_use = se
882
883         cmdl = [None,
884                 ','.join(map(urllib.quote, cmdl)),
885                 si, so, se_use, cwd]
886
887         if timeout is not None and timeout > 0:
888                 cmdl.append('timeout=%d' % timeout)
889
890         if xdump is not None and 'execute-debug' in tb.caps: cmdl += [xdump]
891         if tmpdir is not None: cmdl.append('env=TMPDIR=%s' % tmpdir)
892         if kind=='install': cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
893
894         rc = tb.commandr1('execute', cmdl)
895         try: rc = int(rc)
896         except ValueError: bomb("execute for %s gave invalid response `%s'"
897                                         % (what,rc))
898
899         if se_catch is not None:
900                 debug_file(se_catch.read())
901         if ep is not None:
902                 ep.wait()
903
904         return rc
905
906 #---------- representation of test control files: Field*, Test, etc.
907
908 class FieldBase:
909  def __init__(f, fname, stz, base, tnames, vl):
910         assert(vl)
911         f.stz = stz
912         f.base = base
913         f.tnames = tnames
914         f.vl = vl
915  def words(f):
916         def distribute(vle):
917                 (lno, v) = vle
918                 r = v.split()
919                 r = map((lambda w: (lno, w)), r)
920                 return r
921         return flatten(map(distribute, f.vl))
922  def atmostone(f):
923         if len(vl) == 1:
924                 (f.lno, f.v) = vl[0]
925         else:
926                 raise Unsupported(f.vl[1][0],
927                         'only one %s field allowed' % fn)
928         return f.v
929
930 class FieldIgnore(FieldBase):
931  def parse(f): pass
932
933 class Restriction:
934  def __init__(r,rname,base): pass
935
936 class Restriction_rw_build_tree(Restriction): pass
937 class Restriction_breaks_testbed(Restriction):
938  def __init__(r, rname, base):
939         if 'revert' not in testbed.caps:
940                 raise Unsupported(f.lno,
941                         'Test breaks testbed but testbed cannot revert')
942 class Restriction_needs_root(Restriction):
943  def __init__(r, rname, base):
944         if 'root-on-testbed' not in testbed.caps:
945                 raise Unsupported(f.lno,
946                         'Test needs root on testbed which is not available')
947
948 class Field_Restrictions(FieldBase):
949  def parse(f):
950         for wle in f.words():
951                 (lno, rname) = wle
952                 nrname = rname.replace('-','_')
953                 try: rclass = globals()['Restriction_'+nrname]
954                 except KeyError: raise Unsupported(lno,
955                         'unknown restriction %s' % rname)
956                 r = rclass(nrname, f.base)
957                 f.base['restriction_names'].append(rname)
958                 f.base['restrictions'].append(r)
959
960 class Field_Features(FieldIgnore):
961  def parse(f):
962         for wle in f.words():
963                 (lno, fname) = wle
964                 f.base['feature_names'].append(fname)
965                 nfname = fname.replace('-','_')
966                 try: fclass = globals()['Feature_'+nfname]
967                 except KeyError: continue
968                 ft = fclass(nfname, f.base)
969                 f.base['features'].append(ft)
970
971 class Field_Tests(FieldIgnore): pass
972
973 class Field_Depends(FieldBase):
974  def parse(f):
975         print >>sys.stderr, "Field_Depends:", `f.stz`, `f.base`, `f.tnames`, `f.vl`
976         dl = map(lambda x: x.strip(),
977                 flatten(map(lambda (lno, v): v.split(','), f.vl)))
978         re = regexp.compile('[^-.+:~0-9a-z()<>=*@]')
979         for d in dl:
980                 if re.search(d):
981                         badpkg("Test Depends field contains dependency"
982                                " `%s' with invalid characters" % d)
983         f.base['depends'] = dl
984
985 class Field_Tests_directory(FieldBase):
986  def parse(f):
987         td = atmostone(f)
988         if td.startswith('/'): raise Unspported(f.lno,
989                 'Tests-Directory may not be absolute')
990         f.base['testsdir'] = td
991
992 def run_tests(stanzas, tree):
993         global errorcode, testbed
994         if stanzas == ():
995                 report('*', 'SKIP no tests in this package')
996                 errorcode |= 8
997         for stanza in stanzas:
998                 tests = stanza[' tests']
999                 if not tests:
1000                         report('*', 'SKIP package has metadata but no tests')
1001                         errorcode |= 8
1002                 for t in tests:
1003                         t.prepare()
1004                         t.run(tree)
1005                         if 'breaks-testbed' in t.restriction_names:
1006                                 testbed.needs_reset()
1007                 testbed.needs_reset()
1008
1009 class Test:
1010  def __init__(t, tname, base, act):
1011         if '/' in tname: raise Unsupported(base[' lno'],
1012                 'test name may not contain / character')
1013         for k in base: setattr(t,k,base[k])
1014         t.tname = tname
1015         t.act = act
1016         t.what = act.what+'t-'+tname
1017         if len(base['testsdir']): t.path = base['testsdir'] + '/' + tname
1018         else: t.path = tname
1019         t._debug('constructed; path=%s' % t.path)
1020         t._debug(' .depends=%s' % t.depends)
1021  def _debug(t, m):
1022         debug('& %s: %s' % (t.what, m))
1023  def report(t, m):
1024         report(t.what, m)
1025  def reportfail(t, m):
1026         global errorcode
1027         errorcode |= 4
1028         report(t.what, 'FAIL ' + m)
1029  def prepare(t):
1030         t._debug('preparing')
1031         dn = []
1032         for d in t.depends:
1033                 t._debug(' processing dependency '+d)
1034                 if not '@' in d:
1035                         t._debug('  literal dependency '+d)
1036                         dn.append(d)
1037                 else:
1038                         for (pkg,bin) in t.act.binaries:
1039                                 d = d.replace('@',pkg)
1040                                 t._debug('  synthesised dependency '+d)
1041                                 dn.append(d)
1042         testbed.prepare(dn)
1043  def run(t, tree):
1044         t._debug('[----------------------------------------')
1045         def stdouterr(oe):
1046                 idstr = t.what + '-' + oe
1047                 if opts.output_dir is not None and opts.output_dir.spec_tbp:
1048                         use_dir = opts.output_dir
1049                 else:
1050                         use_dir = testbed.scratch
1051                 return RelativeOutputFile(idstr, use_dir, idstr)
1052
1053         t.act.work.write(True)
1054
1055         af = RelativeInputFile(t.what, tree, t.path)
1056         so = stdouterr('stdout')
1057         se = stdouterr('stderr')
1058
1059         tf = af.read(True)
1060         tmpdir = None
1061         tree.read(True)
1062
1063         rc = testbed.execute('testchmod-'+t.what, ['chmod','+x','--',tf])
1064         if rc: bomb('failed to chmod +x %s' % tf)
1065
1066         if 'needs-root' not in t.restriction_names and opts.user is not None:
1067                 tfl = ['su',opts.user,'-c',tf]
1068                 tmpdir = '%s%s-tmpdir' % (testbed.scratch.read(True), t.what)
1069                 script = 'rm -rf -- "$1"; mkdir -- "$1"'
1070                 if opts.user: script += '; chown %s "$1"' % opts.user
1071                 if 'rw-build-tree' in t.restriction_names:
1072                         script += '; chown -R %s "$2"' % opts.user
1073                 rc = testbed.execute('mktmpdir-'+t.what,
1074                         ['sh','-xec',script,'x',tmpdir,tree.read(True)])
1075                 if rc: bomb("could not create test tmpdir `%s', exit code %d"
1076                                 % (tmpdir, rc))
1077         else:
1078                 tfl = [tf]
1079
1080         rc = testbed.execute('test-'+t.what, tfl,
1081                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
1082                 tmpdir=tmpdir, kind='test')
1083
1084         so_read = so.read()
1085         se_read = se.read()
1086
1087         t._debug(' - - - - - - - - - - results - - - - - - - - - -')
1088         stab = os.stat(se_read)
1089         if stab.st_size != 0:
1090                 l = open(se_read).readline()
1091                 l = l.rstrip('\n \t\r')
1092                 if len(l) > 35: l = l[:35] + '...'
1093                 t.reportfail('status: %d, stderr: %s' % (rc, l))
1094                 t._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
1095                 debug_file(se_read)
1096         elif rc != 0:
1097                 t.reportfail('non-zero exit status %d' % rc)
1098         else:
1099                 t.report('PASS')
1100
1101         stab = os.stat(so_read)
1102         if stab.st_size != 0:
1103                 t._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
1104                 debug_file(so_read)
1105
1106         t._debug('----------------------------------------]')
1107
1108 def read_control(act, tree, control_override):
1109         stanzas = [ ]
1110
1111         if control_override is not None:
1112                 control_af = control_override
1113                 testbed.blame('arg:'+control_override.spec)
1114         else:
1115                 if act.missing_tests_control:
1116                         return ()
1117                 control_af = RelativeInputFile(act.what+'-testcontrol',
1118                         tree, 'debian/tests/control')
1119         try:
1120                 control = open(control_af.read(), 'r')
1121         except (IOError,OSError), oe:
1122                 if oe[0] != errno.ENOENT: raise
1123                 return []
1124
1125         lno = 0
1126         def badctrl(m): act.bomb('tests/control line %d: %s' % (lno, m))
1127         stz = None      # stz[field_name][index] = (lno, value)
1128                         # special field names:
1129                         # stz[' lno'] = number
1130                         # stz[' tests'] = list of Test objects
1131         def end_stanza(stz):
1132                 if stz is None: return
1133                 stz[' errs'] = 0
1134                 stanzas.append(stz)
1135                 stz = None
1136                 hcurrent = None
1137
1138         initre = regexp.compile('([A-Z][-0-9a-z]*)\s*\:\s*(.*)$')
1139         while 1:
1140                 l = control.readline()
1141                 if not l: break
1142                 lno += 1
1143                 if not l.endswith('\n'): badctrl('unterminated line')
1144                 if regexp.compile('\s*\#').match(l): continue
1145                 if not regexp.compile('\S').match(l): end_stanza(stz); continue
1146                 initmat = initre.match(l)
1147                 if initmat:
1148                         (fname, l) = initmat.groups()
1149                         fname = string.capwords(fname)
1150                         if stz is None:
1151                                 stz = { ' lno': lno, ' tests': [] }
1152                         if not stz.has_key(fname): stz[fname] = [ ]
1153                         hcurrent = stz[fname]
1154                 elif regexp.compile('\s').match(l):
1155                         if not hcurrent: badctrl('unexpected continuation')
1156                 else:
1157                         badctrl('syntax error')
1158                 hcurrent.append((lno, l))
1159         end_stanza(stz)
1160
1161         def testbadctrl(stz, lno, m):
1162                 report_badctrl(lno, m)
1163                 stz[' errs'] += 1
1164
1165         for stz in stanzas:
1166                 try:
1167                         try: tnames = stz['Tests']
1168                         except KeyError:
1169                                 tnames = ['*']
1170                                 raise Unsupported(stz[' lno'],
1171                                         'no Tests field')
1172                         tnames = map((lambda lt: lt[1]), tnames)
1173                         tnames = string.join(tnames).split()
1174                         base = {
1175                                 'restriction_names': [],
1176                                 'restrictions': [],
1177                                 'feature_names': [],
1178                                 'features': [],
1179                                 'testsdir': 'debian/tests',
1180                                 'depends' : '@'
1181                         }
1182                         for fname in stz.keys():
1183                                 if fname.startswith(' '): continue
1184                                 vl = stz[fname]
1185                                 try: fclass = globals()['Field_'+
1186                                         fname.replace('-','_')]
1187                                 except KeyError: raise Unsupported(vl[0][0],
1188                                         'unknown metadata field %s' % fname)
1189                                 f = fclass(stz, fname, base, tnames, vl)
1190                                 f.parse()
1191                         for tname in tnames:
1192                                 t = Test(tname, base, act)
1193                                 stz[' tests'].append(t)
1194                 except Unsupported, u:
1195                         for tname in tnames: u.report(tname)
1196                         continue
1197
1198         return stanzas
1199
1200 def print_exception(ei, msgprefix=''):
1201         if msgprefix: pstderr(msgprefix)
1202         (et, q, tb) = ei
1203         if et is Quit:
1204                 pstderr('adt-run: ' + q.m)
1205                 psummary('quitting: '+q.m)
1206                 return q.ec
1207         else:
1208                 pstderr("adt-run: unexpected, exceptional, error:")
1209                 psummary('quitting: unexpected error, consult transcript')
1210                 traceback.print_exc(None, sys.stderr)
1211                 if trace_stream is not None:
1212                         traceback.print_exc(None, trace_stream)
1213                 return 20
1214
1215 def cleanup():
1216         global trace_stream
1217         try:
1218                 rm_ec = 0
1219                 if opts.tmpdir is None and tmpdir is not None:
1220                         rmtree('tmpdir', tmpdir)
1221                 if testbed is not None:
1222                         testbed.reset_apt()
1223                         testbed.stop()
1224                 if rm_ec: bomb('rm -rf -- %s failed, code %d' % (tmpdir, ec))
1225                 if trace_stream is not None:
1226                         trace_stream.close()
1227                         trace_stream = None
1228         except:
1229                 print_exception(sys.exc_info(),
1230                         '\nadt-run: error cleaning up:\n')
1231                 os._exit(20)
1232
1233 #---------- registration, installation etc. of .deb's: Binaries
1234
1235 def determine_package(act):
1236         cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
1237         (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
1238         if rc: badpkg('failed to parse binary package, code %d' % rc)
1239         re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
1240         act.pkg = None
1241         for l in output.split('\n'):
1242                 m = re.match(l)
1243                 if not m: continue
1244                 if act.pkg: badpkg('two Package: lines in control file')
1245                 act.pkg = m.groups()[0]
1246         if not act.pkg: badpkg('no good Package: line in control file')
1247
1248 class Binaries:
1249  def __init__(b):
1250         b.dir = TemporaryDir('binaries')
1251         b.dir.write()
1252         ok = False
1253
1254         if opts.gnupghome is None:
1255                 opts.gnupghome = tmpdir+'/gnupg'
1256
1257         b._debug('initialising')
1258         try:
1259                 for x in ['pubring','secring']:
1260                         os.stat(opts.gnupghome + '/' + x + '.gpg')
1261                 ok = True
1262         except (IOError,OSError), oe:
1263                 if oe.errno != errno.ENOENT: raise
1264
1265         if ok: b._debug('no key generation needed')
1266         else: b.genkey()
1267
1268  def _debug(b, s):
1269         debug('* '+s)
1270
1271  def genkey(b):
1272         b._debug('preparing for key generation')
1273
1274         mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
1275         mkdir_okexist(opts.gnupghome, 0700)
1276
1277         script = '''
1278   exec >&2
1279   cd "$1"
1280   cat <<"END" >key-gen-params
1281 Key-Type: DSA
1282 Key-Length: 1024
1283 Key-Usage: sign
1284 Name-Real: autopkgtest per-run key
1285 Name-Comment: do not trust this key
1286 Name-Email: autopkgtest@example.com
1287 END
1288   set -x
1289   gpg --homedir="$1" --batch --gen-key key-gen-params
1290                 '''
1291         cmdl = ['sh','-ec',script,'x',opts.gnupghome]
1292         rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
1293         if rc: bomb('key generation failed, code %d' % rc)
1294
1295  def apt_configs(b):
1296         return {
1297                 "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
1298         }
1299
1300  def apt_pkg_gdebi_script(b, arg, middle):
1301         script = [
1302                 'import apt_pkg',
1303                 'import urllib',
1304                 'arg = urllib.unquote("%s")' % urllib.quote(arg),
1305                 ]
1306         for (k,v) in b.apt_configs().iteritems():
1307                 v = urllib.quote(v)
1308                 script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
1309                                 % (k, v))
1310         script += [
1311                 'from GDebi.Cache import Cache',
1312                 'cache = Cache()',
1313                 ]
1314         for m in middle:
1315                 script += m + [
1316                 'print res',
1317                 'print d.missingDeps',
1318                 'print d.requiredChanges',
1319                 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
1320                         ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
1321                           'd._failureString)',
1322                 'cache.commit()',
1323                 ''
1324                 ]
1325         return '\n'.join(script)
1326  def apt_get(b):
1327         ag = ['apt-get','-qy']
1328         for kv in b.apt_configs().iteritems():
1329                 ag += ['-o', '%s=%s' % kv]
1330         return ' '.join(ag)
1331
1332  def reset(b):
1333         b._debug('reset')
1334         rmtree('binaries', b.dir.read())
1335         b.dir.invalidate()
1336         b.dir.write()
1337         b.install = []
1338         b.blamed = []
1339         b.registered = set()
1340
1341  def register(b, act, pkg, af, forwhat, blamed):
1342         b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
1343                 % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
1344
1345         if act.ah['deb_'+forwhat] == 'ignore': return
1346
1347         b.blamed += testbed.blamed
1348
1349         leafname = pkg+'.deb'
1350         dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
1351
1352         try: os.remove(dest.write())
1353         except (IOError,OSError), oe:
1354                 if oe.errno != errno.ENOENT: raise e
1355
1356         try: os.link(af.read(), dest.write())
1357         except (IOError,OSError), oe:
1358                 if oe.errno != errno.EXDEV: raise e
1359                 shutil.copy(af.read(), dest)
1360
1361         if act.ah['deb_'+forwhat] == 'install':
1362                 b.install.append(pkg)
1363
1364         b.registered.add(pkg)
1365
1366  def publish(b):
1367         b._debug('publish')
1368
1369         script = '''
1370   exec >&2
1371   cd "$1"
1372   apt-ftparchive packages . >Packages
1373   gzip <Packages >Packages.gz
1374   apt-ftparchive release . >Release
1375   rm -f Release.gpg
1376   gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
1377   gpg --homedir="$2" --batch --export >archive-key.pgp
1378                 '''
1379         cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
1380         rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
1381         if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
1382
1383         b.dir.invalidate(True)
1384         apt_source = b.dir.read(True)
1385
1386         so = TemporaryFile('vlds')
1387         script = '''
1388   exec 3>&1 >&2
1389   apt-key add archive-key.pgp
1390   echo "deb file://'''+apt_source+''' /" >sources.list
1391   cat /etc/apt/sources.list >>sources.list
1392   if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
1393     echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
1394   fi
1395   '''+ b.apt_get() +''' update >&2
1396   cat /var/lib/dpkg/status >&3
1397                 '''
1398         testbed.mungeing_apt()
1399         rc = testbed.execute('apt-key', ['sh','-ec',script],
1400                                 so=so.write(True), cwd=b.dir.write(True),
1401                                 script=script, kind='install')
1402         if rc: bomb('apt setup failed with exit code %d' % rc)
1403
1404         testbed.blamed += b.blamed
1405
1406         b._debug('publish reinstall checking...')
1407         pkgs_reinstall = set()
1408         pkg = None
1409         for l in open(so.read()):
1410                 if l.startswith('Package: '):
1411                         pkg = l[9:].rstrip()
1412                 elif l.startswith('Status: install '):
1413                         if pkg in b.registered:
1414                                 pkgs_reinstall.add(pkg)
1415                                 b._debug(' publish reinstall needs '+pkg)
1416
1417         if pkgs_reinstall:
1418                 for pkg in pkgs_reinstall: testbed.blame(pkg)
1419                 what = 'apt-get-reinstall'
1420                 cmdl = (b.apt_get() + ' --reinstall install '+
1421                         ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
1422                 cmdl = ['sh','-c',cmdl]
1423                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1424                 if rc: badpkg("installation of basic binarries failed,"
1425                                 " exit code %d" % rc)
1426
1427         b._debug('publish install...')
1428         for pkg in b.install:
1429                 what = 'apt-get-install-%s' % pkg
1430                 testbed.blame(pkg)
1431                 cmdl = b.apt_get() + ' install ' + pkg + ' >&2'
1432                 cmdl = ['sh','-c',cmdl]
1433                 rc = testbed.execute(what, cmdl, script=None, kind='install')
1434                 if rc: badpkg("installation of %s failed, exit code %d"
1435                                 % (pkg, rc))
1436
1437         b._debug('publish done')
1438
1439 #---------- processing of sources (building)
1440
1441 def source_rules_command(act,script,what,which,work,cwd,
1442                                 results_lines=0,xargs=[]):
1443         script = [      "exec 3>&1 >&2",
1444                         "set -x"        ] + script
1445         script = '\n'.join(script)
1446         so = TemporaryFile('%s-%s-results' % (what,which))
1447         rc = testbed.execute('%s-%s' % (what,which),
1448                         ['sh','-ec',script]+xargs, script=script,
1449                         so=so.write(True), cwd=cwd, kind='build')
1450         results = open(so.read()).read().rstrip('\n')
1451         if len(results): results = results.split("\n")
1452         else: results = []
1453         if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
1454         if results_lines is not None and len(results) != results_lines:
1455                 badpkg("got %d lines of results from %s where %d expected"
1456                         % (len(results), which, results_lines))
1457         if results_lines==1: return results[0]
1458         return results
1459
1460 def build_source(act, control_override):
1461         act.blame = 'arg:'+act.af.spec
1462         testbed.blame(act.blame)
1463         testbed.prepare1([])
1464         testbed.needs_reset()
1465
1466         what = act.what
1467         basename = act.af.spec
1468         debiancontrol = None
1469         act.binaries = []
1470
1471         def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
1472
1473         if act.kind == 'dsc':
1474                 dsc = act.af
1475                 dsc_file = open(dsc.read())
1476                 in_files = False
1477                 fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
1478                 for l in dsc_file:
1479                         l = l.rstrip('\n')
1480                         if l.startswith('Files:'): in_files = True; continue
1481                         elif l.startswith('#'): pass
1482                         elif not l.startswith(' '):
1483                                 in_files = False
1484                                 if l.startswith('Source:'):
1485                                         act.blame = 'dsc:'+l[7:].strip()
1486                                         testbed.blame(act.blame)
1487                         if not in_files: continue
1488
1489                         m = fre.match(l)
1490                         if not m: badpkg(".dsc contains unparseable line"
1491                                         " in Files: `%s'" % l)
1492                         leaf = m.groups(0)[0]
1493                         subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
1494                                         sibling=True)
1495                         subfile.read(True)
1496                 dsc.read(True)
1497
1498         if act.kind == 'ubtree':
1499                 debiancontrol = RelativeInputFile(what+'-debiancontrol',
1500                         act.af, 'debian/control')
1501                 dsc = TemporaryFile(what+'-fakedsc')
1502                 dsc_w = open(dsc.write(), 'w')
1503                 for l in open(debiancontrol.read()):
1504                         l = l.rstrip('\n')
1505                         if not len(l): break
1506                         print >>dsc_w, l
1507                 print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
1508                 dsc_w.close()
1509
1510         if act.kind == 'dsc':
1511                 testbed.prepare2([])
1512                 script = binaries.apt_pkg_gdebi_script('', [[
1513                                 'from GDebi.DebPackage import DebPackage',
1514                                 'd = DebPackage(cache)',
1515                                 'res = d.satisfyDependsStr("dpkg-dev")',
1516                         ]])
1517                 cmdl = ['python','-c',script]
1518                 whatp = what+'-dpkgsource'
1519                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1520                 if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
1521
1522         work = TemporaryDir(what+'-build')
1523         act.work = work
1524
1525         tmpdir = work.write(True)+'/tmpdir'
1526         tmpdir_script = [
1527                         'TMPDIR="$1"',
1528                         'rm -rf -- "$TMPDIR"',
1529                         'export TMPDIR',
1530                         opts.user_wrap('mkdir -- "$TMPDIR"'),
1531                 ]
1532
1533         if act.kind == 'ubtree':
1534                 spec = '%s/real-tree' % work.write(True)
1535                 create_command = '''
1536                         rm -rf "$spec"
1537                         mkdir "$spec"
1538                         cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
1539                         '''
1540                 initcwd = act.af.read(True)
1541
1542         if act.kind == 'dsc':
1543                 spec = dsc.read(True)
1544                 create_command = '''
1545                         dpkg-source -x $spec
1546                         '''
1547                 initcwd = work.write(True)
1548
1549         script = [
1550                 'spec="$2"',
1551                 'origpwd=`pwd`',
1552                 'cd '+work.write(True)
1553         ]
1554
1555         if opts.user:
1556                 script += ([ 'chown '+opts.user+' .' ] +
1557                         tmpdir_script +
1558                         [ 'spec="$spec" origpwd="$origpwd" '
1559                                 +opts.user_wrap(create_command) ])
1560         else:
1561                 script += (tmpdir_script +
1562                         [ create_command ])
1563
1564         script += [
1565                         'cd */.',
1566                         'pwd >&3',
1567                         'set +e; test -f debian/tests/control; echo $? >&3'
1568                 ]
1569         (result_pwd, control_test_rc) = source_rules_command(
1570                         act,script,what,'extract',work,
1571                         cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
1572
1573         filter = act.ah['dsc_filter']
1574
1575         if control_test_rc == '1': act.missing_tests_control = True
1576
1577         # For optional builds:
1578         #
1579         # We might need to build the package because:
1580         #   - we want its binaries (filter isn't _ and at least one of the
1581         #       deb_... isn't ignore)
1582         #   - the test control file says so
1583         #       (assuming we have any tests)
1584
1585         class NeedBuildException: pass
1586         def build_needed(m):
1587                 debug_b('build needed for %s' % m)
1588                 raise NeedBuildException()
1589
1590         try:
1591                 if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
1592                                       act.ah['deb_fortests'] != 'ignore'):
1593                         build_needed('binaries')
1594
1595                 result_pwd_af = InputDir(what+'-treeforcontrol',
1596                         result_pwd, True)
1597                 stanzas = read_control(act, result_pwd_af, control_override)
1598                 for stanza in stanzas:
1599                         for t in stanza[' tests']:
1600                                 if 'no-build-needed' not in t.feature_names:
1601                                         build_needed('test %s' % t.tname)
1602                                 for d in t.depends:
1603                                         if '@' in d:
1604                                                 build_needed('test %s '
1605                                                  'dependency %s' % (t.tname,d))
1606
1607                 debug_b('build not needed')
1608                 built = False
1609
1610         except NeedBuildException:
1611
1612                 if act.kind != 'dsc':
1613                         testbed.prepare2([])
1614
1615                 script = binaries.apt_pkg_gdebi_script(
1616                         dsc.read(True), [[
1617                         'from GDebi.DscSrcPackage import DscSrcPackage',
1618                         'd = DscSrcPackage(cache, arg)',
1619                         'res = d.checkDeb()',
1620                          ],[
1621                         'from GDebi.DebPackage import DebPackage',
1622                         'd = DebPackage(cache)',
1623                         'res = d.satisfyDependsStr("'+
1624                                         ','.join(build_essential)+
1625                                 '")',
1626                         ]])
1627
1628                 cmdl = ['python','-c',script]
1629                 whatp = what+'-builddeps'
1630                 rc = testbed.execute(what, cmdl, script=script, kind='install')
1631                 if rc: badpkg('build-depends install failed,'
1632                               ' exit code %d' % rc)
1633
1634                 script = tmpdir_script + [
1635                         'cd "$2"',
1636                         'dpkg-checkbuilddeps',
1637                         opts.user_wrap('debian/rules build'),
1638                         ]
1639                 source_rules_command(act,script,what,'build',work,
1640                                 cwd=initcwd, xargs=['x',tmpdir,result_pwd])
1641
1642                 if os.path.dirname(result_pwd)+'/' != work.read(True):
1643                         badpkg("results dir `%s' is not in expected parent"
1644                                " dir `%s'" % (result_pwd, work.read(True)))
1645
1646                 built = True
1647
1648         act.tests_tree = InputDir(what+'-tests-tree',
1649                                 work.read(True)+os.path.basename(result_pwd),
1650                                 True)
1651         if act.ah['dsc_tests']:
1652                 testbed.register_ephemeral(act.work)
1653                 testbed.register_ephemeral(act.tests_tree)
1654
1655         if not built:
1656                 act.blamed = []
1657                 return
1658
1659         act.blamed = copy.copy(testbed.blamed)
1660
1661         debug_b('filter=%s' % filter)
1662         if filter != '_':
1663                 script = tmpdir_script + [
1664                         'cd '+work.write(True)+'/*/.',
1665                         opts.user_wrap(opts.gainroot+' debian/rules binary'),
1666                         'cd ..',
1667                         'echo *.deb >&3',
1668                         ]
1669                 result_debs = source_rules_command(act,script,what,
1670                                 'binary',work,work.write(True),
1671                                 results_lines=1, xargs=['x',tmpdir])
1672                 if result_debs == '*': debs = []
1673                 else: debs = result_debs.split(' ')
1674                 debug_b('debs='+`debs`)
1675                 re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
1676                 for deb in debs:
1677                         m = re.match(deb)
1678                         if not m: badpkg("badly-named binary `%s'" % deb)
1679                         pkg = m.groups()[0]
1680                         debug_b(' deb=%s, pkg=%s' % (deb,pkg))
1681                         for pat in filter.split(','):
1682                                 debug_b('  pat=%s' % pat)
1683                                 if not fnmatch.fnmatchcase(pkg,pat):
1684                                         debug_b('   no match')
1685                                         continue
1686                                 deb_what = pkg+'_'+what+'.deb'
1687                                 bin = RelativeInputFile(deb_what,work,deb,True)
1688                                 debug_b('  deb_what=%s, bin=%s' %
1689                                         (deb_what, str(bin)))
1690                                 binaries.register(act,pkg,bin,
1691                                         'forbuilds',testbed.blamed)
1692                                 act.binaries.append((pkg,bin))
1693                                 break
1694                 debug_b('all done.')
1695
1696 #---------- main processing loop and main program
1697
1698 def process_actions():
1699         global binaries
1700
1701         def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
1702         def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
1703         def debug_a3(m): debug('@@@@@@@@@@ '+m)
1704
1705         debug_a1('starting')
1706         testbed.open()
1707         binaries = Binaries()
1708
1709         for act in opts.actions:
1710                 if act.af is not None and not act.af.spec_tbp:
1711                         testbed.register_ephemeral(act.af)
1712
1713         binaries.reset()
1714         control_override = None
1715
1716         debug_a1('builds ...')
1717         for act in opts.actions:
1718                 debug_a2('%s %s' %
1719                         (act.kind, act.what))
1720
1721                 if act.kind == 'control':
1722                         control_override = act.af
1723                 if act.kind == 'deb':
1724                         testbed.blame('arg:'+act.af.spec)
1725                         determine_package(act)
1726                         testbed.blame('deb:'+act.pkg)
1727                         binaries.register(act,act.pkg,act.af,
1728                                 'forbuilds',testbed.blamed)
1729                 if act.kind == 'dsc' or act.kind == 'ubtree':
1730                         build_source(act, control_override)
1731                 if act.kind == 'tree':
1732                         act.binaries = []
1733                 if act.kind.endswith('tree') or act.kind == 'dsc':
1734                         control_override = None
1735                 if act.kind == 'instantiate':
1736                         pass
1737
1738         debug_a1('builds done.')
1739
1740         binaries.reset()
1741         control_override = None
1742
1743         debug_a1('tests ...')
1744         for act in opts.actions:
1745                 debug_a2('test %s %s' % (act.kind, act.what))
1746
1747                 testbed.needs_reset()
1748                 if act.kind == 'control':
1749                         control_override = act.af
1750                 if act.kind == 'deb':
1751                         binaries.register(act,act.pkg,act.af,'fortests',
1752                                 ['deb:'+act.pkg])
1753                 if act.kind == 'dsc' or act.kind == 'ubtree':
1754                         for (pkg,bin) in act.binaries:
1755                                 binaries.register(act,pkg,bin,'fortests',
1756                                         act.blamed)
1757                 if act.kind == 'dsc':
1758                         if act.ah['dsc_tests']:
1759                                 debug_a3('read control ...')
1760                                 stanzas = read_control(act, act.tests_tree,
1761                                                 control_override)
1762                                 testbed.blamed += act.blamed
1763                                 debug_a3('run_tests ...')
1764                                 run_tests(stanzas, act.tests_tree)
1765                         control_override = None
1766                 if act.kind == 'tree' or act.kind == 'ubtree':
1767                         testbed.blame('arg:'+act.af.spec)
1768                         stanzas = read_control(act, act.af, control_override)
1769                         debug_a3('run_tests ...')
1770                         run_tests(stanzas, act.af)
1771                         control_override = None
1772                 if act.kind == 'instantiate':
1773                         testbed.prepare([])
1774         debug_a1('tests done.')
1775
1776 def main():
1777         global testbed
1778         global tmpdir
1779         try:
1780                 parse_args()
1781         except SystemExit, se:
1782                 os._exit(20)
1783         try:
1784                 setup_trace()
1785                 testbed = Testbed()
1786                 testbed.start()
1787                 finalise_options()
1788                 process_actions()
1789         except:
1790                 ec = print_exception(sys.exc_info(), '')
1791                 cleanup()
1792                 os._exit(ec)
1793         cleanup()
1794         os._exit(errorcode)
1795
1796 main()