+#---------- registration, installation etc. of .deb's: Binaries
+
+def determine_package(act):
+ cmd = 'dpkg-deb --info --'.split(' ')+[act.af.read(),'control']
+ (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
+ if rc: badpkg('failed to parse binary package, code %d' % rc)
+ re = regexp.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
+ act.pkg = None
+ for l in output.split('\n'):
+ m = re.match(l)
+ if not m: continue
+ if act.pkg: badpkg('two Package: lines in control file')
+ act.pkg = m.groups()[0]
+ if not act.pkg: badpkg('no good Package: line in control file')
+
+class Binaries:
+ def __init__(b):
+ b.dir = TemporaryDir('binaries')
+ b.dir.write()
+ ok = False
+
+ if opts.gnupghome is None:
+ opts.gnupghome = tmpdir+'/gnupg'
+
+ b._debug('initialising')
+ try:
+ for x in ['pubring','secring']:
+ os.stat(opts.gnupghome + '/' + x + '.gpg')
+ ok = True
+ except (IOError,OSError), oe:
+ if oe.errno != errno.ENOENT: raise
+
+ if ok: b._debug('no key generation needed')
+ else: b.genkey()
+
+ def _debug(b, s):
+ debug('* '+s)
+
+ def genkey(b):
+ b._debug('preparing for key generation')
+
+ mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
+ mkdir_okexist(opts.gnupghome, 0700)
+
+ script = '''
+ exec >&2
+ cd "$1"
+ cat <<"END" >key-gen-params
+Key-Type: DSA
+Key-Length: 1024
+Key-Usage: sign
+Name-Real: autopkgtest per-run key
+Name-Comment: do not trust this key
+Name-Email: autopkgtest@example.com
+END
+ set -x
+ gpg --homedir="$1" --batch --gen-key key-gen-params
+ '''
+ cmdl = ['sh','-ec',script,'x',opts.gnupghome]
+ rc = subprocess_cooked(cmdl, dbg=('genkey',script))[0]
+ if rc: bomb('key generation failed, code %d' % rc)
+
+ def apt_configs(b):
+ return {
+ "Dir::Etc::sourcelist": b.dir.read(True)+'sources.list',
+ "Debug::pkgProblemResolver": "true",
+ }
+
+ def apt_pkg_gdebi_script(b, arg, middle):
+ script = [
+ 'import apt_pkg',
+ 'import urllib',
+ 'arg = urllib.unquote("%s")' % urllib.quote(arg),
+ ]
+ for (k,v) in b.apt_configs().iteritems():
+ v = urllib.quote(v)
+ script.append('apt_pkg.Config.Set("%s",urllib.unquote("%s"))'
+ % (k, v))
+ script += [
+ 'from GDebi.Cache import Cache',
+ 'cache = Cache()',
+ ]
+ for m in middle:
+ script += m + [
+ 'print res',
+ 'print d.missingDeps',
+ 'print d.requiredChanges',
+ 'if not res: raise "gdebi failed (%s, %s, %s): %s" % '+
+ ' (`res`, `d.missingDeps`, `d.requiredChanges`, '+
+ 'd._failureString)',
+ 'cache.commit()',
+ ''
+ ]
+ return '\n'.join(script)
+ def apt_get(b):
+ ag = ['apt-get','-qy']
+ for kv in b.apt_configs().iteritems():
+ ag += ['-o', '%s=%s' % kv]
+ return ' '.join(ag)
+
+ def reset(b):
+ b._debug('reset')
+ rmtree('binaries', b.dir.read())
+ b.dir.invalidate()
+ b.dir.write()
+ b.install = []
+ b.blamed = []
+ b.registered = set()
+
+ def register(b, act, pkg, af, forwhat, blamed):
+ b._debug('register what=%s deb_%s=%s pkg=%s af=%s'
+ % (act.what, forwhat, act.ah['deb_'+forwhat], pkg, str(af)))
+
+ if act.ah['deb_'+forwhat] == 'ignore': return
+
+ b.blamed += testbed.blamed
+
+ leafname = pkg+'.deb'
+ dest = RelativeOutputFile('binaries--'+leafname, b.dir, leafname)
+
+ try: os.remove(dest.write())
+ except (IOError,OSError), oe:
+ if oe.errno != errno.ENOENT: raise e
+
+ try: os.link(af.read(), dest.write())
+ except (IOError,OSError), oe:
+ if oe.errno != errno.EXDEV: raise e
+ shutil.copy(af.read(), dest)
+
+ if act.ah['deb_'+forwhat] == 'install':
+ b.install.append(pkg)
+
+ b.registered.add(pkg)
+
+ def publish(b):
+ b._debug('publish')
+
+ script = '''
+ exec >&2
+ cd "$1"
+ apt-ftparchive packages . >Packages
+ gzip <Packages >Packages.gz
+ apt-ftparchive release . >Release
+ rm -f Release.gpg
+ gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
+ gpg --homedir="$2" --batch --export >archive-key.pgp
+ '''
+ cmdl = ['sh','-ec',script,'x',b.dir.write(),opts.gnupghome]
+ rc = subprocess_cooked(cmdl, dbg=('ftparchive',script))[0]
+ if rc: bomb('apt-ftparchive or signature failed, code %d' % rc)
+
+ b.dir.invalidate(True)
+ apt_source = b.dir.read(True)
+
+ so = TemporaryFile('vlds')
+ script = '''
+ exec 3>&1 >&2
+ apt-key add archive-key.pgp
+ echo "deb file://'''+apt_source+''' /" >sources.list
+ cat /etc/apt/sources.list >>sources.list
+ if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
+ echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
+ fi
+ '''+ b.apt_get() +''' update >&2
+ cat /var/lib/dpkg/status >&3
+ '''
+ testbed.mungeing_apt()
+ rc = testbed.execute('apt-key', ['sh','-ec',script],
+ so=so.write(True), cwd=b.dir.write(True),
+ script=script, kind='install')
+ if rc: bomb('apt setup failed with exit code %d' % rc)
+
+ testbed.blamed += b.blamed
+
+ b._debug('publish reinstall checking...')
+ pkgs_reinstall = set()
+ pkg = None
+ for l in open(so.read()):
+ if l.startswith('Package: '):
+ pkg = l[9:].rstrip()
+ elif l.startswith('Status: install '):
+ if pkg in b.registered:
+ pkgs_reinstall.add(pkg)
+ b._debug(' publish reinstall needs '+pkg)
+
+ if pkgs_reinstall:
+ for pkg in pkgs_reinstall: testbed.blame(pkg)
+ what = 'apt-get-reinstall'
+ cmdl = (b.apt_get() + ' --reinstall install '+
+ ' '.join([pkg for pkg in pkgs_reinstall])+' >&2')
+ cmdl = ['sh','-c',cmdl]
+ rc = testbed.execute(what, cmdl, script=None, kind='install')
+ if rc: badpkg("installation of basic binarries failed,"
+ " exit code %d" % rc)
+
+ b._debug('publish install...')
+ for pkg in b.install:
+ what = 'apt-get-install-%s' % pkg
+ testbed.blame(pkg)
+ cmdl = b.apt_get() + ' install ' + pkg + ' >&2'
+ cmdl = ['sh','-c',cmdl]
+ rc = testbed.execute(what, cmdl, script=None, kind='install')
+ if rc: badpkg("installation of %s failed, exit code %d"
+ % (pkg, rc))
+
+ b._debug('publish done')
+
+#---------- processing of sources (building)
+
+def source_rules_command(act,script,what,which,work,cwd,
+ results_lines=0,xargs=[]):
+ script = [ "exec 3>&1 >&2",
+ "set -x" ] + script
+ script = '\n'.join(script)
+ so = TemporaryFile('%s-%s-results' % (what,which))
+ rc = testbed.execute('%s-%s' % (what,which),
+ ['sh','-ec',script]+xargs, script=script,
+ so=so.write(True), cwd=cwd, kind='build')
+ results = open(so.read()).read().rstrip('\n')
+ if len(results): results = results.split("\n")
+ else: results = []
+ if rc: badpkg("rules %s failed with exit code %d" % (which,rc))
+ if results_lines is not None and len(results) != results_lines:
+ badpkg("got %d lines of results from %s where %d expected"
+ % (len(results), which, results_lines))
+ if results_lines==1: return results[0]
+ return results
+
+def build_source(act, control_override):
+ act.blame = 'arg:'+act.af.spec
+ testbed.blame(act.blame)
+ testbed.prepare1([])
+ testbed.needs_reset()
+
+ what = act.what
+ basename = act.af.spec
+ debiancontrol = None
+ act.binaries = []
+
+ def debug_b(m): debug('* <%s:%s> %s' % (act.kind, act.what, m))
+
+ if act.kind == 'dsc':
+ dsc = act.af
+ dsc_file = open(dsc.read())
+ in_files = False
+ fre = regexp.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
+ for l in dsc_file:
+ l = l.rstrip('\n')
+ if l.startswith('Files:'): in_files = True; continue
+ elif l.startswith('#'): pass
+ elif not l.startswith(' '):
+ in_files = False
+ if l.startswith('Source:'):
+ act.blame = 'dsc:'+l[7:].strip()
+ testbed.blame(act.blame)
+ if not in_files: continue
+
+ m = fre.match(l)
+ if not m: badpkg(".dsc contains unparseable line"
+ " in Files: `%s'" % l)
+ leaf = m.groups(0)[0]
+ subfile = RelativeInputFile(what+'/'+leaf, dsc, leaf,
+ sibling=True)
+ subfile.read(True)
+ dsc.read(True)
+
+ if act.kind == 'ubtree':
+ debiancontrol = RelativeInputFile(what+'-debiancontrol',
+ act.af, 'debian/control')
+ dsc = TemporaryFile(what+'-fakedsc')
+ dsc_w = open(dsc.write(), 'w')
+ for l in open(debiancontrol.read()):
+ l = l.rstrip('\n')
+ if not len(l): break
+ print >>dsc_w, l
+ print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
+ dsc_w.close()
+
+ if act.kind == 'dsc':
+ testbed.prepare2([])
+ script = binaries.apt_pkg_gdebi_script('', [[
+ 'from GDebi.DebPackage import DebPackage',
+ 'd = DebPackage(cache)',
+ 'res = d.satisfyDependsStr("dpkg-dev")',
+ ]])
+ cmdl = ['python','-c',script]
+ whatp = what+'-dpkgsource'
+ rc = testbed.execute(what, cmdl, script=script, kind='install')
+ if rc: badpkg('dpkg-source install failed, exit code %d' % rc)
+
+ work = TemporaryDir(what+'-build')
+ act.work = work
+
+ tmpdir = work.write(True)+'/tmpdir'
+ tmpdir_script = [
+ 'TMPDIR="$1"',
+ 'rm -rf -- "$TMPDIR"',
+ 'export TMPDIR',
+ opts.user_wrap('mkdir -- "$TMPDIR"'),
+ ]
+
+ if act.kind == 'ubtree':
+ spec = '%s/real-tree' % work.write(True)
+ create_command = '''
+ rm -rf "$spec"
+ mkdir "$spec"
+ cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
+ '''
+ initcwd = act.af.read(True)
+
+ if act.kind == 'dsc':
+ spec = dsc.read(True)
+ create_command = '''
+ dpkg-source -x $spec
+ '''
+ initcwd = work.write(True)
+
+ script = [
+ 'spec="$2"',
+ 'origpwd=`pwd`',
+ 'cd '+work.write(True)
+ ]
+
+ if opts.user:
+ script += ([ 'chown '+opts.user+' .' ] +
+ tmpdir_script +
+ [ 'spec="$spec" origpwd="$origpwd" '
+ +opts.user_wrap(create_command) ])
+ else:
+ script += (tmpdir_script +
+ [ create_command ])
+
+ script += [
+ 'cd [a-z0-9]*-*/.',
+ 'pwd >&3',
+ 'set +e; test -f debian/tests/control; echo $? >&3'
+ ]
+ (result_pwd, control_test_rc) = source_rules_command(
+ act,script,what,'extract',work,
+ cwd=initcwd, results_lines=2, xargs=['x',tmpdir,spec])
+
+ filter = act.ah['dsc_filter']
+
+ if control_test_rc == '1': act.missing_tests_control = True
+
+ # For optional builds:
+ #
+ # We might need to build the package because:
+ # - we want its binaries (filter isn't _ and at least one of the
+ # deb_... isn't ignore)
+ # - the test control file says so
+ # (assuming we have any tests)
+
+ class NeedBuildException: pass
+ def build_needed(m):
+ debug_b('build needed for %s' % m)
+ raise NeedBuildException()
+
+ try:
+ if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
+ act.ah['deb_fortests'] != 'ignore'):
+ build_needed('binaries')
+
+ result_pwd_af = InputDir(what+'-treeforcontrol',
+ result_pwd, True)
+ stanzas = read_control(act, result_pwd_af, control_override)
+ for stanza in stanzas:
+ for t in stanza[' tests']:
+ if 'no-build-needed' not in t.feature_names:
+ build_needed('test %s' % t.tname)
+ for d in t.depends:
+ if '@' in d:
+ build_needed('test %s '
+ 'dependency %s' % (t.tname,d))
+
+ debug_b('build not needed')
+ built = False
+
+ except NeedBuildException:
+
+ if act.kind != 'dsc':
+ testbed.prepare2([])
+
+ script = binaries.apt_pkg_gdebi_script(
+ dsc.read(True), [[
+ 'from GDebi.DscSrcPackage import DscSrcPackage',
+ 'd = DscSrcPackage(cache, arg)',
+ 'res = d.checkDeb()',
+ ],[
+ 'from GDebi.DebPackage import DebPackage',
+ 'd = DebPackage(cache)',
+ 'res = d.satisfyDependsStr("'+
+ ','.join(build_essential)+
+ '")',
+ ]])
+
+ cmdl = ['python','-c',script]
+ whatp = what+'-builddeps'
+ rc = testbed.execute(what, cmdl, script=script, kind='install')
+ if rc: badpkg('build-depends install failed,'
+ ' exit code %d' % rc)
+
+ script = tmpdir_script + [
+ 'cd "$2"',
+ 'dpkg-checkbuilddeps',
+ opts.user_wrap('debian/rules build'),
+ ]
+ source_rules_command(act,script,what,'build',work,
+ cwd=initcwd, xargs=['x',tmpdir,result_pwd])
+
+ if os.path.dirname(result_pwd)+'/' != work.read(True):
+ badpkg("results dir `%s' is not in expected parent"
+ " dir `%s'" % (result_pwd, work.read(True)))
+
+ built = True
+
+ act.tests_tree = InputDir(what+'-tests-tree',
+ work.read(True)+os.path.basename(result_pwd),
+ True)
+ if act.ah['dsc_tests']:
+ testbed.register_ephemeral(act.work)
+ testbed.register_ephemeral(act.tests_tree)
+
+ if not built:
+ act.blamed = []
+ return
+
+ act.blamed = copy.copy(testbed.blamed)
+
+ debug_b('filter=%s' % filter)
+ if filter != '_':
+ script = tmpdir_script + [
+ 'cd '+work.write(True)+'/[a-z0-9]*-*/.',
+ opts.user_wrap(opts.gainroot+' debian/rules binary'),
+ 'cd ..',
+ 'echo *.deb >&3',
+ ]
+ result_debs = source_rules_command(act,script,what,
+ 'binary',work,work.write(True),
+ results_lines=1, xargs=['x',tmpdir])
+ if result_debs == '*.deb': debs = []
+ else: debs = result_debs.split(' ')
+ debug_b('debs='+`debs`)
+ re = regexp.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
+ for deb in debs:
+ m = re.match(deb)
+ if not m: badpkg("badly-named binary `%s'" % deb)
+ pkg = m.groups()[0]
+ debug_b(' deb=%s, pkg=%s' % (deb,pkg))
+ for pat in filter.split(','):
+ debug_b(' pat=%s' % pat)
+ if not fnmatch.fnmatchcase(pkg,pat):
+ debug_b(' no match')
+ continue
+ deb_what = pkg+'_'+what+'.deb'
+ bin = RelativeInputFile(deb_what,work,deb,True)
+ debug_b(' deb_what=%s, bin=%s' %
+ (deb_what, str(bin)))
+ binaries.register(act,pkg,bin,
+ 'forbuilds',testbed.blamed)
+ act.binaries.append((pkg,bin))
+ break
+ debug_b('all done.')
+
+#---------- main processing loop and main program
+
+def process_actions():
+ global binaries
+
+ def debug_a1(m): debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ '+m)
+ def debug_a2(m): debug('@@@@@@@@@@@@@@@@@@@@ '+m)
+ def debug_a3(m): debug('@@@@@@@@@@ '+m)
+
+ debug_a1('starting')
+ testbed.open()
+ binaries = Binaries()
+
+ for act in opts.actions:
+ if act.af is not None and not act.af.spec_tbp:
+ testbed.register_ephemeral(act.af)
+
+ binaries.reset()
+ control_override = None
+
+ debug_a1('builds ...')
+ for act in opts.actions:
+ debug_a2('%s %s' %
+ (act.kind, act.what))
+
+ if act.kind == 'control':
+ control_override = act.af
+ if act.kind == 'deb':
+ testbed.blame('arg:'+act.af.spec)
+ determine_package(act)
+ testbed.blame('deb:'+act.pkg)
+ binaries.register(act,act.pkg,act.af,
+ 'forbuilds',testbed.blamed)
+ if act.kind == 'dsc' or act.kind == 'ubtree':
+ build_source(act, control_override)
+ if act.kind == 'tree':
+ act.binaries = []
+ if act.kind.endswith('tree') or act.kind == 'dsc':
+ control_override = None
+ if act.kind == 'instantiate':
+ pass
+
+ debug_a1('builds done.')
+
+ binaries.reset()
+ control_override = None
+
+ debug_a1('tests ...')
+ for act in opts.actions:
+ debug_a2('test %s %s' % (act.kind, act.what))
+
+ testbed.needs_reset()
+ if act.kind == 'control':
+ control_override = act.af
+ if act.kind == 'deb':
+ binaries.register(act,act.pkg,act.af,'fortests',
+ ['deb:'+act.pkg])
+ if act.kind == 'dsc' or act.kind == 'ubtree':
+ for (pkg,bin) in act.binaries:
+ binaries.register(act,pkg,bin,'fortests',
+ act.blamed)
+ if act.kind == 'dsc':
+ if act.ah['dsc_tests']:
+ debug_a3('read control ...')
+ stanzas = read_control(act, act.tests_tree,
+ control_override)
+ testbed.blamed += act.blamed
+ debug_a3('run_tests ...')
+ run_tests(stanzas, act.tests_tree)
+ control_override = None
+ if act.kind == 'tree' or act.kind == 'ubtree':
+ testbed.blame('arg:'+act.af.spec)
+ stanzas = read_control(act, act.af, control_override)
+ debug_a3('run_tests ...')
+ run_tests(stanzas, act.af)
+ control_override = None
+ if act.kind == 'instantiate':
+ testbed.prepare([])
+ debug_a1('tests done.')
+