chiark / gitweb /
use jarsigner and keytool from same JDK as is being set in JAVA7_HOME
[fdroidserver.git] / fdroidserver / common.py
index 4c3597f2ad8d332bc4f6aef9f84cffc20ef24435..c30d3b4e07ed7d6492a47351f1959d5e83b395be 100644 (file)
@@ -29,13 +29,18 @@ import stat
 import subprocess
 import time
 import operator
-import Queue
 import logging
 import hashlib
 import socket
 import xml.etree.ElementTree as XMLElementTree
 
-from distutils.version import LooseVersion
+try:
+    # Python 2
+    from Queue import Queue
+except ImportError:
+    # Python 3
+    from queue import Queue
+
 from zipfile import ZipFile
 
 import metadata
@@ -56,7 +61,8 @@ default_config = {
         'r9b': None,
         'r10e': "$ANDROID_NDK",
     },
-    'build_tools': "23.0.1",
+    'build_tools': "23.0.2",
+    'java_paths': None,
     'ant': "ant",
     'mvn3': "mvn",
     'gradle': 'gradle',
@@ -122,7 +128,41 @@ def fill_config_defaults(thisconfig):
             thisconfig[k] = exp
             thisconfig[k + '_orig'] = v
 
-    for k in ['ndk_paths']:
+    # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
+    if thisconfig['java_paths'] is None:
+        thisconfig['java_paths'] = dict()
+        for d in sorted(glob.glob('/usr/lib/jvm/j*[6-9]*')
+                        + glob.glob('/usr/java/jdk1.[6-9]*')
+                        + glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
+                        + glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')):
+            if os.path.islink(d):
+                continue
+            j = os.path.basename(d)
+            # the last one found will be the canonical one, so order appropriately
+            for regex in (r'1\.([6-9])\.0\.jdk',  # OSX
+                          r'jdk1\.([6-9])\.0_[0-9]+.jdk',  # OSX and Oracle tarball
+                          r'jdk([6-9])-openjdk',  # Arch
+                          r'java-1\.([6-9])\.0-.*',  # RedHat
+                          r'java-([6-9])-oracle',  # Debian WebUpd8
+                          r'jdk-([6-9])-oracle-.*',  # Debian make-jpkg
+                          r'java-([6-9])-openjdk-[^c][^o][^m].*'):  # Debian
+                m = re.match(regex, j)
+                if m:
+                    osxhome = os.path.join(d, 'Contents', 'Home')
+                    if os.path.exists(osxhome):
+                        thisconfig['java_paths'][m.group(1)] = osxhome
+                    else:
+                        thisconfig['java_paths'][m.group(1)] = d
+
+    for java_version in ('7', '8', '9'):
+        java_home = thisconfig['java_paths'][java_version]
+        jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
+        if os.path.exists(jarsigner):
+            thisconfig['jarsigner'] = jarsigner
+            thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
+            break  # Java7 is preferred, so quit if found
+
+    for k in ['ndk_paths', 'java_paths']:
         d = thisconfig[k]
         for k2 in d.copy():
             v = d[k2]
@@ -185,6 +225,9 @@ def read_config(opts, config_file='config.py'):
     for n in ['ANDROID_HOME', 'ANDROID_SDK']:
         env[n] = config['sdk_path']
 
+    for k, v in config['java_paths'].items():
+        env['JAVA%s_HOME' % k] = v
+
     for k in ["keystorepass", "keypass"]:
         if k in config:
             write_password_file(k)
@@ -212,15 +255,6 @@ def read_config(opts, config_file='config.py'):
     return config
 
 
-def get_ndk_path(version):
-    if version is None:
-        version = 'r10e'  # falls back to latest
-    paths = config['ndk_paths']
-    if version not in paths:
-        return ''
-    return paths[version] or ''
-
-
 def find_sdk_tools_cmd(cmd):
     '''find a working path to a tool from the Android SDK'''
 
@@ -296,7 +330,7 @@ def write_password_file(pwtype, password=None):
     command line argments
     '''
     filename = '.fdroid.' + pwtype + '.txt'
-    fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
+    fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
     if password is None:
         os.write(fd, config[pwtype])
     else:
@@ -354,10 +388,10 @@ def read_app_args(args, allapps, allow_vercodes=False):
         vc = vercodes[appid]
         if not vc:
             continue
-        app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
-        if len(app['builds']) != len(vercodes[appid]):
+        app.builds = [b for b in app.builds if b.vercode in vc]
+        if len(app.builds) != len(vercodes[appid]):
             error = True
-            allvcs = [b['vercode'] for b in app['builds']]
+            allvcs = [b.vercode for b in app.builds]
             for v in vercodes[appid]:
                 if v not in allvcs:
                     logging.critical("No such vercode %s for app %s" % (v, appid))
@@ -369,17 +403,18 @@ def read_app_args(args, allapps, allow_vercodes=False):
 
 
 def get_extension(filename):
-    _, ext = os.path.splitext(filename)
+    base, ext = os.path.splitext(filename)
     if not ext:
-        return ''
-    return ext.lower()[1:]
+        return base, ''
+    return base, ext.lower()[1:]
 
 
 def has_extension(filename, ext):
-    return ext == get_extension(filename)
+    _, f_ext = get_extension(filename)
+    return ext == f_ext
 
 
-apk_regex = None
+apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
 
 
 def clean_description(description):
@@ -396,10 +431,7 @@ def clean_description(description):
 
 
 def apknameinfo(filename):
-    global apk_regex
     filename = os.path.basename(filename)
-    if apk_regex is None:
-        apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
     m = apk_regex.match(filename)
     try:
         result = (m.group(1), m.group(2))
@@ -409,23 +441,23 @@ def apknameinfo(filename):
 
 
 def getapkname(app, build):
-    return "%s_%s.apk" % (app['id'], build['vercode'])
+    return "%s_%s.apk" % (app.id, build.vercode)
 
 
 def getsrcname(app, build):
-    return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
+    return "%s_%s_src.tar.gz" % (app.id, build.vercode)
 
 
 def getappname(app):
-    if app['Name']:
-        return app['Name']
-    if app['Auto Name']:
-        return app['Auto Name']
-    return app['id']
+    if app.Name:
+        return app.Name
+    if app.AutoName:
+        return app.AutoName
+    return app.id
 
 
 def getcvname(app):
-    return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
+    return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
 
 
 def getvcs(vcstype, remote, local):
@@ -519,7 +551,7 @@ class vcs:
 
         try:
             self.gotorevisionx(rev)
-        except FDroidException, e:
+        except FDroidException as e:
             exc = e
 
         # If necessary, write the .fdroidvcs file.
@@ -648,6 +680,8 @@ class vcs_git(vcs):
             for line in lines:
                 if 'git@github.com' in line:
                     line = line.replace('git@github.com:', 'https://github.com/')
+                if 'git@gitlab.com' in line:
+                    line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
                 f.write(line)
 
         p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
@@ -911,7 +945,8 @@ def retrieve_string(app_dir, string, xmlfiles=None):
     def element_content(element):
         if element.text is None:
             return ""
-        return element.text.encode('utf-8')
+        s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
+        return s.strip()
 
     for path in xmlfiles:
         if not os.path.isfile(path):
@@ -955,6 +990,8 @@ def fetch_real_name(app_dir, flavours):
         logging.debug("fetch_real_name: Checking manifest at " + path)
         xml = parse_xml(path)
         app = xml.find('application')
+        if app is None:
+            continue
         if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
             continue
         label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
@@ -1003,20 +1040,31 @@ def remove_debuggable_flags(root_dir):
                         os.path.join(root, 'AndroidManifest.xml'))
 
 
+vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
+vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
+psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
+
+
+def app_matches_packagename(app, package):
+    if not package:
+        return False
+    appid = app.UpdateCheckName or app.id
+    if appid is None or appid == "Ignore":
+        return True
+    return appid == package
+
+
 # Extract some information from the AndroidManifest.xml at the given path.
 # Returns (version, vercode, package), any or all of which might be None.
 # All values returned are strings.
-def parse_androidmanifests(paths, ignoreversions=None):
+def parse_androidmanifests(paths, app):
+
+    ignoreversions = app.UpdateCheckIgnore
+    ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
 
     if not paths:
         return (None, None, None)
 
-    vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
-    vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
-    psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
-
-    ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
-
     max_version = None
     max_vercode = None
     max_package = None
@@ -1041,7 +1089,9 @@ def parse_androidmanifests(paths, ignoreversions=None):
                 if not package:
                     matches = psearch_g(line)
                     if matches:
-                        package = matches.group(2)
+                        s = matches.group(2)
+                        if app_matches_packagename(app, s):
+                            package = s
                 if not version:
                     matches = vnsearch_g(line)
                     if matches:
@@ -1051,17 +1101,22 @@ def parse_androidmanifests(paths, ignoreversions=None):
                     if matches:
                         vercode = matches.group(1)
         else:
-            xml = parse_xml(path)
-            if "package" in xml.attrib:
-                package = xml.attrib["package"].encode('utf-8')
-            if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
-                version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
-                base_dir = os.path.dirname(path)
-                version = retrieve_string_singleline(base_dir, version)
-            if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
-                a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
-                if string_is_integer(a):
-                    vercode = a
+            try:
+                xml = parse_xml(path)
+                if "package" in xml.attrib:
+                    s = xml.attrib["package"].encode('utf-8')
+                    if app_matches_packagename(app, s):
+                        package = s
+                if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
+                    version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+                    base_dir = os.path.dirname(path)
+                    version = retrieve_string_singleline(base_dir, version)
+                if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
+                    a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+                    if string_is_integer(a):
+                        vercode = a
+            except Exception:
+                logging.warning("Problem with xml at {0}".format(path))
 
         # Remember package name, may be defined separately from version+vercode
         if package is None:
@@ -1200,6 +1255,8 @@ def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
 
     return (name, number, libdir)
 
+gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
+
 
 # Prepare the source code for a particular build
 #  'vcs'         - the appropriate vcs object for the application
@@ -1218,17 +1275,17 @@ def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
 
     # Optionally, the actual app source can be in a subdirectory
-    if build['subdir']:
-        root_dir = os.path.join(build_dir, build['subdir'])
+    if build.subdir:
+        root_dir = os.path.join(build_dir, build.subdir)
     else:
         root_dir = build_dir
 
     # Get a working copy of the right revision
-    logging.info("Getting source for revision " + build['commit'])
-    vcs.gotorevision(build['commit'], refresh)
+    logging.info("Getting source for revision " + build.commit)
+    vcs.gotorevision(build.commit, refresh)
 
     # Initialise submodules if required
-    if build['submodules']:
+    if build.submodules:
         logging.info("Initialising submodules")
         vcs.initsubmodules()
 
@@ -1238,31 +1295,31 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         raise BuildException('Missing subdir ' + root_dir)
 
     # Run an init command if one is required
-    if build['init']:
-        cmd = replace_config_vars(build['init'], build)
+    if build.init:
+        cmd = replace_config_vars(build.init, build)
         logging.info("Running 'init' commands in %s" % root_dir)
 
         p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
         if p.returncode != 0:
             raise BuildException("Error running init command for %s:%s" %
-                                 (app['id'], build['version']), p.output)
+                                 (app.id, build.version), p.output)
 
     # Apply patches if any
-    if build['patch']:
+    if build.patch:
         logging.info("Applying patches")
-        for patch in build['patch']:
+        for patch in build.patch:
             patch = patch.strip()
             logging.info("Applying " + patch)
-            patch_path = os.path.join('metadata', app['id'], patch)
+            patch_path = os.path.join('metadata', app.id, patch)
             p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
             if p.returncode != 0:
                 raise BuildException("Failed to apply patch %s" % patch_path)
 
     # Get required source libraries
     srclibpaths = []
-    if build['srclibs']:
+    if build.srclibs:
         logging.info("Collecting source libraries")
-        for lib in build['srclibs']:
+        for lib in build.srclibs:
             srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
 
     for name, number, libpath in srclibpaths:
@@ -1275,8 +1332,12 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
 
     # Update the local.properties file
     localprops = [os.path.join(build_dir, 'local.properties')]
-    if build['subdir']:
-        localprops += [os.path.join(root_dir, 'local.properties')]
+    if build.subdir:
+        parts = build.subdir.split(os.sep)
+        cur = build_dir
+        for d in parts:
+            cur = os.path.join(cur, d)
+            localprops += [os.path.join(cur, 'local.properties')]
     for path in localprops:
         props = ""
         if os.path.isfile(path):
@@ -1288,62 +1349,30 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
             logging.info("Creating local.properties file at %s" % path)
         # Fix old-fashioned 'sdk-location' by copying
         # from sdk.dir, if necessary
-        if build['oldsdkloc']:
+        if build.oldsdkloc:
             sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
                               re.S | re.M).group(1)
             props += "sdk-location=%s\n" % sdkloc
         else:
             props += "sdk.dir=%s\n" % config['sdk_path']
             props += "sdk-location=%s\n" % config['sdk_path']
-        if build['ndk_path']:
+        ndk_path = build.ndk_path()
+        if ndk_path:
             # Add ndk location
-            props += "ndk.dir=%s\n" % build['ndk_path']
-            props += "ndk-location=%s\n" % build['ndk_path']
+            props += "ndk.dir=%s\n" % ndk_path
+            props += "ndk-location=%s\n" % ndk_path
         # Add java.encoding if necessary
-        if build['encoding']:
-            props += "java.encoding=%s\n" % build['encoding']
+        if build.encoding:
+            props += "java.encoding=%s\n" % build.encoding
         with open(path, 'w') as f:
             f.write(props)
 
     flavours = []
-    if build['type'] == 'gradle':
-        flavours = build['gradle']
-
-        version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
-        gradlepluginver = None
-
-        gradle_dirs = [root_dir]
-
-        # Parent dir build.gradle
-        parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
-        if parent_dir.startswith(build_dir):
-            gradle_dirs.append(parent_dir)
-
-        for dir_path in gradle_dirs:
-            if gradlepluginver:
-                break
-            if not os.path.isdir(dir_path):
-                continue
-            for filename in os.listdir(dir_path):
-                if not filename.endswith('.gradle'):
-                    continue
-                path = os.path.join(dir_path, filename)
-                if not os.path.isfile(path):
-                    continue
-                for line in file(path):
-                    match = version_regex.match(line)
-                    if match:
-                        gradlepluginver = match.group(1)
-                        break
+    if build.method() == 'gradle':
+        flavours = build.gradle
 
-        if gradlepluginver:
-            build['gradlepluginver'] = LooseVersion(gradlepluginver)
-        else:
-            logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
-            build['gradlepluginver'] = LooseVersion('0.11')
-
-        if build['target']:
-            n = build["target"].split('-')[1]
+        if build.target:
+            n = build.target.split('-')[1]
             regsub_file(r'compileSdkVersion[ =]+[0-9]+',
                         r'compileSdkVersion %s' % n,
                         os.path.join(root_dir, 'build.gradle'))
@@ -1352,38 +1381,38 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     remove_debuggable_flags(root_dir)
 
     # Insert version code and number into the manifest if necessary
-    if build['forceversion']:
+    if build.forceversion:
         logging.info("Changing the version name")
         for path in manifest_paths(root_dir, flavours):
             if not os.path.isfile(path):
                 continue
             if has_extension(path, 'xml'):
                 regsub_file(r'android:versionName="[^"]*"',
-                            r'android:versionName="%s"' % build['version'],
+                            r'android:versionName="%s"' % build.version,
                             path)
             elif has_extension(path, 'gradle'):
                 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
-                            r"""\1versionName '%s'""" % build['version'],
+                            r"""\1versionName '%s'""" % build.version,
                             path)
 
-    if build['forcevercode']:
+    if build.forcevercode:
         logging.info("Changing the version code")
         for path in manifest_paths(root_dir, flavours):
             if not os.path.isfile(path):
                 continue
             if has_extension(path, 'xml'):
                 regsub_file(r'android:versionCode="[^"]*"',
-                            r'android:versionCode="%s"' % build['vercode'],
+                            r'android:versionCode="%s"' % build.vercode,
                             path)
             elif has_extension(path, 'gradle'):
                 regsub_file(r'versionCode[ =]+[0-9]+',
-                            r'versionCode %s' % build['vercode'],
+                            r'versionCode %s' % build.vercode,
                             path)
 
     # Delete unwanted files
-    if build['rm']:
+    if build.rm:
         logging.info("Removing specified files")
-        for part in getpaths(build_dir, build, 'rm'):
+        for part in getpaths(build_dir, build.rm):
             dest = os.path.join(build_dir, part)
             logging.info("Removing {0}".format(part))
             if os.path.lexists(dest):
@@ -1397,12 +1426,12 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     remove_signing_keys(build_dir)
 
     # Add required external libraries
-    if build['extlibs']:
+    if build.extlibs:
         logging.info("Collecting prebuilt libraries")
         libsdir = os.path.join(root_dir, 'libs')
         if not os.path.exists(libsdir):
             os.mkdir(libsdir)
-        for lib in build['extlibs']:
+        for lib in build.extlibs:
             lib = lib.strip()
             logging.info("...installing extlib {0}".format(lib))
             libf = os.path.basename(lib)
@@ -1412,10 +1441,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
             shutil.copyfile(libsrc, os.path.join(libsdir, libf))
 
     # Run a pre-build command if one is required
-    if build['prebuild']:
+    if build.prebuild:
         logging.info("Running 'prebuild' commands in %s" % root_dir)
 
-        cmd = replace_config_vars(build['prebuild'], build)
+        cmd = replace_config_vars(build.prebuild, build)
 
         # Substitute source library paths into prebuild commands
         for name, number, libpath in srclibpaths:
@@ -1425,20 +1454,20 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
         if p.returncode != 0:
             raise BuildException("Error running prebuild command for %s:%s" %
-                                 (app['id'], build['version']), p.output)
+                                 (app.id, build.version), p.output)
 
     # Generate (or update) the ant build file, build.xml...
-    if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
+    if build.method() == 'ant' and build.update != ['no']:
         parms = ['android', 'update', 'lib-project']
         lparms = ['android', 'update', 'project']
 
-        if build['target']:
-            parms += ['-t', build['target']]
-            lparms += ['-t', build['target']]
-        if build['update'] == ['auto']:
-            update_dirs = ant_subprojects(root_dir) + ['.']
+        if build.target:
+            parms += ['-t', build.target]
+            lparms += ['-t', build.target]
+        if build.update:
+            update_dirs = build.update
         else:
-            update_dirs = build['update']
+            update_dirs = ant_subprojects(root_dir) + ['.']
 
         for d in update_dirs:
             subdir = os.path.join(root_dir, d)
@@ -1462,14 +1491,27 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     return (root_dir, srclibpaths)
 
 
-# Split and extend via globbing the paths from a field
-def getpaths(build_dir, build, field):
-    paths = []
-    for p in build[field]:
+# Extend via globbing the paths from a field and return them as a map from
+# original path to resulting paths
+def getpaths_map(build_dir, globpaths):
+    paths = dict()
+    for p in globpaths:
         p = p.strip()
         full_path = os.path.join(build_dir, p)
         full_path = os.path.normpath(full_path)
-        paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
+        paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
+        if not paths[p]:
+            raise FDroidException("glob path '%s' did not match any files/dirs" % p)
+    return paths
+
+
+# Extend via globbing the paths from a field and return them as a set
+def getpaths(build_dir, globpaths):
+    paths_map = getpaths_map(build_dir, globpaths)
+    paths = set()
+    for k, v in paths_map.iteritems():
+        for p in v:
+            paths.add(p)
     return paths
 
 
@@ -1569,7 +1611,11 @@ def SdkToolsPopen(commands, cwd=None, output=True):
     cmd = commands[0]
     if cmd not in config:
         config[cmd] = find_sdk_tools_cmd(commands[0])
-    return FDroidPopen([config[cmd]] + commands[1:],
+    abscmd = config[cmd]
+    if abscmd is None:
+        logging.critical("Could not find '%s' on your system" % cmd)
+        sys.exit(1)
+    return FDroidPopen([abscmd] + commands[1:],
                        cwd=cwd, output=output)
 
 
@@ -1594,11 +1640,11 @@ def FDroidPopen(commands, cwd=None, output=True):
     try:
         p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
                              stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    except OSError, e:
+    except OSError as e:
         raise BuildException("OSError while trying to execute " +
                              ' '.join(commands) + ': ' + str(e))
 
-    stdout_queue = Queue.Queue()
+    stdout_queue = Queue()
     stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
 
     # Check the queue for output (until there is no more to get)
@@ -1618,17 +1664,15 @@ def FDroidPopen(commands, cwd=None, output=True):
 
 
 gradle_comment = re.compile(r'[ ]*//')
+gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
+gradle_line_matches = [
+    re.compile(r'^[\t ]*signingConfig [^ ]*$'),
+    re.compile(r'.*android\.signingConfigs\.[^{]*$'),
+    re.compile(r'.*\.readLine\(.*'),
+]
 
 
 def remove_signing_keys(build_dir):
-    signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
-    line_matches = [
-        re.compile(r'^[\t ]*signingConfig [^ ]*$'),
-        re.compile(r'.*android\.signingConfigs\.[^{]*$'),
-        re.compile(r'.*variant\.outputFile = .*'),
-        re.compile(r'.*output\.outputFile = .*'),
-        re.compile(r'.*\.readLine\(.*'),
-    ]
     for root, dirs, files in os.walk(build_dir):
         if 'build.gradle' in files:
             path = os.path.join(root, 'build.gradle')
@@ -1657,12 +1701,12 @@ def remove_signing_keys(build_dir):
                         opened -= line.count('}')
                         continue
 
-                    if signing_configs.match(line):
+                    if gradle_signing_configs.match(line):
                         changed = True
                         opened += 1
                         continue
 
-                    if any(s.match(line) for s in line_matches):
+                    if any(s.match(line) for s in gradle_line_matches):
                         changed = True
                         continue
 
@@ -1718,9 +1762,9 @@ def replace_config_vars(cmd, build):
     cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
     cmd = cmd.replace('$$MVN3$$', config['mvn3'])
     if build is not None:
-        cmd = cmd.replace('$$COMMIT$$', build['commit'])
-        cmd = cmd.replace('$$VERSION$$', build['version'])
-        cmd = cmd.replace('$$VERCODE$$', build['vercode'])
+        cmd = cmd.replace('$$COMMIT$$', build.commit)
+        cmd = cmd.replace('$$VERSION$$', build.version)
+        cmd = cmd.replace('$$VERCODE$$', build.vercode)
     return cmd
 
 
@@ -1746,6 +1790,8 @@ def place_srclib(root_dir, number, libpath):
         if not placed:
             o.write('android.library.reference.%d=%s\n' % (number, relpath))
 
+apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
+
 
 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
     """Verify that two apks are the same
@@ -1760,11 +1806,10 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir):
     :returns: None if the verification is successful, otherwise a string
               describing what went wrong.
     """
-    sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
     with ZipFile(signed_apk) as signed_apk_as_zip:
         meta_inf_files = ['META-INF/MANIFEST.MF']
         for f in signed_apk_as_zip.namelist():
-            if sigfile.match(f):
+            if apk_sigfile.match(f):
                 meta_inf_files.append(f)
         if len(meta_inf_files) < 3:
             return "Signature files missing from {0}".format(signed_apk)
@@ -1773,12 +1818,14 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir):
         for meta_inf_file in meta_inf_files:
             unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
 
-    if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
+    if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
         logging.info("...NOT verified - {0}".format(signed_apk))
         return compare_apks(signed_apk, unsigned_apk, tmp_dir)
     logging.info("...successfully verified")
     return None
 
+apk_badchars = re.compile('''[/ :;'"]''')
+
 
 def compare_apks(apk1, apk2, tmp_dir):
     """Compare two apks
@@ -1788,9 +1835,8 @@ def compare_apks(apk1, apk2, tmp_dir):
     trying to do the comparison.
     """
 
-    badchars = re.compile('''[/ :;'"]''')
-    apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4]))  # trim .apk
-    apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4]))  # trim .apk
+    apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4]))  # trim .apk
+    apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4]))  # trim .apk
     for d in [apk1dir, apk2dir]:
         if os.path.exists(d):
             shutil.rmtree(d)
@@ -1874,7 +1920,7 @@ def genkeystore(localconfig):
 
     write_password_file("keystorepass", localconfig['keystorepass'])
     write_password_file("keypass", localconfig['keypass'])
-    p = FDroidPopen(['keytool', '-genkey',
+    p = FDroidPopen([config['keytool'], '-genkey',
                      '-keystore', localconfig['keystore'],
                      '-alias', localconfig['repo_keyalias'],
                      '-keyalg', 'RSA', '-keysize', '4096',
@@ -1888,7 +1934,7 @@ def genkeystore(localconfig):
         raise BuildException("Failed to generate key", p.output)
     os.chmod(localconfig['keystore'], 0o0600)
     # now show the lovely key that was just generated
-    p = FDroidPopen(['keytool', '-list', '-v',
+    p = FDroidPopen([config['keytool'], '-list', '-v',
                      '-keystore', localconfig['keystore'],
                      '-alias', localconfig['repo_keyalias'],
                      '-storepass:file', config['keystorepassfile']])
@@ -1939,7 +1985,7 @@ def get_per_app_repos():
     repos = []
     for root, dirs, files in os.walk(os.getcwd()):
         for d in dirs:
-            print 'checking', root, 'for', d
+            print('checking', root, 'for', d)
             if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
                 # standard parts of an fdroid repo, so never packageNames
                 continue