chiark / gitweb /
Rewrite scanner logic
[fdroidserver.git] / fdroidserver / common.py
index 985afb4abe2627158bb042ac4a6cf2f38b19cf86..d93bf73927d223e710a994bf5ef3043238de9600 100644 (file)
@@ -17,6 +17,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# common.py is imported by all modules, so do not import third-party
+# libraries here as they will become a requirement for all commands.
+
 import os
 import sys
 import re
@@ -28,16 +31,18 @@ import time
 import operator
 import Queue
 import threading
-import magic
 import logging
 import hashlib
 import socket
+import xml.etree.ElementTree as XMLElementTree
 
 from distutils.version import LooseVersion
 from zipfile import ZipFile
 
 import metadata
 
+XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
+
 config = None
 options = None
 env = None
@@ -48,13 +53,15 @@ default_config = {
     'sdk_path': "$ANDROID_HOME",
     'ndk_paths': {
         'r9b': None,
-        'r10d': "$ANDROID_NDK"
+        'r10e': "$ANDROID_NDK",
     },
-    'build_tools': "22.0.0",
+    'build_tools': "23.0.1",
     'ant': "ant",
     'mvn3': "mvn",
     'gradle': 'gradle',
+    'accepted_formats': ['txt', 'yaml'],
     'sync_from_local_copy_dir': False,
+    'per_app_repos': False,
     'make_current_version_link': True,
     'current_version_name_source': 'Name',
     'update_stats': False,
@@ -67,8 +74,8 @@ default_config = {
     'keystore': 'keystore.jks',
     'smartcardoptions': [],
     'char_limits': {
-        'Summary': 50,
-        'Description': 1500
+        'Summary': 80,
+        'Description': 4000,
     },
     'keyaliases': {},
     'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
@@ -84,6 +91,13 @@ default_config = {
 }
 
 
+def setup_global_opts(parser):
+    parser.add_argument("-v", "--verbose", action="store_true", default=False,
+                        help="Spew out even more information than normal")
+    parser.add_argument("-q", "--quiet", action="store_true", default=False,
+                        help="Restrict output to warnings and errors")
+
+
 def fill_config_defaults(thisconfig):
     for k, v in default_config.items():
         if k not in thisconfig:
@@ -117,6 +131,14 @@ def fill_config_defaults(thisconfig):
                 thisconfig[k][k2 + '_orig'] = v
 
 
+def regsub_file(pattern, repl, path):
+    with open(path, 'r') as f:
+        text = f.read()
+    text = re.sub(pattern, repl, text)
+    with open(path, 'w') as f:
+        f.write(text)
+
+
 def read_config(opts, config_file='config.py'):
     """Read the repository config
 
@@ -191,7 +213,7 @@ def read_config(opts, config_file='config.py'):
 
 def get_ndk_path(version):
     if version is None:
-        version = 'r10d'  # latest
+        version = 'r10e'  # falls back to latest
     paths = config['ndk_paths']
     if version not in paths:
         return ''
@@ -345,10 +367,16 @@ def read_app_args(args, allapps, allow_vercodes=False):
     return apps
 
 
-def has_extension(filename, extension):
-    name, ext = os.path.splitext(filename)
-    ext = ext.lower()[1:]
-    return ext == extension
+def get_extension(filename):
+    _, ext = os.path.splitext(filename)
+    if not ext:
+        return ''
+    return ext.lower()[1:]
+
+
+def has_extension(filename, ext):
+    return ext == get_extension(filename)
+
 
 apk_regex = None
 
@@ -454,7 +482,7 @@ class vcs:
     # lifetime of the vcs object.
     # None is acceptable for 'rev' if you know you are cloning a clean copy of
     # the repo - otherwise it must specify a valid revision.
-    def gotorevision(self, rev):
+    def gotorevision(self, rev, refresh=True):
 
         if self.clone_failed:
             raise VCSException("Downloading the repository already failed once, not trying again.")
@@ -485,6 +513,8 @@ class vcs:
             shutil.rmtree(self.local)
 
         exc = None
+        if not refresh:
+            self.refreshed = True
 
         try:
             self.gotorevisionx(rev)
@@ -514,7 +544,7 @@ class vcs:
             raise VCSException('gettags not supported for this vcs type')
         rtags = []
         for tag in self._gettags():
-            if re.match('[-A-Za-z0-9_. ]+$', tag):
+            if re.match('[-A-Za-z0-9_. /]+$', tag):
                 rtags.append(tag)
         return rtags
 
@@ -564,12 +594,14 @@ class vcs_git(vcs):
         else:
             self.checkrepo()
             # Discard any working tree changes
-            p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
+            p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+                             'git', 'reset', '--hard'], cwd=self.local, output=False)
             if p.returncode != 0:
                 raise VCSException("Git reset failed", p.output)
             # Remove untracked files now, in case they're tracked in the target
             # revision (it happens!)
-            p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
+            p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+                             'git', 'clean', '-dffx'], cwd=self.local, output=False)
             if p.returncode != 0:
                 raise VCSException("Git clean failed", p.output)
             if not self.refreshed:
@@ -617,13 +649,6 @@ class vcs_git(vcs):
                     line = line.replace('git@github.com:', 'https://github.com/')
                 f.write(line)
 
-        for cmd in [
-                ['git', 'reset', '--hard'],
-                ['git', 'clean', '-dffx'],
-                ]:
-            p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local, output=False)
-            if p.returncode != 0:
-                raise VCSException("Git submodule reset failed", p.output)
         p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
         if p.returncode != 0:
             raise VCSException("Git submodule sync failed", p.output)
@@ -856,37 +881,43 @@ class vcs_bzr(vcs):
                 p.output.splitlines()]
 
 
-def retrieve_string(app_dir, string, xmlfiles=None):
+def unescape_string(string):
+    if string[0] == '"' and string[-1] == '"':
+        return string[1:-1]
+
+    return string.replace("\\'", "'")
 
-    res_dirs = [
-        os.path.join(app_dir, 'res'),
-        os.path.join(app_dir, 'src', 'main'),
-        ]
+
+def retrieve_string(app_dir, string, xmlfiles=None):
 
     if xmlfiles is None:
         xmlfiles = []
-        for res_dir in res_dirs:
+        for res_dir in [
+            os.path.join(app_dir, 'res'),
+            os.path.join(app_dir, 'src', 'main', 'res'),
+        ]:
             for r, d, f in os.walk(res_dir):
                 if os.path.basename(r) == 'values':
                     xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
 
-    string_search = None
-    if string.startswith('@string/'):
-        string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
-    elif string.startswith('&') and string.endswith(';'):
-        string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
+    if not string.startswith('@string/'):
+        return unescape_string(string)
 
-    if string_search is not None:
-        for xmlfile in xmlfiles:
-            if not os.path.isfile(xmlfile):
-                continue
-            for line in file(xmlfile):
-                matches = string_search(line)
-                if matches:
-                    return retrieve_string(app_dir, matches.group(1), xmlfiles)
-        return None
+    name = string[len('@string/'):]
 
-    return string.replace("\\'", "'")
+    for path in xmlfiles:
+        if not os.path.isfile(path):
+            continue
+        xml = parse_xml(path)
+        element = xml.find('string[@name="' + name + '"]')
+        if element is not None and element.text is not None:
+            return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
+
+    return ''
+
+
+def retrieve_string_singleline(app_dir, string, xmlfiles=None):
+    return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
 
 
 # Return list of existing files that will be used to find the highest vercode
@@ -909,38 +940,20 @@ def manifest_paths(app_dir, flavours):
 
 # Retrieve the package name. Returns the name, or None if not found.
 def fetch_real_name(app_dir, flavours):
-    app_search = re.compile(r'.*<application.*').search
-    name_search = re.compile(r'.*android:label="([^"]+)".*').search
-    app_found = False
-    for f in manifest_paths(app_dir, flavours):
-        if not has_extension(f, 'xml') or not os.path.isfile(f):
+    for path in manifest_paths(app_dir, flavours):
+        if not has_extension(path, 'xml') or not os.path.isfile(path):
             continue
-        logging.debug("fetch_real_name: Checking manifest at " + f)
-        for line in file(f):
-            if not app_found:
-                if app_search(line):
-                    app_found = True
-            if app_found:
-                matches = name_search(line)
-                if matches:
-                    stringname = matches.group(1)
-                    logging.debug("fetch_real_name: using string " + stringname)
-                    result = retrieve_string(app_dir, stringname)
-                    if result:
-                        result = result.strip()
-                    return result
-    return None
-
-
-# Retrieve the version name
-def version_name(original, app_dir, flavours):
-    for f in manifest_paths(app_dir, flavours):
-        if not has_extension(f, 'xml'):
+        logging.debug("fetch_real_name: Checking manifest at " + path)
+        xml = parse_xml(path)
+        app = xml.find('application')
+        if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
             continue
-        string = retrieve_string(app_dir, original)
-        if string:
-            return string
-    return original
+        label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
+        result = retrieve_string_singleline(app_dir, label)
+        if result:
+            result = result.strip()
+        return result
+    return None
 
 
 def get_library_references(root_dir):
@@ -976,10 +989,9 @@ def remove_debuggable_flags(root_dir):
     logging.debug("Removing debuggable flags from %s" % root_dir)
     for root, dirs, files in os.walk(root_dir):
         if 'AndroidManifest.xml' in files:
-            path = os.path.join(root, 'AndroidManifest.xml')
-            p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
-            if p.returncode != 0:
-                raise BuildException("Failed to remove debuggable flags of %s" % path)
+            regsub_file(r'android:debuggable="[^"]*"',
+                        '',
+                        os.path.join(root, 'AndroidManifest.xml'))
 
 
 # Extract some information from the AndroidManifest.xml at the given path.
@@ -990,13 +1002,9 @@ def parse_androidmanifests(paths, ignoreversions=None):
     if not paths:
         return (None, None, None)
 
-    vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
-    vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
-    psearch = re.compile(r'.*package="([^"]+)".*').search
-
     vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
     vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
-    psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
+    psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
 
     ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
 
@@ -1013,31 +1021,40 @@ def parse_androidmanifests(paths, ignoreversions=None):
         gradle = has_extension(path, 'gradle')
         version = None
         vercode = None
-        # Remember package name, may be defined separately from version+vercode
-        package = max_package
+        package = None
 
-        for line in file(path):
-            if not package:
-                if gradle:
+        if gradle:
+            for line in file(path):
+                # Grab first occurence of each to avoid running into
+                # alternative flavours and builds.
+                if not package:
                     matches = psearch_g(line)
-                else:
-                    matches = psearch(line)
-                if matches:
-                    package = matches.group(1)
-            if not version:
-                if gradle:
+                    if matches:
+                        package = matches.group(2)
+                if not version:
                     matches = vnsearch_g(line)
-                else:
-                    matches = vnsearch(line)
-                if matches:
-                    version = matches.group(2 if gradle else 1)
-            if not vercode:
-                if gradle:
+                    if matches:
+                        version = matches.group(2)
+                if not vercode:
                     matches = vcsearch_g(line)
-                else:
-                    matches = vcsearch(line)
-                if matches:
-                    vercode = matches.group(1)
+                    if matches:
+                        vercode = matches.group(1)
+        else:
+            xml = parse_xml(path)
+            if "package" in xml.attrib:
+                package = xml.attrib["package"].encode('utf-8')
+            if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
+                version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+                base_dir = os.path.dirname(path)
+                version = retrieve_string_singleline(base_dir, version)
+            if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
+                a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+                if string_is_integer(a):
+                    vercode = a
+
+        # Remember package name, may be defined separately from version+vercode
+        if package is None:
+            package = max_package
 
         logging.debug("..got package={0}, version={1}, vercode={2}"
                       .format(package, version, vercode))
@@ -1108,8 +1125,8 @@ class BuildException(FDroidException):
 # Returns the path to it. Normally this is the path to be used when referencing
 # it, which may be a subdirectory of the actual project. If you want the base
 # directory of the project, pass 'basepath=True'.
-def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
-              basepath=False, raw=False, prepare=True, preponly=False):
+def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
+              raw=False, prepare=True, preponly=False, refresh=True):
 
     number = None
     subdir = None
@@ -1134,7 +1151,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
         vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
         vcs.srclib = (name, number, sdir)
         if ref:
-            vcs.gotorevision(ref)
+            vcs.gotorevision(ref, refresh)
 
         if raw:
             return vcs
@@ -1152,20 +1169,6 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
     if libdir is None:
         libdir = sdir
 
-    if srclib["Srclibs"]:
-        n = 1
-        for lib in srclib["Srclibs"].replace(';', ',').split(','):
-            s_tuple = None
-            for t in srclibpaths:
-                if t[0] == lib:
-                    s_tuple = t
-                    break
-            if s_tuple is None:
-                raise VCSException('Missing recursive srclib %s for %s' % (
-                    lib, name))
-            place_srclib(libdir, n, s_tuple[2])
-            n += 1
-
     remove_signing_keys(sdir)
     remove_debuggable_flags(sdir)
 
@@ -1199,7 +1202,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
 #   'root' is the root directory, which may be the same as 'build_dir' or may
 #          be a subdirectory of it.
 #   'srclibpaths' is information on the srclibs being used
-def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
+def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
 
     # Optionally, the actual app source can be in a subdirectory
     if build['subdir']:
@@ -1209,9 +1212,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
 
     # Get a working copy of the right revision
     logging.info("Getting source for revision " + build['commit'])
-    vcs.gotorevision(build['commit'])
+    vcs.gotorevision(build['commit'], refresh)
 
-    # Initialise submodules if requred
+    # Initialise submodules if required
     if build['submodules']:
         logging.info("Initialising submodules")
         vcs.initsubmodules()
@@ -1247,8 +1250,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     if build['srclibs']:
         logging.info("Collecting source libraries")
         for lib in build['srclibs']:
-            srclibpaths.append(getsrclib(lib, srclib_dir, build, srclibpaths,
-                                         preponly=onserver))
+            srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
 
     for name, number, libpath in srclibpaths:
         place_srclib(root_dir, int(number) if number else None, libpath)
@@ -1266,9 +1268,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         props = ""
         if os.path.isfile(path):
             logging.info("Updating local.properties file at %s" % path)
-            f = open(path, 'r')
-            props += f.read()
-            f.close()
+            with open(path, 'r') as f:
+                props += f.read()
             props += '\n'
         else:
             logging.info("Creating local.properties file at %s" % path)
@@ -1288,9 +1289,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         # Add java.encoding if necessary
         if build['encoding']:
             props += "java.encoding=%s\n" % build['encoding']
-        f = open(path, 'w')
-        f.write(props)
-        f.close()
+        with open(path, 'w') as f:
+            f.write(props)
 
     flavours = []
     if build['type'] == 'gradle':
@@ -1331,9 +1331,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
 
         if build['target']:
             n = build["target"].split('-')[1]
-            FDroidPopen(['sed', '-i',
-                         's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
-                         'build.gradle'], cwd=root_dir, output=False)
+            regsub_file(r'compileSdkVersion[ =]+[0-9]+',
+                        r'compileSdkVersion %s' % n,
+                        os.path.join(root_dir, 'build.gradle'))
 
     # Remove forced debuggable flags
     remove_debuggable_flags(root_dir)
@@ -1345,34 +1345,27 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
             if not os.path.isfile(path):
                 continue
             if has_extension(path, 'xml'):
-                p = FDroidPopen(['sed', '-i',
-                                 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
-                                 path], output=False)
-                if p.returncode != 0:
-                    raise BuildException("Failed to amend manifest")
+                regsub_file(r'android:versionName="[^"]*"',
+                            r'android:versionName="%s"' % build['version'],
+                            path)
             elif has_extension(path, 'gradle'):
-                p = FDroidPopen(['sed', '-i',
-                                 's/versionName *=* *.*/versionName = "' + build['version'] + '"/g',
-                                 path], output=False)
-                if p.returncode != 0:
-                    raise BuildException("Failed to amend build.gradle")
+                regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
+                            r"""\1versionName '%s'""" % build['version'],
+                            path)
+
     if build['forcevercode']:
         logging.info("Changing the version code")
         for path in manifest_paths(root_dir, flavours):
             if not os.path.isfile(path):
                 continue
             if has_extension(path, 'xml'):
-                p = FDroidPopen(['sed', '-i',
-                                 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
-                                 path], output=False)
-                if p.returncode != 0:
-                    raise BuildException("Failed to amend manifest")
+                regsub_file(r'android:versionCode="[^"]*"',
+                            r'android:versionCode="%s"' % build['vercode'],
+                            path)
             elif has_extension(path, 'gradle'):
-                p = FDroidPopen(['sed', '-i',
-                                 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
-                                 path], output=False)
-                if p.returncode != 0:
-                    raise BuildException("Failed to amend build.gradle")
+                regsub_file(r'versionCode[ =]+[0-9]+',
+                            r'versionCode %s' % build['vercode'],
+                            path)
 
     # Delete unwanted files
     if build['rm']:
@@ -1467,160 +1460,8 @@ def getpaths(build_dir, build, field):
     return paths
 
 
-# Scan the source code in the given directory (and all subdirectories)
-# and return the number of fatal problems encountered
-def scan_source(build_dir, root_dir, thisbuild):
-
-    count = 0
-
-    # Common known non-free blobs (always lower case):
-    usual_suspects = [
-        re.compile(r'flurryagent', re.IGNORECASE),
-        re.compile(r'paypal.*mpl', re.IGNORECASE),
-        re.compile(r'google.*analytics', re.IGNORECASE),
-        re.compile(r'admob.*sdk.*android', re.IGNORECASE),
-        re.compile(r'google.*ad.*view', re.IGNORECASE),
-        re.compile(r'google.*admob', re.IGNORECASE),
-        re.compile(r'google.*play.*services', re.IGNORECASE),
-        re.compile(r'crittercism', re.IGNORECASE),
-        re.compile(r'heyzap', re.IGNORECASE),
-        re.compile(r'jpct.*ae', re.IGNORECASE),
-        re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
-        re.compile(r'bugsense', re.IGNORECASE),
-        re.compile(r'crashlytics', re.IGNORECASE),
-        re.compile(r'ouya.*sdk', re.IGNORECASE),
-        re.compile(r'libspen23', re.IGNORECASE),
-    ]
-
-    scanignore = getpaths(build_dir, thisbuild, 'scanignore')
-    scandelete = getpaths(build_dir, thisbuild, 'scandelete')
-
-    scanignore_worked = set()
-    scandelete_worked = set()
-
-    try:
-        ms = magic.open(magic.MIME_TYPE)
-        ms.load()
-    except AttributeError:
-        ms = None
-
-    def toignore(fd):
-        for p in scanignore:
-            if fd.startswith(p):
-                scanignore_worked.add(p)
-                return True
-        return False
-
-    def todelete(fd):
-        for p in scandelete:
-            if fd.startswith(p):
-                scandelete_worked.add(p)
-                return True
-        return False
-
-    def ignoreproblem(what, fd, fp):
-        logging.info('Ignoring %s at %s' % (what, fd))
-        return 0
-
-    def removeproblem(what, fd, fp):
-        logging.info('Removing %s at %s' % (what, fd))
-        os.remove(fp)
-        return 0
-
-    def warnproblem(what, fd):
-        logging.warn('Found %s at %s' % (what, fd))
-
-    def handleproblem(what, fd, fp):
-        if toignore(fd):
-            return ignoreproblem(what, fd, fp)
-        if todelete(fd):
-            return removeproblem(what, fd, fp)
-        logging.error('Found %s at %s' % (what, fd))
-        return 1
-
-    # Iterate through all files in the source code
-    for r, d, f in os.walk(build_dir, topdown=True):
-
-        # It's topdown, so checking the basename is enough
-        for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
-            if ignoredir in d:
-                d.remove(ignoredir)
-
-        for curfile in f:
-
-            # Path (relative) to the file
-            fp = os.path.join(r, curfile)
-            fd = fp[len(build_dir) + 1:]
-
-            try:
-                mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
-            except UnicodeError:
-                warnproblem('malformed magic number', fd)
-
-            if mime == 'application/x-sharedlib':
-                count += handleproblem('shared library', fd, fp)
-
-            elif mime == 'application/x-archive':
-                count += handleproblem('static library', fd, fp)
-
-            elif mime == 'application/x-executable':
-                count += handleproblem('binary executable', fd, fp)
-
-            elif mime == 'application/x-java-applet':
-                count += handleproblem('Java compiled class', fd, fp)
-
-            elif mime in (
-                    'application/jar',
-                    'application/zip',
-                    'application/java-archive',
-                    'application/octet-stream',
-                    'binary', ):
-
-                if has_extension(fp, 'apk'):
-                    removeproblem('APK file', fd, fp)
-
-                elif has_extension(fp, 'jar'):
-
-                    if any(suspect.match(curfile) for suspect in usual_suspects):
-                        count += handleproblem('usual supect', fd, fp)
-                    else:
-                        warnproblem('JAR file', fd)
-
-                elif has_extension(fp, 'zip'):
-                    warnproblem('ZIP file', fd)
-
-                else:
-                    warnproblem('unknown compressed or binary file', fd)
-
-            elif has_extension(fp, 'java') and os.path.isfile(fp):
-                if not os.path.isfile(fp):
-                    continue
-                for line in file(fp):
-                    if 'DexClassLoader' in line:
-                        count += handleproblem('DexClassLoader', fd, fp)
-                        break
-    if ms is not None:
-        ms.close()
-
-    for p in scanignore:
-        if p not in scanignore_worked:
-            logging.error('Unused scanignore path: %s' % p)
-            count += 1
-
-    for p in scandelete:
-        if p not in scandelete_worked:
-            logging.error('Unused scandelete path: %s' % p)
-            count += 1
-
-    # Presence of a jni directory without buildjni=yes might
-    # indicate a problem (if it's not a problem, explicitly use
-    # buildjni=no to bypass this check)
-    if (os.path.exists(os.path.join(root_dir, 'jni')) and
-            not thisbuild['buildjni']):
-        logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
-        count += 1
-
-    return count
+def natural_key(s):
+    return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
 
 
 class KnownApks:
@@ -1638,20 +1479,23 @@ class KnownApks:
         self.changed = False
 
     def writeifchanged(self):
-        if self.changed:
-            if not os.path.exists('stats'):
-                os.mkdir('stats')
-            f = open(self.path, 'w')
-            lst = []
-            for apk, app in self.apks.iteritems():
-                appid, added = app
-                line = apk + ' ' + appid
-                if added:
-                    line += ' ' + time.strftime('%Y-%m-%d', added)
-                lst.append(line)
-            for line in sorted(lst):
+        if not self.changed:
+            return
+
+        if not os.path.exists('stats'):
+            os.mkdir('stats')
+
+        lst = []
+        for apk, app in self.apks.iteritems():
+            appid, added = app
+            line = apk + ' ' + appid
+            if added:
+                line += ' ' + time.strftime('%Y-%m-%d', added)
+            lst.append(line)
+
+        with open(self.path, 'w') as f:
+            for line in sorted(lst, key=natural_key):
                 f.write(line + '\n')
-            f.close()
 
     # Record an apk (if it's new, otherwise does nothing)
     # Returns the date it was added.
@@ -2050,9 +1894,9 @@ def genkeystore(localconfig):
                      '-keypass:file', config['keypassfile'],
                      '-dname', localconfig['keydname']])
     # TODO keypass should be sent via stdin
-    os.chmod(localconfig['keystore'], 0o0600)
     if p.returncode != 0:
         raise BuildException("Failed to generate key", p.output)
+    os.chmod(localconfig['keystore'], 0o0600)
     # now show the lovely key that was just generated
     p = FDroidPopen(['keytool', '-list', '-v',
                      '-keystore', localconfig['keystore'],
@@ -2079,3 +1923,38 @@ def write_to_config(thisconfig, key, value=None):
         data += '\n'
     with open('config.py', 'w') as f:
         f.writelines(data)
+
+
+def parse_xml(path):
+    return XMLElementTree.parse(path).getroot()
+
+
+def string_is_integer(string):
+    try:
+        int(string)
+        return True
+    except ValueError:
+        return False
+
+
+def get_per_app_repos():
+    '''per-app repos are dirs named with the packageName of a single app'''
+
+    # Android packageNames are Java packages, they may contain uppercase or
+    # lowercase letters ('A' through 'Z'), numbers, and underscores
+    # ('_'). However, individual package name parts may only start with
+    # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
+    p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
+
+    repos = []
+    for root, dirs, files in os.walk(os.getcwd()):
+        for d in dirs:
+            print 'checking', root, 'for', d
+            if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
+                # standard parts of an fdroid repo, so never packageNames
+                continue
+            elif p.match(d) \
+                    and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
+                repos.append(d)
+        break
+    return repos