chiark / gitweb /
build: write out full rsync options
[fdroidserver.git] / fdroidserver / build.py
index 755f5764568195102a0d27e06fd4cad7830e4ee9..bbb5a2b4d0b2e7492210c1298f5ada9177e85b83 100644 (file)
@@ -23,20 +23,23 @@ import shutil
 import glob
 import subprocess
 import re
+import resource
 import tarfile
 import traceback
 import time
-import json
 import requests
 import tempfile
 from configparser import ConfigParser
 from argparse import ArgumentParser
 import logging
+from gettext import ngettext
 
+from . import _
 from . import common
 from . import net
 from . import metadata
 from . import scanner
+from . import vmtools
 from .common import FDroidPopen, SdkToolsPopen
 from .exception import FDroidException, BuildException, VCSException
 
@@ -46,205 +49,6 @@ except ImportError:
     pass
 
 
-def get_builder_vm_id():
-    vd = os.path.join('builder', '.vagrant')
-    if os.path.isdir(vd):
-        # Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
-        with open(os.path.join(vd, 'machines', 'default',
-                               'virtualbox', 'id')) as vf:
-            id = vf.read()
-        return id
-    else:
-        # Vagrant 1.0 - it's a json file...
-        with open(os.path.join('builder', '.vagrant')) as vf:
-            v = json.load(vf)
-        return v['active']['default']
-
-
-def got_valid_builder_vm():
-    """Returns True if we have a valid-looking builder vm
-    """
-    if not os.path.exists(os.path.join('builder', 'Vagrantfile')):
-        return False
-    vd = os.path.join('builder', '.vagrant')
-    if not os.path.exists(vd):
-        return False
-    if not os.path.isdir(vd):
-        # Vagrant 1.0 - if the directory is there, it's valid...
-        return True
-    # Vagrant 1.2 - the directory can exist, but the id can be missing...
-    if not os.path.exists(os.path.join(vd, 'machines', 'default',
-                                       'virtualbox', 'id')):
-        return False
-    return True
-
-
-def vagrant(params, cwd=None, printout=False):
-    """Run a vagrant command.
-
-    :param: list of parameters to pass to vagrant
-    :cwd: directory to run in, or None for current directory
-    :returns: (ret, out) where ret is the return code, and out
-               is the stdout (and stderr) from vagrant
-    """
-    p = FDroidPopen(['vagrant'] + params, cwd=cwd)
-    return (p.returncode, p.output)
-
-
-def get_vagrant_sshinfo():
-    """Get ssh connection info for a vagrant VM
-
-    :returns: A dictionary containing 'hostname', 'port', 'user'
-        and 'idfile'
-    """
-    if subprocess.call('vagrant ssh-config >sshconfig',
-                       cwd='builder', shell=True) != 0:
-        raise BuildException("Error getting ssh config")
-    vagranthost = 'default'  # Host in ssh config file
-    sshconfig = paramiko.SSHConfig()
-    sshf = open(os.path.join('builder', 'sshconfig'), 'r')
-    sshconfig.parse(sshf)
-    sshf.close()
-    sshconfig = sshconfig.lookup(vagranthost)
-    idfile = sshconfig['identityfile']
-    if isinstance(idfile, list):
-        idfile = idfile[0]
-    elif idfile.startswith('"') and idfile.endswith('"'):
-        idfile = idfile[1:-1]
-    return {'hostname': sshconfig['hostname'],
-            'port': int(sshconfig['port']),
-            'user': sshconfig['user'],
-            'idfile': idfile}
-
-
-def get_clean_vm(reset=False):
-    """Get a clean VM ready to do a buildserver build.
-
-    This might involve creating and starting a new virtual machine from
-    scratch, or it might be as simple (unless overridden by the reset
-    parameter) as re-using a snapshot created previously.
-
-    A BuildException will be raised if anything goes wrong.
-
-    :reset: True to force creating from scratch.
-    :returns: A dictionary containing 'hostname', 'port', 'user'
-        and 'idfile'
-    """
-    # Reset existing builder machine to a clean state if possible.
-    vm_ok = False
-    if not reset:
-        logging.info("Checking for valid existing build server")
-
-        if got_valid_builder_vm():
-            logging.info("...VM is present")
-            p = FDroidPopen(['VBoxManage', 'snapshot',
-                             get_builder_vm_id(), 'list',
-                             '--details'], cwd='builder')
-            if 'fdroidclean' in p.output:
-                logging.info("...snapshot exists - resetting build server to "
-                             "clean state")
-                retcode, output = vagrant(['status'], cwd='builder')
-
-                if 'running' in output:
-                    logging.info("...suspending")
-                    vagrant(['suspend'], cwd='builder')
-                    logging.info("...waiting a sec...")
-                    time.sleep(10)
-                p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
-                                 'restore', 'fdroidclean'],
-                                cwd='builder')
-
-                if p.returncode == 0:
-                    logging.info("...reset to snapshot - server is valid")
-                    retcode, output = vagrant(['up'], cwd='builder')
-                    if retcode != 0:
-                        raise BuildException("Failed to start build server")
-                    logging.info("...waiting a sec...")
-                    time.sleep(10)
-                    sshinfo = get_vagrant_sshinfo()
-                    vm_ok = True
-                else:
-                    logging.info("...failed to reset to snapshot")
-            else:
-                logging.info("...snapshot doesn't exist - "
-                             "VBoxManage snapshot list:\n" + p.output)
-
-    # If we can't use the existing machine for any reason, make a
-    # new one from scratch.
-    if not vm_ok:
-        if os.path.exists('builder'):
-            logging.info("Removing broken/incomplete/unwanted build server")
-            vagrant(['destroy', '-f'], cwd='builder')
-            shutil.rmtree('builder')
-        os.mkdir('builder')
-
-        p = subprocess.Popen(['vagrant', '--version'],
-                             universal_newlines=True,
-                             stdout=subprocess.PIPE)
-        vver = p.communicate()[0].strip().split(' ')[1]
-        if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4:
-            raise BuildException("Unsupported vagrant version {0}".format(vver))
-
-        with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf:
-            vf.write('Vagrant.configure("2") do |config|\n')
-            vf.write('config.vm.box = "buildserver"\n')
-            vf.write('config.vm.synced_folder ".", "/vagrant", disabled: true\n')
-            vf.write('end\n')
-
-        logging.info("Starting new build server")
-        retcode, _ = vagrant(['up'], cwd='builder')
-        if retcode != 0:
-            raise BuildException("Failed to start build server")
-
-        # Open SSH connection to make sure it's working and ready...
-        logging.info("Connecting to virtual machine...")
-        sshinfo = get_vagrant_sshinfo()
-        sshs = paramiko.SSHClient()
-        sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
-                     port=sshinfo['port'], timeout=300,
-                     look_for_keys=False,
-                     key_filename=sshinfo['idfile'])
-        sshs.close()
-
-        logging.info("Saving clean state of new build server")
-        retcode, _ = vagrant(['suspend'], cwd='builder')
-        if retcode != 0:
-            raise BuildException("Failed to suspend build server")
-        logging.info("...waiting a sec...")
-        time.sleep(10)
-        p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
-                         'take', 'fdroidclean'],
-                        cwd='builder')
-        if p.returncode != 0:
-            raise BuildException("Failed to take snapshot")
-        logging.info("...waiting a sec...")
-        time.sleep(10)
-        logging.info("Restarting new build server")
-        retcode, _ = vagrant(['up'], cwd='builder')
-        if retcode != 0:
-            raise BuildException("Failed to start build server")
-        logging.info("...waiting a sec...")
-        time.sleep(10)
-        # Make sure it worked...
-        p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
-                         'list', '--details'],
-                        cwd='builder')
-        if 'fdroidclean' not in p.output:
-            raise BuildException("Failed to take snapshot.")
-
-    return sshinfo
-
-
-def release_vm():
-    """Release the VM previously started with get_clean_vm().
-
-    This should always be called.
-    """
-    logging.info("Suspending build server")
-    subprocess.call(['vagrant', 'suspend'], cwd='builder')
-
-
 # Note that 'force' here also implies test mode.
 def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
     """Do a build on the builder vm.
@@ -268,7 +72,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
     else:
         logging.getLogger("paramiko").setLevel(logging.WARN)
 
-    sshinfo = get_clean_vm()
+    sshinfo = vmtools.get_clean_builder('builder')
 
     try:
         if not buildserverid:
@@ -295,22 +99,21 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
 
         # Helper to copy the contents of a directory to the server...
         def send_dir(path):
-            root = os.path.dirname(path)
-            main = os.path.basename(path)
-            ftp.mkdir(main)
-            for r, d, f in os.walk(path):
-                rr = os.path.relpath(r, root)
-                ftp.chdir(rr)
-                for dd in d:
-                    ftp.mkdir(dd)
-                for ff in f:
-                    lfile = os.path.join(root, rr, ff)
-                    if not os.path.islink(lfile):
-                        ftp.put(lfile, ff)
-                        ftp.chmod(ff, os.stat(lfile).st_mode)
-                for i in range(len(rr.split('/'))):
-                    ftp.chdir('..')
-            ftp.chdir('..')
+            logging.debug("rsyncing " + path + " to " + ftp.getcwd())
+            try:
+                subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' +
+                                         'ssh -o StrictHostKeyChecking=no' +
+                                         ' -o UserKnownHostsFile=/dev/null' +
+                                         ' -o LogLevel=FATAL' +
+                                         ' -o IdentitiesOnly=yes' +
+                                         ' -o PasswordAuthentication=no' +
+                                         ' -p ' + str(sshinfo['port']) +
+                                         ' -i ' + sshinfo['idfile'],
+                                         path,
+                                         sshinfo['user'] + "@" + sshinfo['hostname'] + ":" + ftp.getcwd()],
+                                        stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                raise FDroidException(str(e), e.output.decode())
 
         logging.info("Preparing server for build...")
         serverpath = os.path.abspath(os.path.dirname(__file__))
@@ -335,8 +138,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         ftp.mkdir('metadata')
         ftp.mkdir('srclibs')
         ftp.chdir('metadata')
-        ftp.put(os.path.join('metadata', app.id + '.txt'),
-                app.id + '.txt')
+        ftp.put(app.metadatapath, os.path.basename(app.metadatapath))
+
         # And patches if there are any...
         if os.path.exists(os.path.join('metadata', app.id)):
             send_dir(os.path.join('metadata', app.id))
@@ -361,7 +164,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
                         ftp.mkdir(d)
                     ftp.chdir(d)
                 ftp.put(libsrc, lp[-1])
-                for _ in lp[:-1]:
+                for _ignored in lp[:-1]:
                     ftp.chdir('..')
         # Copy any srclibs that are required...
         srclibpaths = []
@@ -409,23 +212,33 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         cmdline += " %s:%s" % (app.id, build.versionCode)
         chan.exec_command('bash --login -c "' + cmdline + '"')
 
-        output = bytes()
-        output += get_android_tools_version_log(build.ndk_path()).encode()
-        while not chan.exit_status_ready():
-            while chan.recv_ready():
-                output += chan.recv(1024)
-            time.sleep(0.1)
+        # Fetch build process output ...
+        try:
+            cmd_stdout = chan.makefile('rb', 1024)
+            output = bytes()
+            output += get_android_tools_version_log(build.ndk_path()).encode()
+            while not chan.exit_status_ready():
+                line = cmd_stdout.readline()
+                if line:
+                    if options.verbose:
+                        logging.debug("buildserver > " + str(line, 'utf-8').rstrip())
+                    output += line
+                else:
+                    time.sleep(0.05)
+            for line in cmd_stdout.readlines():
+                if options.verbose:
+                    logging.debug("buildserver > " + str(line, 'utf-8').rstrip())
+                output += line
+        finally:
+            cmd_stdout.close()
+
+        # Check build process exit status ...
         logging.info("...getting exit status")
         returncode = chan.recv_exit_status()
-        while True:
-            get = chan.recv(1024)
-            if len(get) == 0:
-                break
-            output += get
         if returncode != 0:
             raise BuildException(
                 "Build.py failed on server for {0}:{1}".format(
-                    app.id, build.versionName), str(output, 'utf-8'))
+                    app.id, build.versionName), None if options.verbose else str(output, 'utf-8'))
 
         # Retreive logs...
         toolsversion_log = common.get_toolsversion_logname(app, build)
@@ -450,14 +263,14 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
                 ftp.get(tarball, os.path.join(output_dir, tarball))
         except Exception:
             raise BuildException(
-                "Build failed for %s:%s - missing output files".format(
-                    app.id, build.versionName), output)
+                "Build failed for {0}:{1} - missing output files".format(
+                    app.id, build.versionName), None if options.verbose else str(output, 'utf-8'))
         ftp.close()
 
     finally:
-
         # Suspend the build server.
-        release_vm()
+        vm = vmtools.get_build_vm('builder')
+        vm.suspend()
 
 
 def force_gradle_build_tools(build_dir, build_tools):
@@ -474,14 +287,13 @@ def force_gradle_build_tools(build_dir, build_tools):
                                path)
 
 
-def capitalize_intact(string):
-    """Like str.capitalize(), but leave the rest of the string intact without
-    switching it to lowercase."""
+def transform_first_char(string, method):
+    """Uses method() on the first character of string."""
     if len(string) == 0:
         return string
     if len(string) == 1:
-        return string.upper()
-    return string[0].upper() + string[1:]
+        return method(string)
+    return method(string[0]) + string[1:]
 
 
 def has_native_code(apkobj):
@@ -592,10 +404,23 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
 
     # create ..._toolsversion.log when running in builder vm
     if onserver:
+        # before doing anything, run the sudo commands to setup the VM
+        if build.sudo:
+            logging.info("Running 'sudo' commands in %s" % os.getcwd())
+
+            p = FDroidPopen(['sudo', 'bash', '-x', '-c', build.sudo])
+            if p.returncode != 0:
+                raise BuildException("Error running sudo command for %s:%s" %
+                                     (app.id, build.versionName), p.output)
+
         log_path = os.path.join(log_dir,
                                 common.get_toolsversion_logname(app, build))
         with open(log_path, 'w') as f:
             f.write(get_android_tools_version_log(build.ndk_path()))
+    else:
+        if build.sudo:
+            logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s'
+                            % (app.id, build.versionName, build.sudo))
 
     # Prepare the source code...
     root_dir, srclibpaths = common.prepare_source(vcs, app, build,
@@ -630,7 +455,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         if flavours == ['yes']:
             flavours = []
 
-        flavours_cmd = ''.join([capitalize_intact(flav) for flav in flavours])
+        flavours_cmd = ''.join([transform_first_char(flav, str.upper) for flav in flavours])
 
         gradletasks += ['assemble' + flavours_cmd + 'Release']
 
@@ -650,6 +475,9 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
     elif bmethod == 'kivy':
         pass
 
+    elif bmethod == 'buildozer':
+        pass
+
     elif bmethod == 'ant':
         logging.info("Cleaning Ant project...")
         p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
@@ -670,7 +498,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
                 if f in files:
                     os.remove(os.path.join(root, f))
 
-        if 'build.gradle' in files:
+        if any(f in files for f in ['build.gradle', 'settings.gradle']):
             # Even when running clean, gradle stores task/artifact caches in
             # .gradle/ as binary files. To avoid overcomplicating the scanner,
             # manually delete them, just like `gradle clean` should have removed
@@ -693,12 +521,15 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
     else:
         # Scan before building...
         logging.info("Scanning source for common problems...")
-        count = scanner.scan_source(build_dir, root_dir, build)
+        count = scanner.scan_source(build_dir, build)
         if count > 0:
             if force:
-                logging.warn('Scanner found %d problems' % count)
+                logging.warning(ngettext('Scanner found {} problem',
+                                         'Scanner found {} problems', count).format(count))
             else:
-                raise BuildException("Can't build due to %d errors while scanning" % count)
+                raise BuildException(ngettext(
+                    "Can't build due to {} error while scanning",
+                    "Can't build due to {} errors while scanning", count).format(count))
 
     if not options.notarball:
         # Build the source tarball right before we build the release...
@@ -844,6 +675,73 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         cmd.append('release')
         p = FDroidPopen(cmd, cwd=distdir)
 
+    elif bmethod == 'buildozer':
+        logging.info("Building Kivy project using buildozer...")
+
+        # parse buildozer.spez
+        spec = os.path.join(root_dir, 'buildozer.spec')
+        if not os.path.exists(spec):
+            raise BuildException("Expected to find buildozer-compatible spec at {0}"
+                                 .format(spec))
+        defaults = {'orientation': 'landscape', 'icon': '',
+                    'permissions': '', 'android.api': "19"}
+        bconfig = ConfigParser(defaults, allow_no_value=True)
+        bconfig.read(spec)
+
+        # update spec with sdk and ndk locations to prevent buildozer from
+        # downloading.
+        loc_ndk = common.env['ANDROID_NDK']
+        loc_sdk = common.env['ANDROID_SDK']
+        if loc_ndk == '$ANDROID_NDK':
+            loc_ndk = loc_sdk + '/ndk-bundle'
+
+        bc_ndk = None
+        bc_sdk = None
+        try:
+            bc_ndk = bconfig.get('app', 'android.sdk_path')
+        except Exception:
+            pass
+        try:
+            bc_sdk = bconfig.get('app', 'android.ndk_path')
+        except Exception:
+            pass
+
+        if bc_sdk is None:
+            bconfig.set('app', 'android.sdk_path', loc_sdk)
+        if bc_ndk is None:
+            bconfig.set('app', 'android.ndk_path', loc_ndk)
+
+        fspec = open(spec, 'w')
+        bconfig.write(fspec)
+        fspec.close()
+
+        logging.info("sdk_path = %s" % loc_sdk)
+        logging.info("ndk_path = %s" % loc_ndk)
+
+        p = None
+        # execute buildozer
+        cmd = ['buildozer', 'android', 'release']
+        try:
+            p = FDroidPopen(cmd, cwd=root_dir)
+        except Exception:
+            pass
+
+        # buidozer not installed ? clone repo and run
+        if (p is None or p.returncode != 0):
+            cmd = ['git', 'clone', 'https://github.com/kivy/buildozer.git']
+            p = subprocess.Popen(cmd, cwd=root_dir, shell=False)
+            p.wait()
+            if p.returncode != 0:
+                raise BuildException("Distribute build failed")
+
+            cmd = ['python', 'buildozer/buildozer/scripts/client.py', 'android', 'release']
+            p = FDroidPopen(cmd, cwd=root_dir)
+
+        # expected to fail.
+        # Signing will fail if not set by environnment vars (cf. p4a docs).
+        # But the unsigned apk will be ok.
+        p.returncode = 0
+
     elif bmethod == 'gradle':
         logging.info("Building Gradle project...")
 
@@ -896,11 +794,11 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
                            '{0}-{1}-release.apk'.format(
                                bconfig.get('app', 'title'),
                                bconfig.get('app', 'version')))
-    elif omethod == 'gradle':
+
+    elif omethod == 'buildozer':
         src = None
         for apks_dir in [
-                os.path.join(root_dir, 'build', 'outputs', 'apk'),
-                os.path.join(root_dir, 'build', 'apk'),
+                os.path.join(root_dir, '.buildozer', 'android', 'platform', 'build', 'dists', bconfig.get('app', 'title'), 'bin'),
                 ]:
             for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
                 apks = glob.glob(os.path.join(apks_dir, apkglob))
@@ -917,6 +815,37 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         if src is None:
             raise BuildException('Failed to find any output apks')
 
+    elif omethod == 'gradle':
+        src = None
+        apk_dirs = [
+            # gradle plugin >= 3.0
+            os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'),
+            # gradle plugin < 3.0 and >= 0.11
+            os.path.join(root_dir, 'build', 'outputs', 'apk'),
+            # really old path
+            os.path.join(root_dir, 'build', 'apk'),
+            ]
+        # If we build with gradle flavours with gradle plugin >= 3.0 the apk will be in
+        # a subdirectory corresponding to the flavour command used, but with different
+        # capitalization.
+        if flavours_cmd:
+            apk_dirs.append(os.path.join(root_dir, 'build', 'outputs', 'apk', transform_first_char(flavours_cmd, str.lower), 'release'))
+        for apks_dir in apk_dirs:
+            for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
+                apks = glob.glob(os.path.join(apks_dir, apkglob))
+
+                if len(apks) > 1:
+                    raise BuildException('More than one resulting apks found in %s' % apks_dir,
+                                         '\n'.join(apks))
+                if len(apks) == 1:
+                    src = apks[0]
+                    break
+            if src is not None:
+                break
+
+        if src is None:
+            raise BuildException('Failed to find any output apks')
+
     elif omethod == 'ant':
         stdout_apk = '\n'.join([
             line for line in p.output.splitlines() if '.apk' in line])
@@ -934,7 +863,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         src = os.path.normpath(apks[0])
 
     # Make sure it's not debuggable...
-    if common.isApkAndDebuggable(src, config):
+    if common.isApkAndDebuggable(src):
         raise BuildException("APK is debuggable")
 
     # By way of a sanity check, make sure the version and version
@@ -989,7 +918,7 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
        this is the 'unsigned' directory.
     :param repo_dir: The repo directory - used for checking if the build is
        necessary.
-    :paaram also_check_dir: An additional location for checking if the build
+    :param also_check_dir: An additional location for checking if the build
        is necessary (usually the archive repo)
     :param test: True if building in test mode, in which case the build will
        always happen, even if the output already exists. In test mode, the
@@ -1071,33 +1000,33 @@ def parse_commandline():
 
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
-    parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
+    parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
     parser.add_argument("-l", "--latest", action="store_true", default=False,
-                        help="Build only the latest version of each package")
+                        help=_("Build only the latest version of each package"))
     parser.add_argument("-s", "--stop", action="store_true", default=False,
-                        help="Make the build stop on exceptions")
+                        help=_("Make the build stop on exceptions"))
     parser.add_argument("-t", "--test", action="store_true", default=False,
-                        help="Test mode - put output in the tmp directory only, and always build, even if the output already exists.")
+                        help=_("Test mode - put output in the tmp directory only, and always build, even if the output already exists."))
     parser.add_argument("--server", action="store_true", default=False,
-                        help="Use build server")
+                        help=_("Use build server"))
     parser.add_argument("--resetserver", action="store_true", default=False,
-                        help="Reset and create a brand new build server, even if the existing one appears to be ok.")
+                        help=_("Reset and create a brand new build server, even if the existing one appears to be ok."))
     parser.add_argument("--on-server", dest="onserver", action="store_true", default=False,
-                        help="Specify that we're running on the build server")
+                        help=_("Specify that we're running on the build server"))
     parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False,
-                        help="Skip scanning the source code for binaries and other problems")
+                        help=_("Skip scanning the source code for binaries and other problems"))
     parser.add_argument("--dscanner", action="store_true", default=False,
-                        help="Setup an emulator, install the apk on it and perform a drozer scan")
+                        help=_("Setup an emulator, install the APK on it and perform a Drozer scan"))
     parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False,
-                        help="Don't create a source tarball, useful when testing a build")
+                        help=_("Don't create a source tarball, useful when testing a build"))
     parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True,
-                        help="Don't refresh the repository, useful when testing a build with no internet connection")
+                        help=_("Don't refresh the repository, useful when testing a build with no internet connection"))
     parser.add_argument("-f", "--force", action="store_true", default=False,
-                        help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
+                        help=_("Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode."))
     parser.add_argument("-a", "--all", action="store_true", default=False,
-                        help="Build all applications available")
+                        help=_("Build all applications available"))
     parser.add_argument("-w", "--wiki", default=False, action="store_true",
-                        help="Update the wiki")
+                        help=_("Update the wiki"))
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W
@@ -1184,7 +1113,7 @@ def main():
 
     # Read all app and srclib metadata
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(not options.onserver, pkgs)
+    allapps = metadata.read_metadata(not options.onserver, pkgs, sort_by_time=True)
     apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):
@@ -1194,6 +1123,19 @@ def main():
     if not apps:
         raise FDroidException("No apps to process.")
 
+    # make sure enough open files are allowed to process everything
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if len(apps) > soft:
+        try:
+            soft = len(apps) * 2
+            if soft > hard:
+                soft = hard
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+            logging.debug(_('Set open file limit to {integer}')
+                          .format(integer=soft))
+        except (OSError, ValueError) as e:
+            logging.warning(_('Setting open file limit failed: ') + str(e))
+
     if options.latest:
         for app in apps.values():
             for build in reversed(app.builds):
@@ -1229,6 +1171,7 @@ def main():
                     vcs, build_dir = common.setup_vcs(app)
                     first = False
 
+                logging.info("Using %s" % vcs.clientversion())
                 logging.debug("Checking " + build.versionName)
                 if trybuild(app, build, build_dir, output_dir, log_dir,
                             also_check_dir, srclib_dir, extlib_dir,
@@ -1252,7 +1195,7 @@ def main():
                         url = url.replace('%v', build.versionName)
                         url = url.replace('%c', str(build.versionCode))
                         logging.info("...retrieving " + url)
-                        of = common.get_release_filename(app, build) + '.binary'
+                        of = re.sub(r'.apk$', '.binary.apk', common.get_release_filename(app, build))
                         of = os.path.join(output_dir, of)
                         try:
                             net.download_file(url, local_filename=of)
@@ -1302,6 +1245,7 @@ def main():
                 logging.error("VCS error while building app %s: %s" % (
                     appid, reason))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = vcse
                 wikilog = str(vcse)
@@ -1316,6 +1260,7 @@ def main():
                     f.write(str(e))
                 logging.error("Could not build app %s: %s" % (appid, e))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = e
                 wikilog = e.get_wikitext()
@@ -1323,6 +1268,7 @@ def main():
                 logging.error("Could not build app %s due to unknown error: %s" % (
                     appid, traceback.format_exc()))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = e
                 wikilog = str(e)
@@ -1376,10 +1322,11 @@ def main():
                         break
 
                 if not apk_path:
-                    raise Exception("No signed APK found at path: {0}".format(apk_path))
+                    raise Exception("No signed APK found at path: {path}".format(path=apk_path))
 
                 if not os.path.isdir(repo_dir):
-                    exit(1)
+                    logging.critical("directory does not exists '{path}'".format(path=repo_dir))
+                    sys.exit(1)
 
                 logging.info("Performing Drozer scan on {0}.".format(app))
                 docker.perform_drozer_scan(apk_path, app.id, repo_dir)
@@ -1392,13 +1339,18 @@ def main():
         logging.info("Cleaning up after ourselves.")
         docker.clean()
 
-    logging.info("Finished.")
+    logging.info(_("Finished"))
     if len(build_succeeded) > 0:
-        logging.info(str(len(build_succeeded)) + ' builds succeeded')
+        logging.info(ngettext("{} build succeeded",
+                              "{} builds succeeded", len(build_succeeded)).format(len(build_succeeded)))
     if len(failed_apps) > 0:
-        logging.info(str(len(failed_apps)) + ' builds failed')
+        logging.info(ngettext("{} build failed",
+                              "{} builds failed", len(failed_apps)).format(len(failed_apps)))
 
-    sys.exit(0)
+    # hack to ensure this exits, even is some threads are still running
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(0)
 
 
 if __name__ == "__main__":