chiark / gitweb /
build: enable watchdog timer for each build that kills in 2 hours
[fdroidserver.git] / fdroidserver / build.py
index f2fbad046e68e64216b657803a1d1c3f87cf07e0..cb030ebb4abb8981d697dd67948694bf3495481f 100644 (file)
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
 import os
 import shutil
 import glob
 import subprocess
 import re
+import resource
+import sys
 import tarfile
+import threading
 import traceback
 import time
 import requests
@@ -31,7 +33,9 @@ import tempfile
 from configparser import ConfigParser
 from argparse import ArgumentParser
 import logging
+from gettext import ngettext
 
+from . import _
 from . import common
 from . import net
 from . import metadata
@@ -75,7 +79,9 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         if not buildserverid:
             buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
                                                      'cat /home/vagrant/buildserverid'],
-                                                    cwd='builder').rstrip()
+                                                    cwd='builder').rstrip().decode()
+            logging.debug(_('Fetched buildserverid from VM: {buildserverid}')
+                          .format(buildserverid=buildserverid))
 
         # Open SSH connection...
         logging.info("Connecting to virtual machine...")
@@ -96,22 +102,22 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
 
         # Helper to copy the contents of a directory to the server...
         def send_dir(path):
-            root = os.path.dirname(path)
-            main = os.path.basename(path)
-            ftp.mkdir(main)
-            for r, d, f in os.walk(path):
-                rr = os.path.relpath(r, root)
-                ftp.chdir(rr)
-                for dd in d:
-                    ftp.mkdir(dd)
-                for ff in f:
-                    lfile = os.path.join(root, rr, ff)
-                    if not os.path.islink(lfile):
-                        ftp.put(lfile, ff)
-                        ftp.chmod(ff, os.stat(lfile).st_mode)
-                for i in range(len(rr.split('/'))):
-                    ftp.chdir('..')
-            ftp.chdir('..')
+            logging.debug("rsyncing " + path + " to " + ftp.getcwd())
+            # TODO this should move to `vagrant rsync` from >= v1.5
+            try:
+                subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' +
+                                         'ssh -o StrictHostKeyChecking=no' +
+                                         ' -o UserKnownHostsFile=/dev/null' +
+                                         ' -o LogLevel=FATAL' +
+                                         ' -o IdentitiesOnly=yes' +
+                                         ' -o PasswordAuthentication=no' +
+                                         ' -p ' + str(sshinfo['port']) +
+                                         ' -i ' + sshinfo['idfile'],
+                                         path,
+                                         sshinfo['user'] + "@" + sshinfo['hostname'] + ":" + ftp.getcwd()],
+                                        stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                raise FDroidException(str(e), e.output.decode())
 
         logging.info("Preparing server for build...")
         serverpath = os.path.abspath(os.path.dirname(__file__))
@@ -136,8 +142,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         ftp.mkdir('metadata')
         ftp.mkdir('srclibs')
         ftp.chdir('metadata')
-        ftp.put(os.path.join('metadata', app.id + '.txt'),
-                app.id + '.txt')
+        ftp.put(app.metadatapath, os.path.basename(app.metadatapath))
+
         # And patches if there are any...
         if os.path.exists(os.path.join('metadata', app.id)):
             send_dir(os.path.join('metadata', app.id))
@@ -162,7 +168,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
                         ftp.mkdir(d)
                     ftp.chdir(d)
                 ftp.put(libsrc, lp[-1])
-                for _ in lp[:-1]:
+                for _ignored in lp[:-1]:
                     ftp.chdir('..')
         # Copy any srclibs that are required...
         srclibpaths = []
@@ -210,23 +216,33 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         cmdline += " %s:%s" % (app.id, build.versionCode)
         chan.exec_command('bash --login -c "' + cmdline + '"')
 
-        output = bytes()
-        output += get_android_tools_version_log(build.ndk_path()).encode()
-        while not chan.exit_status_ready():
-            while chan.recv_ready():
-                output += chan.recv(1024)
-            time.sleep(0.1)
+        # Fetch build process output ...
+        try:
+            cmd_stdout = chan.makefile('rb', 1024)
+            output = bytes()
+            output += common.get_android_tools_version_log(build.ndk_path()).encode()
+            while not chan.exit_status_ready():
+                line = cmd_stdout.readline()
+                if line:
+                    if options.verbose:
+                        logging.debug("buildserver > " + str(line, 'utf-8').rstrip())
+                    output += line
+                else:
+                    time.sleep(0.05)
+            for line in cmd_stdout.readlines():
+                if options.verbose:
+                    logging.debug("buildserver > " + str(line, 'utf-8').rstrip())
+                output += line
+        finally:
+            cmd_stdout.close()
+
+        # Check build process exit status ...
         logging.info("...getting exit status")
         returncode = chan.recv_exit_status()
-        while True:
-            get = chan.recv(1024)
-            if len(get) == 0:
-                break
-            output += get
         if returncode != 0:
             raise BuildException(
                 "Build.py failed on server for {0}:{1}".format(
-                    app.id, build.versionName), str(output, 'utf-8'))
+                    app.id, build.versionName), None if options.verbose else str(output, 'utf-8'))
 
         # Retreive logs...
         toolsversion_log = common.get_toolsversion_logname(app, build)
@@ -251,8 +267,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
                 ftp.get(tarball, os.path.join(output_dir, tarball))
         except Exception:
             raise BuildException(
-                "Build failed for %s:%s - missing output files".format(
-                    app.id, build.versionName), output)
+                "Build failed for {0}:{1} - missing output files".format(
+                    app.id, build.versionName), None if options.verbose else str(output, 'utf-8'))
         ftp.close()
 
     finally:
@@ -275,14 +291,13 @@ def force_gradle_build_tools(build_dir, build_tools):
                                path)
 
 
-def capitalize_intact(string):
-    """Like str.capitalize(), but leave the rest of the string intact without
-    switching it to lowercase."""
+def transform_first_char(string, method):
+    """Uses method() on the first character of string."""
     if len(string) == 0:
         return string
     if len(string) == 1:
-        return string.upper()
-    return string[0].upper() + string[1:]
+        return method(string)
+    return method(string[0]) + string[1:]
 
 
 def has_native_code(apkobj):
@@ -393,10 +408,33 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
 
     # create ..._toolsversion.log when running in builder vm
     if onserver:
+        # before doing anything, run the sudo commands to setup the VM
+        if build.sudo:
+            logging.info("Running 'sudo' commands in %s" % os.getcwd())
+
+            p = FDroidPopen(['sudo', 'bash', '-x', '-c', build.sudo])
+            if p.returncode != 0:
+                raise BuildException("Error running sudo command for %s:%s" %
+                                     (app.id, build.versionName), p.output)
+
+        p = FDroidPopen(['sudo', 'passwd', '--lock', 'root'])
+        if p.returncode != 0:
+            raise BuildException("Error locking root account for %s:%s" %
+                                 (app.id, build.versionName), p.output)
+
+        p = FDroidPopen(['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo'])
+        if p.returncode != 0:
+            raise BuildException("Error removing sudo for %s:%s" %
+                                 (app.id, build.versionName), p.output)
+
         log_path = os.path.join(log_dir,
                                 common.get_toolsversion_logname(app, build))
         with open(log_path, 'w') as f:
-            f.write(get_android_tools_version_log(build.ndk_path()))
+            f.write(common.get_android_tools_version_log(build.ndk_path()))
+    else:
+        if build.sudo:
+            logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s'
+                            % (app.id, build.versionName, build.sudo))
 
     # Prepare the source code...
     root_dir, srclibpaths = common.prepare_source(vcs, app, build,
@@ -431,7 +469,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         if flavours == ['yes']:
             flavours = []
 
-        flavours_cmd = ''.join([capitalize_intact(flav) for flav in flavours])
+        flavours_cmd = ''.join([transform_first_char(flav, str.upper) for flav in flavours])
 
         gradletasks += ['assemble' + flavours_cmd + 'Release']
 
@@ -451,6 +489,9 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
     elif bmethod == 'kivy':
         pass
 
+    elif bmethod == 'buildozer':
+        pass
+
     elif bmethod == 'ant':
         logging.info("Cleaning Ant project...")
         p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
@@ -471,12 +512,18 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
                 if f in files:
                     os.remove(os.path.join(root, f))
 
-        if 'build.gradle' in files:
+        if any(f in files for f in ['build.gradle', 'settings.gradle']):
             # Even when running clean, gradle stores task/artifact caches in
             # .gradle/ as binary files. To avoid overcomplicating the scanner,
             # manually delete them, just like `gradle clean` should have removed
-            # the build/ dirs.
-            del_dirs(['build', '.gradle'])
+            # the build/* dirs.
+            del_dirs([os.path.join('build', 'android-profile'),
+                      os.path.join('build', 'generated'),
+                      os.path.join('build', 'intermediates'),
+                      os.path.join('build', 'outputs'),
+                      os.path.join('build', 'reports'),
+                      os.path.join('build', 'tmp'),
+                      '.gradle'])
             del_files(['gradlew', 'gradlew.bat'])
 
         if 'pom.xml' in files:
@@ -497,9 +544,12 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         count = scanner.scan_source(build_dir, build)
         if count > 0:
             if force:
-                logging.warn('Scanner found %d problems' % count)
+                logging.warning(ngettext('Scanner found {} problem',
+                                         'Scanner found {} problems', count).format(count))
             else:
-                raise BuildException("Can't build due to %d errors while scanning" % count)
+                raise BuildException(ngettext(
+                    "Can't build due to {} error while scanning",
+                    "Can't build due to {} errors while scanning", count).format(count))
 
     if not options.notarball:
         # Build the source tarball right before we build the release...
@@ -645,6 +695,73 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         cmd.append('release')
         p = FDroidPopen(cmd, cwd=distdir)
 
+    elif bmethod == 'buildozer':
+        logging.info("Building Kivy project using buildozer...")
+
+        # parse buildozer.spez
+        spec = os.path.join(root_dir, 'buildozer.spec')
+        if not os.path.exists(spec):
+            raise BuildException("Expected to find buildozer-compatible spec at {0}"
+                                 .format(spec))
+        defaults = {'orientation': 'landscape', 'icon': '',
+                    'permissions': '', 'android.api': "19"}
+        bconfig = ConfigParser(defaults, allow_no_value=True)
+        bconfig.read(spec)
+
+        # update spec with sdk and ndk locations to prevent buildozer from
+        # downloading.
+        loc_ndk = common.env['ANDROID_NDK']
+        loc_sdk = common.env['ANDROID_SDK']
+        if loc_ndk == '$ANDROID_NDK':
+            loc_ndk = loc_sdk + '/ndk-bundle'
+
+        bc_ndk = None
+        bc_sdk = None
+        try:
+            bc_ndk = bconfig.get('app', 'android.sdk_path')
+        except Exception:
+            pass
+        try:
+            bc_sdk = bconfig.get('app', 'android.ndk_path')
+        except Exception:
+            pass
+
+        if bc_sdk is None:
+            bconfig.set('app', 'android.sdk_path', loc_sdk)
+        if bc_ndk is None:
+            bconfig.set('app', 'android.ndk_path', loc_ndk)
+
+        fspec = open(spec, 'w')
+        bconfig.write(fspec)
+        fspec.close()
+
+        logging.info("sdk_path = %s" % loc_sdk)
+        logging.info("ndk_path = %s" % loc_ndk)
+
+        p = None
+        # execute buildozer
+        cmd = ['buildozer', 'android', 'release']
+        try:
+            p = FDroidPopen(cmd, cwd=root_dir)
+        except Exception:
+            pass
+
+        # buidozer not installed ? clone repo and run
+        if (p is None or p.returncode != 0):
+            cmd = ['git', 'clone', 'https://github.com/kivy/buildozer.git']
+            p = subprocess.Popen(cmd, cwd=root_dir, shell=False)
+            p.wait()
+            if p.returncode != 0:
+                raise BuildException("Distribute build failed")
+
+            cmd = ['python', 'buildozer/buildozer/scripts/client.py', 'android', 'release']
+            p = FDroidPopen(cmd, cwd=root_dir)
+
+        # expected to fail.
+        # Signing will fail if not set by environnment vars (cf. p4a docs).
+        # But the unsigned apk will be ok.
+        p.returncode = 0
+
     elif bmethod == 'gradle':
         logging.info("Building Gradle project...")
 
@@ -697,11 +814,11 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
                            '{0}-{1}-release.apk'.format(
                                bconfig.get('app', 'title'),
                                bconfig.get('app', 'version')))
-    elif omethod == 'gradle':
+
+    elif omethod == 'buildozer':
         src = None
         for apks_dir in [
-                os.path.join(root_dir, 'build', 'outputs', 'apk'),
-                os.path.join(root_dir, 'build', 'apk'),
+                os.path.join(root_dir, '.buildozer', 'android', 'platform', 'build', 'dists', bconfig.get('app', 'title'), 'bin'),
                 ]:
             for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
                 apks = glob.glob(os.path.join(apks_dir, apkglob))
@@ -718,6 +835,37 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         if src is None:
             raise BuildException('Failed to find any output apks')
 
+    elif omethod == 'gradle':
+        src = None
+        apk_dirs = [
+            # gradle plugin >= 3.0
+            os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'),
+            # gradle plugin < 3.0 and >= 0.11
+            os.path.join(root_dir, 'build', 'outputs', 'apk'),
+            # really old path
+            os.path.join(root_dir, 'build', 'apk'),
+            ]
+        # If we build with gradle flavours with gradle plugin >= 3.0 the apk will be in
+        # a subdirectory corresponding to the flavour command used, but with different
+        # capitalization.
+        if flavours_cmd:
+            apk_dirs.append(os.path.join(root_dir, 'build', 'outputs', 'apk', transform_first_char(flavours_cmd, str.lower), 'release'))
+        for apks_dir in apk_dirs:
+            for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
+                apks = glob.glob(os.path.join(apks_dir, apkglob))
+
+                if len(apks) > 1:
+                    raise BuildException('More than one resulting apks found in %s' % apks_dir,
+                                         '\n'.join(apks))
+                if len(apks) == 1:
+                    src = apks[0]
+                    break
+            if src is not None:
+                break
+
+        if src is None:
+            raise BuildException('Failed to find any output apks')
+
     elif omethod == 'ant':
         stdout_apk = '\n'.join([
             line for line in p.output.splitlines() if '.apk' in line])
@@ -823,7 +971,7 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
     if server:
         # When using server mode, still keep a local cache of the repo, by
         # grabbing the source now.
-        vcs.gotorevision(build.commit)
+        vcs.gotorevision(build.commit, refresh)
 
         build_server(app, build, vcs, build_dir, output_dir, log_dir, force)
     else:
@@ -831,40 +979,11 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
     return True
 
 
-def get_android_tools_versions(ndk_path=None):
-    '''get a list of the versions of all installed Android SDK/NDK components'''
-
-    global config
-    sdk_path = config['sdk_path']
-    if sdk_path[-1] != '/':
-        sdk_path += '/'
-    components = []
-    if ndk_path:
-        ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
-        if os.path.isfile(ndk_release_txt):
-            with open(ndk_release_txt, 'r') as fp:
-                components.append((os.path.basename(ndk_path), fp.read()[:-1]))
-
-    pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
-    for root, dirs, files in os.walk(sdk_path):
-        if 'source.properties' in files:
-            source_properties = os.path.join(root, 'source.properties')
-            with open(source_properties, 'r') as fp:
-                m = pattern.search(fp.read())
-                if m:
-                    components.append((root[len(sdk_path):], m.group(1)))
-
-    return components
-
-
-def get_android_tools_version_log(ndk_path):
-    '''get a list of the versions of all installed Android SDK/NDK components'''
-    log = '== Installed Android Tools ==\n\n'
-    components = get_android_tools_versions(ndk_path)
-    for name, version in sorted(components):
-        log += '* ' + name + ' (' + version + ')\n'
-
-    return log
+def force_halt_build():
+    """Halt the currently running Vagrant VM, to be called from a Timer"""
+    logging.error(_('Force halting build after timeout!'))
+    vm = vmtools.get_build_vm('builder')
+    vm.halt()
 
 
 def parse_commandline():
@@ -872,33 +991,33 @@ def parse_commandline():
 
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
-    parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
+    parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
     parser.add_argument("-l", "--latest", action="store_true", default=False,
-                        help="Build only the latest version of each package")
+                        help=_("Build only the latest version of each package"))
     parser.add_argument("-s", "--stop", action="store_true", default=False,
-                        help="Make the build stop on exceptions")
+                        help=_("Make the build stop on exceptions"))
     parser.add_argument("-t", "--test", action="store_true", default=False,
-                        help="Test mode - put output in the tmp directory only, and always build, even if the output already exists.")
+                        help=_("Test mode - put output in the tmp directory only, and always build, even if the output already exists."))
     parser.add_argument("--server", action="store_true", default=False,
-                        help="Use build server")
+                        help=_("Use build server"))
     parser.add_argument("--resetserver", action="store_true", default=False,
-                        help="Reset and create a brand new build server, even if the existing one appears to be ok.")
+                        help=_("Reset and create a brand new build server, even if the existing one appears to be ok."))
     parser.add_argument("--on-server", dest="onserver", action="store_true", default=False,
-                        help="Specify that we're running on the build server")
+                        help=_("Specify that we're running on the build server"))
     parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False,
-                        help="Skip scanning the source code for binaries and other problems")
+                        help=_("Skip scanning the source code for binaries and other problems"))
     parser.add_argument("--dscanner", action="store_true", default=False,
-                        help="Setup an emulator, install the apk on it and perform a drozer scan")
+                        help=_("Setup an emulator, install the APK on it and perform a Drozer scan"))
     parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False,
-                        help="Don't create a source tarball, useful when testing a build")
+                        help=_("Don't create a source tarball, useful when testing a build"))
     parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True,
-                        help="Don't refresh the repository, useful when testing a build with no internet connection")
+                        help=_("Don't refresh the repository, useful when testing a build with no internet connection"))
     parser.add_argument("-f", "--force", action="store_true", default=False,
-                        help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
+                        help=_("Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode."))
     parser.add_argument("-a", "--all", action="store_true", default=False,
-                        help="Build all applications available")
+                        help=_("Build all applications available"))
     parser.add_argument("-w", "--wiki", default=False, action="store_true",
-                        help="Update the wiki")
+                        help=_("Update the wiki"))
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W
@@ -916,11 +1035,13 @@ def parse_commandline():
 options = None
 config = None
 buildserverid = None
+fdroidserverid = None
+start_timestamp = time.gmtime()
 
 
 def main():
 
-    global options, config, buildserverid
+    global options, config, buildserverid, fdroidserverid
 
     options, parser = parse_commandline()
 
@@ -985,7 +1106,7 @@ def main():
 
     # Read all app and srclib metadata
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(not options.onserver, pkgs)
+    allapps = metadata.read_metadata(not options.onserver, pkgs, options.refresh, sort_by_time=True)
     apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):
@@ -995,6 +1116,19 @@ def main():
     if not apps:
         raise FDroidException("No apps to process.")
 
+    # make sure enough open files are allowed to process everything
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if len(apps) > soft:
+        try:
+            soft = len(apps) * 2
+            if soft > hard:
+                soft = hard
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+            logging.debug(_('Set open file limit to {integer}')
+                          .format(integer=soft))
+        except (OSError, ValueError) as e:
+            logging.warning(_('Setting open file limit failed: ') + str(e))
+
     if options.latest:
         for app in apps.values():
             for build in reversed(app.builds):
@@ -1017,10 +1151,17 @@ def main():
         first = True
 
         for build in app.builds:
+            if options.server:  # enable watchdog timer
+                timer = threading.Timer(7200, force_halt_build)
+                timer.start()
+            else:
+                timer = None
+
             wikilog = None
+            build_starttime = common.get_wiki_timestamp()
             tools_version_log = ''
             if not options.onserver:
-                tools_version_log = get_android_tools_version_log(build.ndk_path())
+                tools_version_log = common.get_android_tools_version_log(build.ndk_path())
             try:
 
                 # For the first build of a particular app, we need to set up
@@ -1030,6 +1171,7 @@ def main():
                     vcs, build_dir = common.setup_vcs(app)
                     first = False
 
+                logging.info("Using %s" % vcs.clientversion())
                 logging.debug("Checking " + build.versionName)
                 if trybuild(app, build, build_dir, output_dir, log_dir,
                             also_check_dir, srclib_dir, extlib_dir,
@@ -1053,7 +1195,7 @@ def main():
                         url = url.replace('%v', build.versionName)
                         url = url.replace('%c', str(build.versionCode))
                         logging.info("...retrieving " + url)
-                        of = common.get_release_filename(app, build) + '.binary'
+                        of = re.sub(r'.apk$', '.binary.apk', common.get_release_filename(app, build))
                         of = os.path.join(output_dir, of)
                         try:
                             net.download_file(url, local_filename=of)
@@ -1103,6 +1245,7 @@ def main():
                 logging.error("VCS error while building app %s: %s" % (
                     appid, reason))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = vcse
                 wikilog = str(vcse)
@@ -1112,11 +1255,12 @@ def main():
                     f.write('versionCode: %s\nversionName: %s\ncommit: %s\n' %
                             (build.versionCode, build.versionName, build.commit))
                     f.write('Build completed at '
-                            + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + '\n')
+                            + common.get_wiki_timestamp() + '\n')
                     f.write('\n' + tools_version_log + '\n')
                     f.write(str(e))
                 logging.error("Could not build app %s: %s" % (appid, e))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = e
                 wikilog = e.get_wikitext()
@@ -1124,6 +1268,7 @@ def main():
                 logging.error("Could not build app %s due to unknown error: %s" % (
                     appid, traceback.format_exc()))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = e
                 wikilog = str(e)
@@ -1135,10 +1280,12 @@ def main():
                     newpage = site.Pages[lastbuildpage]
                     with open(os.path.join('tmp', 'fdroidserverid')) as fp:
                         fdroidserverid = fp.read().rstrip()
-                    txt = "* build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + '\n' \
+                    txt = "* build session started at " + common.get_wiki_timestamp(start_timestamp) + '\n' \
+                          + "* this build started at " + build_starttime + '\n' \
+                          + "* this build completed at " + common.get_wiki_timestamp() + '\n' \
                           + '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \
                           + fdroidserverid + ' ' + fdroidserverid + ']\n\n'
-                    if options.onserver:
+                    if buildserverid:
                         txt += '* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \
                                + buildserverid + ' ' + buildserverid + ']\n\n'
                     txt += tools_version_log + '\n\n'
@@ -1150,6 +1297,9 @@ def main():
                 except Exception as e:
                     logging.error("Error while attempting to publish build log: %s" % e)
 
+            if timer:
+                timer.cancel()  # kill the watchdog timer
+
     for app in build_succeeded:
         logging.info("success: %s" % (app.id))
 
@@ -1177,10 +1327,11 @@ def main():
                         break
 
                 if not apk_path:
-                    raise Exception("No signed APK found at path: {0}".format(apk_path))
+                    raise Exception("No signed APK found at path: {path}".format(path=apk_path))
 
                 if not os.path.isdir(repo_dir):
-                    exit(1)
+                    logging.critical("directory does not exists '{path}'".format(path=repo_dir))
+                    sys.exit(1)
 
                 logging.info("Performing Drozer scan on {0}.".format(app))
                 docker.perform_drozer_scan(apk_path, app.id, repo_dir)
@@ -1193,13 +1344,47 @@ def main():
         logging.info("Cleaning up after ourselves.")
         docker.clean()
 
-    logging.info("Finished.")
+    logging.info(_("Finished"))
     if len(build_succeeded) > 0:
-        logging.info(str(len(build_succeeded)) + ' builds succeeded')
+        logging.info(ngettext("{} build succeeded",
+                              "{} builds succeeded", len(build_succeeded)).format(len(build_succeeded)))
     if len(failed_apps) > 0:
-        logging.info(str(len(failed_apps)) + ' builds failed')
+        logging.info(ngettext("{} build failed",
+                              "{} builds failed", len(failed_apps)).format(len(failed_apps)))
 
-    sys.exit(0)
+    if options.wiki:
+        wiki_page_path = 'build_' + time.strftime('%s', start_timestamp)
+        newpage = site.Pages[wiki_page_path]
+        txt = ''
+        txt += "* command line: <code>%s</code>\n" % ' '.join(sys.argv)
+        txt += "* started at %s\n" % common.get_wiki_timestamp(start_timestamp)
+        txt += "* completed at %s\n" % common.get_wiki_timestamp()
+        if buildserverid:
+            txt += ('* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
+                    .format(id=buildserverid))
+        if fdroidserverid:
+            txt += ('* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
+                    .format(id=fdroidserverid))
+        if os.cpu_count():
+            txt += "* host processors: %d\n" % os.cpu_count()
+        if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK):
+            with open('/proc/meminfo') as fp:
+                for line in fp:
+                    m = re.search(r'MemTotal:\s*([0-9].*)', line)
+                    if m:
+                        txt += "* host RAM: %s\n" % m.group(1)
+                        break
+        txt += "* successful builds: %d\n" % len(build_succeeded)
+        txt += "* failed builds: %d\n" % len(failed_apps)
+        txt += "\n\n"
+        newpage.save(txt, summary='Run log')
+        newpage = site.Pages['build']
+        newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+
+    # hack to ensure this exits, even is some threads are still running
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(0)
 
 
 if __name__ == "__main__":