Merge branch 'log_git' into 'master'

[fdroidserver.git] / fdroidserver / build.py

diff --git a/fdroidserver/build.py b/fdroidserver/build.py
index 60c63603a93579578526c3ed7f3e117cbea9a3c4..2bacd5550fd5adfd220d61906bf5d644a94f5ac9 100644 (file)
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@ -23,6 +23,7 @@ import shutil
 import glob
 import subprocess
 import re
+import resource
 import tarfile
 import traceback
 import time
@@ -98,22 +99,19 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
 
         # Helper to copy the contents of a directory to the server...
         def send_dir(path):
-            startroot = os.path.dirname(path)
-            main = os.path.basename(path)
-            ftp.mkdir(main)
-            for root, dirs, files in os.walk(path):
-                rr = os.path.relpath(root, startroot)
-                ftp.chdir(rr)
-                for d in dirs:
-                    ftp.mkdir(d)
-                for f in files:
-                    lfile = os.path.join(startroot, rr, f)
-                    if not os.path.islink(lfile):
-                        ftp.put(lfile, f)
-                        ftp.chmod(f, os.stat(lfile).st_mode)
-                for i in range(len(rr.split('/'))):
-                    ftp.chdir('..')
-            ftp.chdir('..')
+            logging.debug("rsyncing " + path + " to " + ftp.getcwd())
+            subprocess.check_call(['rsync', '-rple',
+                                   'ssh -o StrictHostKeyChecking=no' +
+                                   ' -o UserKnownHostsFile=/dev/null' +
+                                   ' -o LogLevel=FATAL' +
+                                   ' -o IdentitiesOnly=yes' +
+                                   ' -o PasswordAuthentication=no' +
+                                   ' -p ' + str(sshinfo['port']) +
+                                   ' -i ' + sshinfo['idfile'],
+                                   path,
+                                   sshinfo['user'] +
+                                   "@" + sshinfo['hostname'] +
+                                   ":" + ftp.getcwd()])
 
         logging.info("Preparing server for build...")
         serverpath = os.path.abspath(os.path.dirname(__file__))
@@ -498,7 +496,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
                 if f in files:
                     os.remove(os.path.join(root, f))
 
-        if 'build.gradle' in files:
+        if any(f in files for f in ['build.gradle', 'settings.gradle']):
             # Even when running clean, gradle stores task/artifact caches in
             # .gradle/ as binary files. To avoid overcomplicating the scanner,
             # manually delete them, just like `gradle clean` should have removed
@@ -1113,7 +1111,7 @@ def main():
 
     # Read all app and srclib metadata
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(not options.onserver, pkgs)
+    allapps = metadata.read_metadata(not options.onserver, pkgs, sort_by_time=True)
     apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):
@@ -1123,6 +1121,19 @@ def main():
     if not apps:
         raise FDroidException("No apps to process.")
 
+    # make sure enough open files are allowed to process everything
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if len(apps) > soft:
+        try:
+            soft = len(apps) * 2
+            if soft > hard:
+                soft = hard
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+            logging.debug(_('Set open file limit to {integer}')
+                          .format(integer=soft))
+        except (OSError, ValueError) as e:
+            logging.warning(_('Setting open file limit failed: ') + str(e))
+
     if options.latest:
         for app in apps.values():
             for build in reversed(app.builds):
@@ -1158,6 +1169,7 @@ def main():
                     vcs, build_dir = common.setup_vcs(app)
                     first = False
 
+                logging.info("Using %s" % vcs.clientversion())
                 logging.debug("Checking " + build.versionName)
                 if trybuild(app, build, build_dir, output_dir, log_dir,
                             also_check_dir, srclib_dir, extlib_dir,
@@ -1231,6 +1243,7 @@ def main():
                 logging.error("VCS error while building app %s: %s" % (
                     appid, reason))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = vcse
                 wikilog = str(vcse)
@@ -1245,6 +1258,7 @@ def main():
                     f.write(str(e))
                 logging.error("Could not build app %s: %s" % (appid, e))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = e
                 wikilog = e.get_wikitext()
@@ -1252,6 +1266,7 @@ def main():
                 logging.error("Could not build app %s due to unknown error: %s" % (
                     appid, traceback.format_exc()))
                 if options.stop:
+                    logging.debug("Error encoutered, stopping by user request.")
                     sys.exit(1)
                 failed_apps[appid] = e
                 wikilog = str(e)
@@ -1305,10 +1320,11 @@ def main():
                         break
 
                 if not apk_path:
-                    raise Exception("No signed APK found at path: {0}".format(apk_path))
+                    raise Exception("No signed APK found at path: {path}".format(path=apk_path))
 
                 if not os.path.isdir(repo_dir):
-                    exit(1)
+                    logging.critical("directory does not exists '{path}'".format(path=repo_dir))
+                    sys.exit(1)
 
                 logging.info("Performing Drozer scan on {0}.".format(app))
                 docker.perform_drozer_scan(apk_path, app.id, repo_dir)
@@ -1329,7 +1345,10 @@ def main():
         logging.info(ngettext("{} build failed",
                               "{} builds failed", len(failed_apps)).format(len(failed_apps)))
 
-    sys.exit(0)
+    # hack to ensure this exits, even is some threads are still running
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(0)
 
 
 if __name__ == "__main__":