Merge branch 'log_git' into 'master'

[fdroidserver.git] / fdroidserver / build.py

diff --git a/fdroidserver/build.py b/fdroidserver/build.py
index a9ee02d7d84d7f12a5dbb47fbd63fe0bbe091291..2bacd5550fd5adfd220d61906bf5d644a94f5ac9 100644 (file)
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@ -23,6 +23,7 @@ import shutil
 import glob
 import subprocess
 import re
+import resource
 import tarfile
 import traceback
 import time
@@ -98,22 +99,19 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
 
         # Helper to copy the contents of a directory to the server...
         def send_dir(path):
-            startroot = os.path.dirname(path)
-            main = os.path.basename(path)
-            ftp.mkdir(main)
-            for root, dirs, files in os.walk(path):
-                rr = os.path.relpath(root, startroot)
-                ftp.chdir(rr)
-                for d in dirs:
-                    ftp.mkdir(d)
-                for f in files:
-                    lfile = os.path.join(startroot, rr, f)
-                    if not os.path.islink(lfile):
-                        ftp.put(lfile, f)
-                        ftp.chmod(f, os.stat(lfile).st_mode)
-                for i in range(len(rr.split('/'))):
-                    ftp.chdir('..')
-            ftp.chdir('..')
+            logging.debug("rsyncing " + path + " to " + ftp.getcwd())
+            subprocess.check_call(['rsync', '-rple',
+                                   'ssh -o StrictHostKeyChecking=no' +
+                                   ' -o UserKnownHostsFile=/dev/null' +
+                                   ' -o LogLevel=FATAL' +
+                                   ' -o IdentitiesOnly=yes' +
+                                   ' -o PasswordAuthentication=no' +
+                                   ' -p ' + str(sshinfo['port']) +
+                                   ' -i ' + sshinfo['idfile'],
+                                   path,
+                                   sshinfo['user'] +
+                                   "@" + sshinfo['hostname'] +
+                                   ":" + ftp.getcwd()])
 
         logging.info("Preparing server for build...")
         serverpath = os.path.abspath(os.path.dirname(__file__))
@@ -1113,7 +1111,7 @@ def main():
 
     # Read all app and srclib metadata
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(not options.onserver, pkgs)
+    allapps = metadata.read_metadata(not options.onserver, pkgs, sort_by_time=True)
     apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):
@@ -1123,6 +1121,19 @@ def main():
     if not apps:
         raise FDroidException("No apps to process.")
 
+    # make sure enough open files are allowed to process everything
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if len(apps) > soft:
+        try:
+            soft = len(apps) * 2
+            if soft > hard:
+                soft = hard
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+            logging.debug(_('Set open file limit to {integer}')
+                          .format(integer=soft))
+        except (OSError, ValueError) as e:
+            logging.warning(_('Setting open file limit failed: ') + str(e))
+
     if options.latest:
         for app in apps.values():
             for build in reversed(app.builds):
@@ -1156,9 +1167,9 @@ def main():
                 # there are any.
                 if first:
                     vcs, build_dir = common.setup_vcs(app)
-                    logging.info("Using %s" % vcs.clientversion())
                     first = False
 
+                logging.info("Using %s" % vcs.clientversion())
                 logging.debug("Checking " + build.versionName)
                 if trybuild(app, build, build_dir, output_dir, log_dir,
                             also_check_dir, srclib_dir, extlib_dir,
@@ -1334,7 +1345,10 @@ def main():
         logging.info(ngettext("{} build failed",
                               "{} builds failed", len(failed_apps)).format(len(failed_apps)))
 
-    sys.exit(0)
+    # hack to ensure this exits, even is some threads are still running
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(0)
 
 
 if __name__ == "__main__":