build: write out full rsync options

[fdroidserver.git] / fdroidserver / build.py

diff --git a/fdroidserver/build.py b/fdroidserver/build.py
index 0d0f687a01f9c84a7b0492c18ad855385d463063..bbb5a2b4d0b2e7492210c1298f5ada9177e85b83 100644 (file)
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@ -23,6 +23,7 @@ import shutil
 import glob
 import subprocess
 import re
+import resource
 import tarfile
 import traceback
 import time
@@ -99,18 +100,20 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         # Helper to copy the contents of a directory to the server...
         def send_dir(path):
             logging.debug("rsyncing " + path + " to " + ftp.getcwd())
-            subprocess.check_call(['rsync', '-rple',
-                                   'ssh -o StrictHostKeyChecking=no' +
-                                   ' -o UserKnownHostsFile=/dev/null' +
-                                   ' -o LogLevel=FATAL' +
-                                   ' -o IdentitiesOnly=yes' +
-                                   ' -o PasswordAuthentication=no' +
-                                   ' -p ' + str(sshinfo['port']) +
-                                   ' -i ' + sshinfo['idfile'],
-                                   path,
-                                   sshinfo['user'] +
-                                   "@" + sshinfo['hostname'] +
-                                   ":" + ftp.getcwd()])
+            try:
+                subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' +
+                                         'ssh -o StrictHostKeyChecking=no' +
+                                         ' -o UserKnownHostsFile=/dev/null' +
+                                         ' -o LogLevel=FATAL' +
+                                         ' -o IdentitiesOnly=yes' +
+                                         ' -o PasswordAuthentication=no' +
+                                         ' -p ' + str(sshinfo['port']) +
+                                         ' -i ' + sshinfo['idfile'],
+                                         path,
+                                         sshinfo['user'] + "@" + sshinfo['hostname'] + ":" + ftp.getcwd()],
+                                        stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                raise FDroidException(str(e), e.output.decode())
 
         logging.info("Preparing server for build...")
         serverpath = os.path.abspath(os.path.dirname(__file__))
@@ -1110,7 +1113,7 @@ def main():
 
     # Read all app and srclib metadata
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(not options.onserver, pkgs)
+    allapps = metadata.read_metadata(not options.onserver, pkgs, sort_by_time=True)
     apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):
@@ -1120,6 +1123,19 @@ def main():
     if not apps:
         raise FDroidException("No apps to process.")
 
+    # make sure enough open files are allowed to process everything
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if len(apps) > soft:
+        try:
+            soft = len(apps) * 2
+            if soft > hard:
+                soft = hard
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+            logging.debug(_('Set open file limit to {integer}')
+                          .format(integer=soft))
+        except (OSError, ValueError) as e:
+            logging.warning(_('Setting open file limit failed: ') + str(e))
+
     if options.latest:
         for app in apps.values():
             for build in reversed(app.builds):
@@ -1331,7 +1347,10 @@ def main():
         logging.info(ngettext("{} build failed",
                               "{} builds failed", len(failed_apps)).format(len(failed_apps)))
 
-    sys.exit(0)
+    # hack to ensure this exits, even is some threads are still running
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(0)
 
 
 if __name__ == "__main__":