From 2b6825ccfdfd26828883d8bb83055fe376223e3b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 28 Nov 2017 10:39:35 +0100 Subject: [PATCH] build: set open file limit based on how many apps are being processed When running `fdroid build --all` on a buildserver with thousands of apps, it was frequently hitting the open file limit. This increases the open file limit based on how many apps are being process. It is doubled to provide a margin of safety. There are probably open file leaks which ideally would be fixed, but this is also useful to make things more resilient to all the random stuff apps include in their build systems. --- fdroidserver/build.py | 14 ++++++++++++++ jenkins-build-all | 1 - 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 740d7f2c..16901e78 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -23,6 +23,7 @@ import shutil import glob import subprocess import re +import resource import tarfile import traceback import time @@ -1120,6 +1121,19 @@ def main(): if not apps: raise FDroidException("No apps to process.") + # make sure enough open files are allowed to process everything + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if len(apps) > soft: + try: + soft = len(apps) * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug(_('Set open file limit to {integer}') + .format(integer=soft)) + except (OSError, ValueError) as e: + logging.warning(_('Setting open file limit failed: ') + str(e)) + if options.latest: for app in apps.values(): for build in reversed(app.builds): diff --git a/jenkins-build-all b/jenkins-build-all index 2abac58c..d41d920f 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -31,7 +31,6 @@ else echo "No virtualization is used." fi sudo /bin/chmod -R a+rX /var/lib/libvirt/images -ulimit -n 2048 echo 'maximum allowed number of open file descriptors: ' `ulimit -n` ls -ld /var/lib/libvirt/images ls -l /var/lib/libvirt/images || echo no access -- 2.30.2