build: add global soft timeout of 12 hours

[fdroidserver.git] / fdroidserver / build.py

diff --git a/fdroidserver/build.py b/fdroidserver/build.py
index f3b25c9c3a6a9dfbe64d761936caf06337893a5d..05700cf7099b19193c84dd5d6074b12dff1974e8 100644 (file)
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@ -17,14 +17,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
 import os
 import shutil
 import glob
 import subprocess
 import re
 import resource
+import sys
 import tarfile
+import threading
 import traceback
 import time
 import requests
@@ -239,9 +240,12 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         logging.info("...getting exit status")
         returncode = chan.recv_exit_status()
         if returncode != 0:
-            raise BuildException(
-                "Build.py failed on server for {0}:{1}".format(
-                    app.id, build.versionName), None if options.verbose else str(output, 'utf-8'))
+            if timeout_event.is_set():
+                message = "Timeout exceeded! Build VM force-stopped for {0}:{1}"
+            else:
+                message = "Build.py failed on server for {0}:{1}"
+            raise BuildException(message.format(app.id, build.versionName),
+                                 None if options.verbose else str(output, 'utf-8'))
 
         # Retreive logs...
         toolsversion_log = common.get_toolsversion_logname(app, build)
@@ -978,6 +982,14 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
     return True
 
 
+def force_halt_build():
+    """Halt the currently running Vagrant VM, to be called from a Timer"""
+    logging.error(_('Force halting build after timeout!'))
+    timeout_event.set()
+    vm = vmtools.get_build_vm('builder')
+    vm.halt()
+
+
 def parse_commandline():
     """Parse the command line. Returns options, parser."""
 
@@ -1027,12 +1039,14 @@ def parse_commandline():
 options = None
 config = None
 buildserverid = None
-starttime = common.get_wiki_timestamp()
+fdroidserverid = None
+start_timestamp = time.gmtime()
+timeout_event = threading.Event()
 
 
 def main():
 
-    global options, config, buildserverid
+    global options, config, buildserverid, fdroidserverid
 
     options, parser = parse_commandline()
 
@@ -1137,15 +1151,23 @@ def main():
     # Build applications...
     failed_apps = {}
     build_succeeded = []
-    max_apps_per_run = 50
+    # Only build for 12 hours, then stop gracefully
+    endtime = time.time() + 12 * 60 * 60
+    max_build_time_reached = False
     for appid, app in apps.items():
-        max_apps_per_run -= 1
-        if max_apps_per_run < 1:
-            break
 
         first = True
 
         for build in app.builds:
+            if time.time() > endtime:
+                max_build_time_reached = True
+                break
+            if options.server:  # enable watchdog timer
+                timer = threading.Timer(7200, force_halt_build)
+                timer.start()
+            else:
+                timer = None
+
             wikilog = None
             build_starttime = common.get_wiki_timestamp()
             tools_version_log = ''
@@ -1269,7 +1291,7 @@ def main():
                     newpage = site.Pages[lastbuildpage]
                     with open(os.path.join('tmp', 'fdroidserverid')) as fp:
                         fdroidserverid = fp.read().rstrip()
-                    txt = "* build session started at " + starttime + '\n' \
+                    txt = "* build session started at " + common.get_wiki_timestamp(start_timestamp) + '\n' \
                           + "* this build started at " + build_starttime + '\n' \
                           + "* this build completed at " + common.get_wiki_timestamp() + '\n' \
                           + '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \
@@ -1286,6 +1308,13 @@ def main():
                 except Exception as e:
                     logging.error("Error while attempting to publish build log: %s" % e)
 
+            if timer:
+                timer.cancel()  # kill the watchdog timer
+
+        if max_build_time_reached:
+            logging.info("Stopping after global build timeout...")
+            break
+
     for app in build_succeeded:
         logging.info("success: %s" % (app.id))
 
@@ -1338,6 +1367,35 @@ def main():
         logging.info(ngettext("{} build failed",
                               "{} builds failed", len(failed_apps)).format(len(failed_apps)))
 
+    if options.wiki:
+        wiki_page_path = 'build_' + time.strftime('%s', start_timestamp)
+        newpage = site.Pages[wiki_page_path]
+        txt = ''
+        txt += "* command line: <code>%s</code>\n" % ' '.join(sys.argv)
+        txt += "* started at %s\n" % common.get_wiki_timestamp(start_timestamp)
+        txt += "* completed at %s\n" % common.get_wiki_timestamp()
+        if buildserverid:
+            txt += ('* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
+                    .format(id=buildserverid))
+        if fdroidserverid:
+            txt += ('* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
+                    .format(id=fdroidserverid))
+        if os.cpu_count():
+            txt += "* host processors: %d\n" % os.cpu_count()
+        if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK):
+            with open('/proc/meminfo') as fp:
+                for line in fp:
+                    m = re.search(r'MemTotal:\s*([0-9].*)', line)
+                    if m:
+                        txt += "* host RAM: %s\n" % m.group(1)
+                        break
+        txt += "* successful builds: %d\n" % len(build_succeeded)
+        txt += "* failed builds: %d\n" % len(failed_apps)
+        txt += "\n\n"
+        newpage.save(txt, summary='Run log')
+        newpage = site.Pages['build']
+        newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+
     # hack to ensure this exits, even is some threads are still running
     sys.stdout.flush()
     sys.stderr.flush()