buildserver: add copy_caches_from_host config option

For people using slow, expensive, and/or flaky internet, liberal use of
caching can make a huge difference.  The restricted environment of the
gpjenkins box has been a good test environment for this (Tor-only,
whitelist of allowed IPs to visit, home internet connection).

diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py
index b6c0183d013893689f4b79658626247d1c537ab2..7015cf5814acd0872d3d161c6d702f2f642254a5 100644 (file)
--- a/examples/makebuildserver.config.py
+++ b/examples/makebuildserver.config.py
@@ -28,6 +28,16 @@
 #
 # apt_package_cache = True
 
+# The buildserver can use some local caches to speed up builds,
+# especially when the internet connection is slow and/or expensive.
+# If enabled, the buildserver setup will look for standard caches in
+# your HOME dir and copy them to the buildserver VM. Be aware: this
+# will reduce the isolation of the buildserver from your host machine,
+# so the buildserver will provide an environment only as trustworthy
+# as the host machine's environment.
+#
+# copy_caches_from_host = True
+
 # To specify which Debian mirror the build server VM should use, by
 # default it uses http.debian.net, which auto-detects which is the
 # best mirror to use.

diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver
index ae3391a09c90a9c75f6b74e01386f57f794fbe5f..a2923ad48451ca10ac2766aaf4f8beb9b099240a 100755 (executable)
--- a/jenkins-build-makebuildserver
+++ b/jenkins-build-makebuildserver
@@ -49,6 +49,7 @@ cd $WORKSPACE
 echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
 echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
 echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
+echo "copy_caches_from_host = True" >> $WORKSPACE/makebuildserver.config.py
 ./makebuildserver --verbose --clean
 
 # this can be handled in the jenkins job, or here:

diff --git a/makebuildserver b/makebuildserver
index 6578e27c2d1f912399fb4f1fbc390c2cfece6b59..4985b22fbc81bbfb5a16c93b31a80ea0ea102b98 100755 (executable)
--- a/makebuildserver
+++ b/makebuildserver
@@ -2,6 +2,7 @@
 
 import os
 import pathlib
+import re
 import requests
 import stat
 import sys
@@ -39,6 +40,7 @@ config = {
     ],
     'debian_mirror': 'http://http.debian.net/debian/',
     'apt_package_cache': False,
+    'copy_caches_from_host': False,
     'boot_timeout': 600,
     'cachedir': cachedir,
     'cpus': 1,
@@ -437,6 +439,27 @@ def main():
     print("Configuring build server VM")
     v.up(provision=True)
 
+    if config['copy_caches_from_host']:
+        ssh_config = v.ssh_config()
+        user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
+        hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
+        port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
+        key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
+
+        for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
+            fullpath = os.path.join(os.getenv('HOME'), d)
+            if os.path.isdir(fullpath):
+                # TODO newer versions of vagrant provide `vagrant rsync`
+                run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
+                subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
+                                 'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
+                                 fullpath + '/',
+                                 user + '@' + hostname + ':~/' + d + '/'])
+
+        # this file changes every time but should not be cached
+        run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
+        run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
+
     print("Writing buildserver ID")
     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
                          universal_newlines=True)