test ./makebuildserver by running a build in jenkins

[fdroidserver.git] / makebuildserver

diff --git a/makebuildserver b/makebuildserver
index 5a8aa6e4ab870cdcbd4060b5162ad5fad0a435ed..0ec3eebed45101cba12c92df93fa302ff657b4f5 100755 (executable)
--- a/makebuildserver
+++ b/makebuildserver
@@ -41,12 +41,32 @@ parser.add_option("-v", "--verbose", action="store_true", default=False,
                   help="Spew out even more information than normal")
 parser.add_option("-c", "--clean", action="store_true", default=False,
                   help="Build from scratch, rather than attempting to update the existing server")
-parser.add_option("--debian-mirror", default="http://ftp.uk.debian.org/debian/",
-                  help="Use the specified Debian mirror in the box's /etc/apt/sources.list.")
 options, args = parser.parse_args()
 
-config = {}
-execfile('makebs.config.py', config)
+# set up default config
+cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
+config = {
+    'arch64': False,
+    'basebox': 'jessie32',
+    # TODO in py3, convert this to pathlib.Path(absolute_path_string).as_uri()
+    'baseboxurl': [
+        'file://' + os.path.join(cachedir, 'jessie32.box'),
+        'https://f-droid.org/jessie32.box',
+    ],
+    'debian_mirror': 'http://http.debian.net/debian/',
+    'apt_package_cache': False,
+    'boot_timeout': 600,
+    'cachedir': cachedir,
+    'cpus': 1,
+    'memory': 1024,
+}
+
+# load config file, if present
+if os.path.exists('makebuildserver.config.py'):
+    execfile('makebuildserver.config.py', config)
+elif os.path.exists('makebs.config.py'):
+    # this is the old name for the config file
+    execfile('makebs.config.py', config)
 
 if not os.path.exists('makebuildserver') or not os.path.exists(serverdir):
     print 'This must be run from the correct directory!'
@@ -59,9 +79,9 @@ if options.clean:
     vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose)
 
 # Update cached files.
-cachedir = os.path.join('buildserver', 'cache')
+cachedir = config['cachedir']
 if not os.path.exists(cachedir):
-    os.mkdir(cachedir)
+    os.makedirs(cachedir, 0755)
 
 cachefiles = [
     ('android-sdk_r24.4.1-linux.tgz',
@@ -193,6 +213,8 @@ cachefiles = [
     ('build-tools-23.0.2.zip',
      'https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
+    # the binaries that Google uses are here:
+    # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
     ('gradle-1.4-bin.zip',
      'https://services.gradle.org/distributions/gradle-1.4-bin.zip',
      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
@@ -241,6 +263,12 @@ cachefiles = [
     ('gradle-2.8-bin.zip',
      'https://services.gradle.org/distributions/gradle-2.8-bin.zip',
      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
+    ('gradle-2.9-bin.zip',
+     'https://services.gradle.org/distributions/gradle-2.9-bin.zip',
+     'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
+    ('gradle-2.10-bin.zip',
+     'https://services.gradle.org/distributions/gradle-2.10-bin.zip',
+     '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
     ('Kivy-1.7.2.tar.gz',
      'https://pypi.python.org/packages/source/K/Kivy/Kivy-1.7.2.tar.gz',
      '0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1'),
@@ -283,6 +311,9 @@ def sha256_for_file(path):
 
 for f, src, shasum in cachefiles:
     relpath = os.path.join(cachedir, f)
+    # if download fails to connect, it'll make a zero size file
+    if os.path.exists(relpath) and os.stat(relpath).st_size == 0:
+        os.remove(relpath)
     if not os.path.exists(relpath):
         print "Downloading " + f + " to cache"
         if subprocess.call(['wget', src, '-O', f], cwd=cachedir) != 0:
@@ -324,16 +355,34 @@ Vagrant.configure("2") do |config|
     v.customize ["modifyvm", :id, "--cpus", "{3}"]
   end
 
+  config.vm.boot_timeout = {4}
+
   config.vm.provision :shell, :path => "fixpaths.sh"
 """.format(config['basebox'],
            baseboxurl,
            config['memory'],
-           config.get('cpus', 1))
+           config.get('cpus', 1),
+           config['boot_timeout'])
 if 'aptproxy' in config and config['aptproxy']:
     vagrantfile += """
   config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
 """.format(config['aptproxy'])
 
+# buildserver/ is shared to the VM's /vagrant by default so the old default
+# does not need a custom mount
+if cachedir != 'buildserver/cache':
+    vagrantfile += """
+  config.vm.synced_folder '{0}', '/vagrant/cache'
+""".format(cachedir)
+
+# cache .deb packages on the host via a mount trick
+if config['apt_package_cache']:
+    aptcachedir = cachedir + '/apt/archives'
+    vagrantfile += """
+  config.vm.synced_folder "{0}", "/var/cache/apt/archives",
+    owner: 'root', group: 'root', create: true
+""".format(aptcachedir)
+
 vagrantfile += """
   config.vm.provision :chef_solo do |chef|
     chef.cookbooks_path = "cookbooks"
@@ -353,7 +402,7 @@ vagrantfile += """
     chef.add_recipe "kivy"
   end
 end
-""" % (options.debian_mirror)
+""" % (config['debian_mirror'])
 
 # Check against the existing Vagrantfile, and if they differ, we need to
 # create a new box: