chiark / gitweb /
~ianmdlvl / fdroidserver.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
Weblate
[fdroidserver.git] / makebuildserver
diff --git a/makebuildserver b/makebuildserver
index 2e2b1a7b1b5bf094531936f56822f9d562203d68..cb46035e834591b2b7053d302afc6dc1ad9c8210 100755 (executable)
--- a/makebuildserver
+++ b/makebuildserver
@@ -13,7 +13,6 @@ import hashlib
 import yaml
 import json
 import logging
-import textwrap
 from clint.textui import progress
 from optparse import OptionParser
 import fdroidserver.tail
@@ -30,7 +29,8 @@ parser.add_option('--skip-cache-update', action="store_true", default=False,
                   help="""Skip downloading and checking cache."""
                        """This assumes that the cache is already downloaded completely.""")
 parser.add_option('--keep-box-file', action="store_true", default=False,
-                  help="""Box file will not be deleted after adding it to box storage.""")
+                  help="""Box file will not be deleted after adding it to box storage"""
+                       """ (KVM-only).""")
 options, args = parser.parse_args()
 
 logger = logging.getLogger('fdroidserver-makebuildserver')
@@ -58,8 +58,7 @@ logger.debug('cachedir set to: %s', cachedir)
 config = {
     'basebox': 'jessie64',
     'baseboxurl': [
-        pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
-        'https://f-droid.org/jessie64.box',
+        pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
     ],
     'debian_mirror': 'http://http.debian.net/debian/',
     'apt_package_cache': False,
@@ -67,7 +66,7 @@ config = {
     'boot_timeout': 600,
     'cachedir': cachedir,
     'cpus': 1,
-    'memory': 1024,
+    'memory': 2048,
     'hwvirtex': 'off',
     'vm_provider': 'virtualbox',
 }
@@ -80,7 +79,6 @@ if os.path.isfile('/usr/bin/systemd-detect-virt'):
     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
         config['vm_provider'] = 'libvirt'
-        config['domain'] = 'buildserver_default'
     elif virt != 'none':
         logger.info('Running in an unsupported VM guest (%s)!', virt)
     logger.debug('detected virt: %s', virt)
@@ -125,8 +123,11 @@ if config['apt_package_cache']:
         shutil.rmtree(aptcachepartial)
 
 cachefiles = [
-    ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
-     '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
+    # Don't update sdk tools beyond 25.2.5.
+    # Support for android update project has been removed and there is no replacement.
+    # Until we find a solution for that we need to stay at this revision.
+    ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
+     '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
@@ -173,8 +174,10 @@ cachefiles = [
      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
     ('https://dl.google.com/android/repository/platform-24_r02.zip',
      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
-    ('https://dl.google.com/android/repository/platform-25_r01.zip',
-     'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
+    ('https://dl.google.com/android/repository/platform-25_r03.zip',
+     '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
+    ('https://dl.google.com/android/repository/platform-26_r02.zip',
+     '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
@@ -233,6 +236,22 @@ cachefiles = [
      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
+    ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
+     '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
+    ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
+     '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
+    ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
+     'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
+    ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
+     'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
+    ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
+     '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
+    ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
+     '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
+    ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
+     '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
+    ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
+     'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
     # the binaries that Google uses are here:
     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
@@ -299,6 +318,26 @@ cachefiles = [
      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
+    ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
+     '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
+    ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
+     '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
+    ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
+     'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
+    ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
+     '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
+    ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
+     'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
+    ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
+     '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
+    ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
+     'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
+    ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
+     '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
+    ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
+     '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
+    ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
+     'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
@@ -311,8 +350,12 @@ cachefiles = [
      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
-    ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
-     '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
+    ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
+     '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
+    ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
+     'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
+    ('https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip',
+     'a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e'),
     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 ]
@@ -374,7 +417,7 @@ def update_cache(cachedir, cachefiles):
 
         if download:
             r = requests.get(srcurl, headers=resume_header,
-                             stream=True, verify=False, allow_redirects=True)
+                             stream=True, allow_redirects=True)
             content_length = int(r.headers.get('content-length'))
             with open(local_filename, 'ab') as f:
                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
@@ -402,8 +445,9 @@ def debug_log_vagrant_vm(vm_dir, config):
         if config['vm_provider'] == 'libvirt':
             logger.debug('> virsh -c qmeu:///system list --all')
             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
-            logger.debug('> virsh -c qemu:///system snapshot-list %s', config['domain'])
-            subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', config['domain']])
+            domain = 'buildserver_default'
+            logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
+            subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 
 
 def main():
@@ -496,9 +540,9 @@ def main():
     debug_log_vagrant_vm(serverdir, config)
     try:
         v.up(provision=True)
-    except fdroidserver.vmtools.FDroidBuildVmException as e:
+    except subprocess.CalledProcessError:
         debug_log_vagrant_vm(serverdir, config)
-        logger.exception('could not bring buildserver vm up. %s', e)
+        logger.error("'vagrant up' failed, is the base box missing?")
         sys.exit(1)
 
     if config['copy_caches_from_host']:
@@ -511,10 +555,16 @@ def main():
         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
             fullpath = os.path.join(os.getenv('HOME'), d)
             if os.path.isdir(fullpath):
-                # TODO newer versions of vagrant provide `vagrant rsync`
+                ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
+                                        '-o StrictHostKeyChecking=no',
+                                        '-o UserKnownHostsFile=/dev/null',
+                                        '-o LogLevel=FATAL',
+                                        '-o IdentitiesOnly=yes',
+                                        '-o PasswordAuthentication=no'))
+                # TODO vagrant 1.5+ provides `vagrant rsync`
                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
-                subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
-                                 'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
+                subprocess.call(['rsync', '-ax', '--delete', '-e',
+                                 ssh_command,
                                  fullpath + '/',
                                  user + '@' + hostname + ':~/' + d + '/'])
 
@@ -536,22 +586,7 @@ def main():
     if os.path.exists(boxfile):
         os.remove(boxfile)
 
-    vagrantfile = textwrap.dedent("""\
-                  Vagrant.configure("2") do |config|
-                    config.ssh.username = "vagrant"
-                    config.ssh.password = "vagrant"
-
-                    config.vm.provider :libvirt do |libvirt|
-
-                      libvirt.driver = "kvm"
-                      libvirt.host = ""
-                      libvirt.connect_via_ssh = false
-                      libvirt.storage_pool_name = "default"
-
-                    end
-                  end""")
-
-    vm.package(output=boxfile, vagrantfile=vagrantfile, keep_box_file=options.keep_box_file)
+    vm.package(output=boxfile)
 
     logger.info("Adding box")
     vm.box_add('buildserver', boxfile, force=True)
Unnamed repository; edit this file 'description' to name the repository.