makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 182      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 184      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 186      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 188      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 189     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 190      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 192      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 194      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 196      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 198      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 199     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 200      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 201     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 202      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 204      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 206      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 208      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 210      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 212      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 213     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 214      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 215     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 216      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 217     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 218      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 220      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 222      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 224      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 225     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 226      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 228      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 230      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 232      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 233     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 234      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 236      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 238      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 240      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 241     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 242      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 243     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 244      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 246      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 248      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 249     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 250      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 251     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 252      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 253     ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
 254      'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
 255     ('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
 256      '5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
 257     # the binaries that Google uses are here:
 258     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 259     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 260      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 261     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 262      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 263     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 264      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 265     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 266      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 267     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 268      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 269     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 270      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 271     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 272      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 273     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 274      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 275     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 276      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 277     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 278      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 279     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 280      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 281     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 282      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 283     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 284      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 285     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 286      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 287     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 288      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 289     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 290      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 291     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 292      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 293     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 294      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 295     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 296      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 297     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 298      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 299     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 300      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 301     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 302      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 303     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 304      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 305     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 306      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 307     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 308      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 309     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 310      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 311     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 312      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 313     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 314      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 315     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 316      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 317     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 318      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 319     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 320      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 321     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 322      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 323     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 324      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 325     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 326      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 327     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 328      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 329     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 330      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 331     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 332      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 333     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 334      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 335     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 336      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 337     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 338      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 339     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 340      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 341     ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
 342      'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
 343     ('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
 344      'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
 345     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 346      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 347     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 348      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 349     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 350      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 351     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 352      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 353     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 354      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 355     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 356      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 357     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 358      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 359     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 360      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 361     ('https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip',
 362      'a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e'),
 363 ]
 364
 365
 366 def sha256_for_file(path):
 367     with open(path, 'rb') as f:
 368         s = hashlib.sha256()
 369         while True:
 370             data = f.read(4096)
 371             if not data:
 372                 break
 373             s.update(data)
 374         return s.hexdigest()
 375
 376
 377 def run_via_vagrant_ssh(v, cmdlist):
 378     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 379         cmd = cmdlist
 380     else:
 381         cmd = ' '.join(cmdlist)
 382     v._run_vagrant_command(['ssh', '-c', cmd])
 383
 384
 385 def update_cache(cachedir, cachefiles):
 386     for srcurl, shasum in cachefiles:
 387         filename = os.path.basename(srcurl)
 388         local_filename = os.path.join(cachedir, filename)
 389
 390         if os.path.exists(local_filename):
 391             local_length = os.path.getsize(local_filename)
 392         else:
 393             local_length = -1
 394
 395         resume_header = {}
 396         download = True
 397
 398         try:
 399             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 400             if r.status_code == 200:
 401                 content_length = int(r.headers.get('content-length'))
 402             else:
 403                 content_length = local_length  # skip the download
 404         except requests.exceptions.RequestException as e:
 405             content_length = local_length  # skip the download
 406             logger.warn('%s', e)
 407
 408         if local_length == content_length:
 409             download = False
 410         elif local_length > content_length:
 411             logger.info('deleting corrupt file from cache: %s', local_filename)
 412             os.remove(local_filename)
 413             logger.info("Downloading %s to cache", filename)
 414         elif local_length > -1 and local_length < content_length:
 415             logger.info("Resuming download of %s", local_filename)
 416             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 417         else:
 418             logger.info("Downloading %s to cache", filename)
 419
 420         if download:
 421             r = requests.get(srcurl, headers=resume_header,
 422                              stream=True, allow_redirects=True)
 423             content_length = int(r.headers.get('content-length'))
 424             with open(local_filename, 'ab') as f:
 425                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 426                                           expected_size=(content_length / 65536) + 1):
 427                     if chunk:  # filter out keep-alive new chunks
 428                         f.write(chunk)
 429
 430         v = sha256_for_file(local_filename)
 431         if v == shasum:
 432             logger.info("\t...shasum verified for %s", local_filename)
 433         else:
 434             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 435             os.remove(local_filename)
 436             sys.exit(1)
 437
 438
 439 def debug_log_vagrant_vm(vm_dir, config):
 440     if options.verbosity >= 3:
 441         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 442         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 443         logger.debug('> vagrant status')
 444         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 445         logger.debug('> vagrant box list')
 446         subprocess.call(['vagrant', 'box', 'list'])
 447         if config['vm_provider'] == 'libvirt':
 448             logger.debug('> virsh -c qmeu:///system list --all')
 449             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 450             domain = 'buildserver_default'
 451             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 452             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 453
 454
 455 def main():
 456     global cachedir, cachefiles, config, tail
 457
 458     if options.skip_cache_update:
 459         logger.info('skipping cache update and verification...')
 460     else:
 461         update_cache(cachedir, cachefiles)
 462
 463     # use VirtualBox software virtualization if hardware is not available,
 464     # like if this is being run in kvm or some other VM platform, like
 465     # http://jenkins.debian.net, the values are 'on' or 'off'
 466     if sys.platform.startswith('darwin'):
 467         # all < 10 year old Macs work, and OSX servers as VM host are very
 468         # rare, but this could also be auto-detected if someone codes it
 469         config['hwvirtex'] = 'on'
 470         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 471     elif os.path.exists('/proc/cpuinfo'):
 472         with open('/proc/cpuinfo') as f:
 473             contents = f.read()
 474         if 'vmx' in contents or 'svm' in contents:
 475             config['hwvirtex'] = 'on'
 476             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 477
 478     serverdir = os.path.join(os.getcwd(), 'buildserver')
 479     logfilename = os.path.join(serverdir, 'up.log')
 480     if not os.path.exists(logfilename):
 481         open(logfilename, 'a').close()  # create blank file
 482     log_cm = vagrant.make_file_cm(logfilename)
 483     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 484
 485     if options.verbosity >= 2:
 486         tail = fdroidserver.tail.Tail(logfilename)
 487         tail.start()
 488
 489     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 490     if options.clean:
 491         vm.destroy()
 492
 493     # Check against the existing Vagrantfile.yaml, and if they differ, we
 494     # need to create a new box:
 495     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 496     writevf = True
 497     if os.path.exists(vf):
 498         logger.info('Halting %s', serverdir)
 499         v.halt()
 500         with open(vf, 'r', encoding='utf-8') as f:
 501             oldconfig = yaml.load(f)
 502         if config != oldconfig:
 503             logger.info("Server configuration has changed, rebuild from scratch is required")
 504             vm.destroy()
 505         else:
 506             logger.info("Re-provisioning existing server")
 507             writevf = False
 508     else:
 509         logger.info("No existing server - building from scratch")
 510     if writevf:
 511         with open(vf, 'w', encoding='utf-8') as f:
 512             yaml.dump(config, f)
 513
 514     if config['vm_provider'] == 'libvirt':
 515         found_basebox = False
 516         needs_mutate = False
 517         for box in v.box_list():
 518             if box.name == config['basebox']:
 519                 found_basebox = True
 520                 if box.provider != 'libvirt':
 521                     needs_mutate = True
 522                 continue
 523         if not found_basebox:
 524             if isinstance(config['baseboxurl'], str):
 525                 baseboxurl = config['baseboxurl']
 526             else:
 527                 baseboxurl = config['baseboxurl'][0]
 528             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 529             v.box_add(config['basebox'], baseboxurl)
 530             needs_mutate = True
 531         if needs_mutate:
 532             logger.info('Converting %s to libvirt format', config['basebox'])
 533             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 534             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 535             v.box_remove(config['basebox'], 'virtualbox')
 536
 537     logger.info("Configuring build server VM")
 538     debug_log_vagrant_vm(serverdir, config)
 539     try:
 540         v.up(provision=True)
 541     except subprocess.CalledProcessError:
 542         debug_log_vagrant_vm(serverdir, config)
 543         logger.error("'vagrant up' failed, is the base box missing?")
 544         sys.exit(1)
 545
 546     if config['copy_caches_from_host']:
 547         ssh_config = v.ssh_config()
 548         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 549         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 550         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 551         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 552
 553         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 554             fullpath = os.path.join(os.getenv('HOME'), d)
 555             if os.path.isdir(fullpath):
 556                 ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
 557                                         '-o StrictHostKeyChecking=no',
 558                                         '-o UserKnownHostsFile=/dev/null',
 559                                         '-o LogLevel=FATAL',
 560                                         '-o IdentitiesOnly=yes',
 561                                         '-o PasswordAuthentication=no'))
 562                 # TODO vagrant 1.5+ provides `vagrant rsync`
 563                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 564                 subprocess.call(['rsync', '-ax', '--delete', '-e',
 565                                  ssh_command,
 566                                  fullpath + '/',
 567                                  user + '@' + hostname + ':~/' + d + '/'])
 568
 569         # this file changes every time but should not be cached
 570         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 571         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 572
 573     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 574                          universal_newlines=True)
 575     buildserverid = p.communicate()[0].strip()
 576     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 577     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 578
 579     logger.info("Stopping build server VM")
 580     v.halt()
 581
 582     logger.info("Packaging")
 583     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 584     if os.path.exists(boxfile):
 585         os.remove(boxfile)
 586
 587     vm.package(output=boxfile)
 588
 589     logger.info("Adding box")
 590     vm.box_add('buildserver', boxfile, force=True)
 591
 592     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 593         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 594         sys.exit(1)
 595
 596     if not options.keep_box_file:
 597         logger.debug('box added to vagrant, ' +
 598                      'removing generated box file \'%s\'',
 599                      boxfile)
 600         os.remove(boxfile)
 601
 602
 603 if __name__ == '__main__':
 604     try:
 605         main()
 606     finally:
 607         if tail is not None:
 608             tail.stop()