makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/platform-27_r01.zip',
 182      'cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac'),
 183     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 184      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 186      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 188      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 189     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 190      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 191     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 192      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 194      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 196      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 198      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 199     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 200      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 201     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 202      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 203     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 204      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 206      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 208      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 210      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 212      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 213     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 214      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 215     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 216      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 217     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 218      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 219     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 220      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 222      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 224      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 225     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 226      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 227     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 228      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 230      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 232      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 233     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 234      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 235     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 236      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 238      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 240      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 241     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 242      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 243     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 244      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 246      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 248      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 249     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 250      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 251     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 252      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 253     # the binaries that Google uses are here:
 254     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 255     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 256      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 257     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 258      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 259     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 260      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 261     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 262      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 263     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 264      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 265     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 266      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 267     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 268      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 269     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 270      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 271     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 272      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 273     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 274      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 275     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 276      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 277     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 278      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 279     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 280      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 281     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 282      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 283     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 284      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 285     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 286      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 287     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 288      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 289     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 290      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 291     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 292      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 293     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 294      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 295     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 296      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 297     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 298      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 299     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 300      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 301     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 302      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 303     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 304      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 305     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 306      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 307     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 308      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 309     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 310      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 311     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 312      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 313     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 314      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 315     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 316      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 317     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 318      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 319     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 320      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 321     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 322      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 323     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 324      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 325     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 326      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 327     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 328      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 329     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 330      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 331     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 332      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 333     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 334      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 335     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 336      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 337     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 338      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 339     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 340      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 341     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 342      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 343     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 344      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 345     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 346      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 347     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 348      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 349     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 350      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 351     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 352      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 353 ]
 354
 355
 356 def sha256_for_file(path):
 357     with open(path, 'rb') as f:
 358         s = hashlib.sha256()
 359         while True:
 360             data = f.read(4096)
 361             if not data:
 362                 break
 363             s.update(data)
 364         return s.hexdigest()
 365
 366
 367 def run_via_vagrant_ssh(v, cmdlist):
 368     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 369         cmd = cmdlist
 370     else:
 371         cmd = ' '.join(cmdlist)
 372     v._run_vagrant_command(['ssh', '-c', cmd])
 373
 374
 375 def update_cache(cachedir, cachefiles):
 376     for srcurl, shasum in cachefiles:
 377         filename = os.path.basename(srcurl)
 378         local_filename = os.path.join(cachedir, filename)
 379
 380         if os.path.exists(local_filename):
 381             local_length = os.path.getsize(local_filename)
 382         else:
 383             local_length = -1
 384
 385         resume_header = {}
 386         download = True
 387
 388         try:
 389             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 390             if r.status_code == 200:
 391                 content_length = int(r.headers.get('content-length'))
 392             else:
 393                 content_length = local_length  # skip the download
 394         except requests.exceptions.RequestException as e:
 395             content_length = local_length  # skip the download
 396             logger.warn('%s', e)
 397
 398         if local_length == content_length:
 399             download = False
 400         elif local_length > content_length:
 401             logger.info('deleting corrupt file from cache: %s', local_filename)
 402             os.remove(local_filename)
 403             logger.info("Downloading %s to cache", filename)
 404         elif local_length > -1 and local_length < content_length:
 405             logger.info("Resuming download of %s", local_filename)
 406             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 407         else:
 408             logger.info("Downloading %s to cache", filename)
 409
 410         if download:
 411             r = requests.get(srcurl, headers=resume_header,
 412                              stream=True, allow_redirects=True)
 413             content_length = int(r.headers.get('content-length'))
 414             with open(local_filename, 'ab') as f:
 415                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 416                                           expected_size=(content_length / 65536) + 1):
 417                     if chunk:  # filter out keep-alive new chunks
 418                         f.write(chunk)
 419
 420         v = sha256_for_file(local_filename)
 421         if v == shasum:
 422             logger.info("\t...shasum verified for %s", local_filename)
 423         else:
 424             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 425             os.remove(local_filename)
 426             sys.exit(1)
 427
 428
 429 def debug_log_vagrant_vm(vm_dir, config):
 430     if options.verbosity >= 3:
 431         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 432         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 433         logger.debug('> vagrant status')
 434         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 435         logger.debug('> vagrant box list')
 436         subprocess.call(['vagrant', 'box', 'list'])
 437         if config['vm_provider'] == 'libvirt':
 438             logger.debug('> virsh -c qmeu:///system list --all')
 439             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 440             domain = 'buildserver_default'
 441             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 442             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 443
 444
 445 def main():
 446     global cachedir, cachefiles, config, tail
 447
 448     if options.skip_cache_update:
 449         logger.info('skipping cache update and verification...')
 450     else:
 451         update_cache(cachedir, cachefiles)
 452
 453     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 454     logger.info("Setting executable bit for %s", local_qt_filename)
 455     os.chmod(local_qt_filename, 0o755)
 456
 457     # use VirtualBox software virtualization if hardware is not available,
 458     # like if this is being run in kvm or some other VM platform, like
 459     # http://jenkins.debian.net, the values are 'on' or 'off'
 460     if sys.platform.startswith('darwin'):
 461         # all < 10 year old Macs work, and OSX servers as VM host are very
 462         # rare, but this could also be auto-detected if someone codes it
 463         config['hwvirtex'] = 'on'
 464         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 465     elif os.path.exists('/proc/cpuinfo'):
 466         with open('/proc/cpuinfo') as f:
 467             contents = f.read()
 468         if 'vmx' in contents or 'svm' in contents:
 469             config['hwvirtex'] = 'on'
 470             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 471
 472     serverdir = os.path.join(os.getcwd(), 'buildserver')
 473     logfilename = os.path.join(serverdir, 'up.log')
 474     if not os.path.exists(logfilename):
 475         open(logfilename, 'a').close()  # create blank file
 476     log_cm = vagrant.make_file_cm(logfilename)
 477     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 478
 479     if options.verbosity >= 2:
 480         tail = fdroidserver.tail.Tail(logfilename)
 481         tail.start()
 482
 483     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 484     if options.clean:
 485         vm.destroy()
 486
 487     # Check against the existing Vagrantfile.yaml, and if they differ, we
 488     # need to create a new box:
 489     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 490     writevf = True
 491     if os.path.exists(vf):
 492         logger.info('Halting %s', serverdir)
 493         v.halt()
 494         with open(vf, 'r', encoding='utf-8') as f:
 495             oldconfig = yaml.load(f)
 496         if config != oldconfig:
 497             logger.info("Server configuration has changed, rebuild from scratch is required")
 498             vm.destroy()
 499         else:
 500             logger.info("Re-provisioning existing server")
 501             writevf = False
 502     else:
 503         logger.info("No existing server - building from scratch")
 504     if writevf:
 505         with open(vf, 'w', encoding='utf-8') as f:
 506             yaml.dump(config, f)
 507
 508     if config['vm_provider'] == 'libvirt':
 509         found_basebox = False
 510         needs_mutate = False
 511         for box in v.box_list():
 512             if box.name == config['basebox']:
 513                 found_basebox = True
 514                 if box.provider != 'libvirt':
 515                     needs_mutate = True
 516                 continue
 517         if not found_basebox:
 518             if isinstance(config['baseboxurl'], str):
 519                 baseboxurl = config['baseboxurl']
 520             else:
 521                 baseboxurl = config['baseboxurl'][0]
 522             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 523             v.box_add(config['basebox'], baseboxurl)
 524             needs_mutate = True
 525         if needs_mutate:
 526             logger.info('Converting %s to libvirt format', config['basebox'])
 527             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 528             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 529             v.box_remove(config['basebox'], 'virtualbox')
 530
 531     logger.info("Configuring build server VM")
 532     debug_log_vagrant_vm(serverdir, config)
 533     try:
 534         v.up(provision=True)
 535     except subprocess.CalledProcessError:
 536         debug_log_vagrant_vm(serverdir, config)
 537         logger.error("'vagrant up' failed, is the base box missing?")
 538         sys.exit(1)
 539
 540     if config['copy_caches_from_host']:
 541         ssh_config = v.ssh_config()
 542         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 543         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 544         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 545         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 546
 547         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 548             fullpath = os.path.join(os.getenv('HOME'), d)
 549             if os.path.isdir(fullpath):
 550                 # TODO newer versions of vagrant provide `vagrant rsync`
 551                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 552                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 553                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 554                                  fullpath + '/',
 555                                  user + '@' + hostname + ':~/' + d + '/'])
 556
 557         # this file changes every time but should not be cached
 558         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 559         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 560
 561     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 562                          universal_newlines=True)
 563     buildserverid = p.communicate()[0].strip()
 564     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 565     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 566
 567     logger.info("Stopping build server VM")
 568     v.halt()
 569
 570     logger.info("Packaging")
 571     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 572     if os.path.exists(boxfile):
 573         os.remove(boxfile)
 574
 575     vm.package(output=boxfile)
 576
 577     logger.info("Adding box")
 578     vm.box_add('buildserver', boxfile, force=True)
 579
 580     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 581         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 582         sys.exit(1)
 583
 584     if not options.keep_box_file:
 585         logger.debug('box added to vagrant, ' +
 586                      'removing generated box file \'%s\'',
 587                      boxfile)
 588         os.remove(boxfile)
 589
 590
 591 if __name__ == '__main__':
 592     try:
 593         main()
 594     finally:
 595         if tail is not None:
 596             tail.stop()