makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 182      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 184      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 186      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 188      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 189     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 190      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 192      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 194      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 196      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 198      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 199     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 200      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 201     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 202      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 204      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 206      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 208      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 210      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 212      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 213     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 214      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 215     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 216      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 217     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 218      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 220      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 222      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 224      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 225     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 226      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 228      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 230      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 232      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 233     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 234      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 236      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 238      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 240      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 241     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 242      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 243     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 244      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 246      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 248      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 249     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 250      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 251     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 252      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 253     ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
 254      'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
 255     # the binaries that Google uses are here:
 256     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 257     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 258      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 259     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 260      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 261     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 262      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 263     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 264      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 265     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 266      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 267     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 268      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 269     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 270      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 271     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 272      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 273     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 274      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 275     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 276      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 277     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 278      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 279     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 280      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 281     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 282      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 283     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 284      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 285     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 286      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 287     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 288      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 289     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 290      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 291     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 292      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 293     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 294      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 295     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 296      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 297     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 298      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 299     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 300      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 301     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 302      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 303     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 304      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 305     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 306      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 307     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 308      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 309     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 310      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 311     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 312      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 313     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 314      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 315     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 316      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 317     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 318      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 319     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 320      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 321     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 322      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 323     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 324      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 325     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 326      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 327     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 328      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 329     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 330      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 331     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 332      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 333     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 334      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 335     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 336      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 337     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 338      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 339     ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
 340      'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
 341     ('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
 342      'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
 343     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 344      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 345     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 346      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 347     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 348      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 349     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 350      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 351     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 352      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 353     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 354      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 355     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 356      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 357     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 358      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 359     ('https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip',
 360      'a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e'),
 361     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 362      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 363 ]
 364
 365
 366 def sha256_for_file(path):
 367     with open(path, 'rb') as f:
 368         s = hashlib.sha256()
 369         while True:
 370             data = f.read(4096)
 371             if not data:
 372                 break
 373             s.update(data)
 374         return s.hexdigest()
 375
 376
 377 def run_via_vagrant_ssh(v, cmdlist):
 378     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 379         cmd = cmdlist
 380     else:
 381         cmd = ' '.join(cmdlist)
 382     v._run_vagrant_command(['ssh', '-c', cmd])
 383
 384
 385 def update_cache(cachedir, cachefiles):
 386     for srcurl, shasum in cachefiles:
 387         filename = os.path.basename(srcurl)
 388         local_filename = os.path.join(cachedir, filename)
 389
 390         if os.path.exists(local_filename):
 391             local_length = os.path.getsize(local_filename)
 392         else:
 393             local_length = -1
 394
 395         resume_header = {}
 396         download = True
 397
 398         try:
 399             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 400             if r.status_code == 200:
 401                 content_length = int(r.headers.get('content-length'))
 402             else:
 403                 content_length = local_length  # skip the download
 404         except requests.exceptions.RequestException as e:
 405             content_length = local_length  # skip the download
 406             logger.warn('%s', e)
 407
 408         if local_length == content_length:
 409             download = False
 410         elif local_length > content_length:
 411             logger.info('deleting corrupt file from cache: %s', local_filename)
 412             os.remove(local_filename)
 413             logger.info("Downloading %s to cache", filename)
 414         elif local_length > -1 and local_length < content_length:
 415             logger.info("Resuming download of %s", local_filename)
 416             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 417         else:
 418             logger.info("Downloading %s to cache", filename)
 419
 420         if download:
 421             r = requests.get(srcurl, headers=resume_header,
 422                              stream=True, allow_redirects=True)
 423             content_length = int(r.headers.get('content-length'))
 424             with open(local_filename, 'ab') as f:
 425                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 426                                           expected_size=(content_length / 65536) + 1):
 427                     if chunk:  # filter out keep-alive new chunks
 428                         f.write(chunk)
 429
 430         v = sha256_for_file(local_filename)
 431         if v == shasum:
 432             logger.info("\t...shasum verified for %s", local_filename)
 433         else:
 434             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 435             os.remove(local_filename)
 436             sys.exit(1)
 437
 438
 439 def debug_log_vagrant_vm(vm_dir, config):
 440     if options.verbosity >= 3:
 441         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 442         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 443         logger.debug('> vagrant status')
 444         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 445         logger.debug('> vagrant box list')
 446         subprocess.call(['vagrant', 'box', 'list'])
 447         if config['vm_provider'] == 'libvirt':
 448             logger.debug('> virsh -c qmeu:///system list --all')
 449             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 450             domain = 'buildserver_default'
 451             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 452             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 453
 454
 455 def main():
 456     global cachedir, cachefiles, config, tail
 457
 458     if options.skip_cache_update:
 459         logger.info('skipping cache update and verification...')
 460     else:
 461         update_cache(cachedir, cachefiles)
 462
 463     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 464     logger.info("Setting executable bit for %s", local_qt_filename)
 465     os.chmod(local_qt_filename, 0o755)
 466
 467     # use VirtualBox software virtualization if hardware is not available,
 468     # like if this is being run in kvm or some other VM platform, like
 469     # http://jenkins.debian.net, the values are 'on' or 'off'
 470     if sys.platform.startswith('darwin'):
 471         # all < 10 year old Macs work, and OSX servers as VM host are very
 472         # rare, but this could also be auto-detected if someone codes it
 473         config['hwvirtex'] = 'on'
 474         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 475     elif os.path.exists('/proc/cpuinfo'):
 476         with open('/proc/cpuinfo') as f:
 477             contents = f.read()
 478         if 'vmx' in contents or 'svm' in contents:
 479             config['hwvirtex'] = 'on'
 480             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 481
 482     serverdir = os.path.join(os.getcwd(), 'buildserver')
 483     logfilename = os.path.join(serverdir, 'up.log')
 484     if not os.path.exists(logfilename):
 485         open(logfilename, 'a').close()  # create blank file
 486     log_cm = vagrant.make_file_cm(logfilename)
 487     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 488
 489     if options.verbosity >= 2:
 490         tail = fdroidserver.tail.Tail(logfilename)
 491         tail.start()
 492
 493     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 494     if options.clean:
 495         vm.destroy()
 496
 497     # Check against the existing Vagrantfile.yaml, and if they differ, we
 498     # need to create a new box:
 499     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 500     writevf = True
 501     if os.path.exists(vf):
 502         logger.info('Halting %s', serverdir)
 503         v.halt()
 504         with open(vf, 'r', encoding='utf-8') as f:
 505             oldconfig = yaml.load(f)
 506         if config != oldconfig:
 507             logger.info("Server configuration has changed, rebuild from scratch is required")
 508             vm.destroy()
 509         else:
 510             logger.info("Re-provisioning existing server")
 511             writevf = False
 512     else:
 513         logger.info("No existing server - building from scratch")
 514     if writevf:
 515         with open(vf, 'w', encoding='utf-8') as f:
 516             yaml.dump(config, f)
 517
 518     if config['vm_provider'] == 'libvirt':
 519         found_basebox = False
 520         needs_mutate = False
 521         for box in v.box_list():
 522             if box.name == config['basebox']:
 523                 found_basebox = True
 524                 if box.provider != 'libvirt':
 525                     needs_mutate = True
 526                 continue
 527         if not found_basebox:
 528             if isinstance(config['baseboxurl'], str):
 529                 baseboxurl = config['baseboxurl']
 530             else:
 531                 baseboxurl = config['baseboxurl'][0]
 532             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 533             v.box_add(config['basebox'], baseboxurl)
 534             needs_mutate = True
 535         if needs_mutate:
 536             logger.info('Converting %s to libvirt format', config['basebox'])
 537             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 538             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 539             v.box_remove(config['basebox'], 'virtualbox')
 540
 541     logger.info("Configuring build server VM")
 542     debug_log_vagrant_vm(serverdir, config)
 543     try:
 544         v.up(provision=True)
 545     except subprocess.CalledProcessError:
 546         debug_log_vagrant_vm(serverdir, config)
 547         logger.error("'vagrant up' failed, is the base box missing?")
 548         sys.exit(1)
 549
 550     if config['copy_caches_from_host']:
 551         ssh_config = v.ssh_config()
 552         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 553         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 554         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 555         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 556
 557         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 558             fullpath = os.path.join(os.getenv('HOME'), d)
 559             if os.path.isdir(fullpath):
 560                 ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
 561                                         '-o StrictHostKeyChecking=no',
 562                                         '-o UserKnownHostsFile=/dev/null',
 563                                         '-o LogLevel=FATAL',
 564                                         '-o IdentitiesOnly=yes',
 565                                         '-o PasswordAuthentication=no'))
 566                 # TODO vagrant 1.5+ provides `vagrant rsync`
 567                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 568                 subprocess.call(['rsync', '-ax', '--delete', '-e',
 569                                  ssh_command,
 570                                  fullpath + '/',
 571                                  user + '@' + hostname + ':~/' + d + '/'])
 572
 573         # this file changes every time but should not be cached
 574         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 575         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 576
 577     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 578                          universal_newlines=True)
 579     buildserverid = p.communicate()[0].strip()
 580     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 581     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 582
 583     logger.info("Stopping build server VM")
 584     v.halt()
 585
 586     logger.info("Packaging")
 587     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 588     if os.path.exists(boxfile):
 589         os.remove(boxfile)
 590
 591     vm.package(output=boxfile)
 592
 593     logger.info("Adding box")
 594     vm.box_add('buildserver', boxfile, force=True)
 595
 596     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 597         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 598         sys.exit(1)
 599
 600     if not options.keep_box_file:
 601         logger.debug('box added to vagrant, ' +
 602                      'removing generated box file \'%s\'',
 603                      boxfile)
 604         os.remove(boxfile)
 605
 606
 607 if __name__ == '__main__':
 608     try:
 609         main()
 610     finally:
 611         if tail is not None:
 612             tail.stop()