makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     # Download platform-27 from F-Droid's server because google has two different files for that filename.
 182     # For reproducibility and general sanity we chose the newer one from Nov 17.
 183     # For reference the older one from Nov 8 has a sha256sum of cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac
 184     ('https://verification.f-droid.org/build-metadata/96ddff1a5034fcc4340f2d482635eeaccaa6707b6b0f82d26d1435476a2f52e5/platform-27_r01.zip',
 185      '96ddff1a5034fcc4340f2d482635eeaccaa6707b6b0f82d26d1435476a2f52e5'),
 186     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 187      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 188     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 189      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 190     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 191      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 192     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 193      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 194     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 195      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 196     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 197      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 198     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 199      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 200     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 201      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 202     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 203      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 204     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 205      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 206     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 207      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 209      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 210     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 211      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 212     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 213      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 214     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 215      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 216     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 217      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 218     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 219      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 220     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 221      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 222     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 223      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 224     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 225      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 226     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 227      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 228     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 229      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 230     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 231      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 232     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 233      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 234     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 235      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 236     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 237      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 238     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 239      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 240     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 241      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 242     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 243      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 244     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 245      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 246     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 247      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 248     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 249      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 250     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 251      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 252     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 253      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 254     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 255      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 256     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 257      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 258     ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
 259      'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
 260     ('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
 261      '5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
 262     # the binaries that Google uses are here:
 263     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 264     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 265      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 266     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 267      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 268     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 269      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 270     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 271      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 272     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 273      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 274     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 275      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 276     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 277      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 278     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 279      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 280     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 281      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 282     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 283      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 284     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 285      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 286     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 287      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 288     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 289      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 290     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 291      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 292     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 293      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 294     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 295      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 296     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 297      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 298     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 299      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 300     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 301      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 302     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 303      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 304     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 305      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 306     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 307      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 308     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 309      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 310     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 311      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 312     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 313      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 314     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 315      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 316     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 317      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 318     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 319      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 320     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 321      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 322     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 323      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 324     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 325      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 326     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 327      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 328     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 329      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 330     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 331      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 332     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 333      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 334     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 335      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 336     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 337      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 338     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 339      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 340     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 341      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 342     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 343      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 344     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 345      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 346     ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
 347      'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
 348     ('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
 349      'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
 350     ('https://downloads.gradle.org/distributions/gradle-4.5-bin.zip',
 351      '03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3'),
 352     ('https://downloads.gradle.org/distributions/gradle-4.5.1-bin.zip',
 353      '3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e'),
 354     ('https://downloads.gradle.org/distributions/gradle-4.6-bin.zip',
 355      '98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915'),
 356     ('https://downloads.gradle.org/distributions/gradle-4.7-bin.zip',
 357      'fca5087dc8b50c64655c000989635664a73b11b9bd3703c7d6cabd31b7dcdb04'),
 358     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 359      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 360     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 361      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 362     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 363      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 364     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 365      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 366     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 367      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 368     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 369      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 370     ('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip',
 371      'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'),
 372 ]
 373
 374
 375 def sha256_for_file(path):
 376     with open(path, 'rb') as f:
 377         s = hashlib.sha256()
 378         while True:
 379             data = f.read(4096)
 380             if not data:
 381                 break
 382             s.update(data)
 383         return s.hexdigest()
 384
 385
 386 def run_via_vagrant_ssh(v, cmdlist):
 387     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 388         cmd = cmdlist
 389     else:
 390         cmd = ' '.join(cmdlist)
 391     v._run_vagrant_command(['ssh', '-c', cmd])
 392
 393
 394 def update_cache(cachedir, cachefiles):
 395     for srcurl, shasum in cachefiles:
 396         filename = os.path.basename(srcurl)
 397         local_filename = os.path.join(cachedir, filename)
 398
 399         if os.path.exists(local_filename):
 400             local_length = os.path.getsize(local_filename)
 401         else:
 402             local_length = -1
 403
 404         resume_header = {}
 405         download = True
 406
 407         try:
 408             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 409             if r.status_code == 200:
 410                 content_length = int(r.headers.get('content-length'))
 411             else:
 412                 content_length = local_length  # skip the download
 413         except requests.exceptions.RequestException as e:
 414             content_length = local_length  # skip the download
 415             logger.warn('%s', e)
 416
 417         if local_length == content_length:
 418             download = False
 419         elif local_length > content_length:
 420             logger.info('deleting corrupt file from cache: %s', local_filename)
 421             os.remove(local_filename)
 422             logger.info("Downloading %s to cache", filename)
 423         elif local_length > -1 and local_length < content_length:
 424             logger.info("Resuming download of %s", local_filename)
 425             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 426         else:
 427             logger.info("Downloading %s to cache", filename)
 428
 429         if download:
 430             r = requests.get(srcurl, headers=resume_header,
 431                              stream=True, allow_redirects=True)
 432             content_length = int(r.headers.get('content-length'))
 433             with open(local_filename, 'ab') as f:
 434                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 435                                           expected_size=(content_length / 65536) + 1):
 436                     if chunk:  # filter out keep-alive new chunks
 437                         f.write(chunk)
 438
 439         v = sha256_for_file(local_filename)
 440         if v == shasum:
 441             logger.info("\t...shasum verified for %s", local_filename)
 442         else:
 443             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 444             os.remove(local_filename)
 445             sys.exit(1)
 446
 447
 448 def debug_log_vagrant_vm(vm_dir, config):
 449     if options.verbosity >= 3:
 450         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 451         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 452         logger.debug('> vagrant status')
 453         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 454         logger.debug('> vagrant box list')
 455         subprocess.call(['vagrant', 'box', 'list'])
 456         if config['vm_provider'] == 'libvirt':
 457             logger.debug('> virsh -c qmeu:///system list --all')
 458             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 459             domain = 'buildserver_default'
 460             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 461             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 462
 463
 464 def main():
 465     global cachedir, cachefiles, config, tail
 466
 467     if options.skip_cache_update:
 468         logger.info('skipping cache update and verification...')
 469     else:
 470         update_cache(cachedir, cachefiles)
 471
 472     # use VirtualBox software virtualization if hardware is not available,
 473     # like if this is being run in kvm or some other VM platform, like
 474     # http://jenkins.debian.net, the values are 'on' or 'off'
 475     if sys.platform.startswith('darwin'):
 476         # all < 10 year old Macs work, and OSX servers as VM host are very
 477         # rare, but this could also be auto-detected if someone codes it
 478         config['hwvirtex'] = 'on'
 479         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 480     elif os.path.exists('/proc/cpuinfo'):
 481         with open('/proc/cpuinfo') as f:
 482             contents = f.read()
 483         if 'vmx' in contents or 'svm' in contents:
 484             config['hwvirtex'] = 'on'
 485             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 486
 487     serverdir = os.path.join(os.getcwd(), 'buildserver')
 488     logfilename = os.path.join(serverdir, 'up.log')
 489     if not os.path.exists(logfilename):
 490         open(logfilename, 'a').close()  # create blank file
 491     log_cm = vagrant.make_file_cm(logfilename)
 492     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 493
 494     if options.verbosity >= 2:
 495         tail = fdroidserver.tail.Tail(logfilename)
 496         tail.start()
 497
 498     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 499     if options.clean:
 500         vm.destroy()
 501
 502     # Check against the existing Vagrantfile.yaml, and if they differ, we
 503     # need to create a new box:
 504     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 505     writevf = True
 506     if os.path.exists(vf):
 507         logger.info('Halting %s', serverdir)
 508         v.halt()
 509         with open(vf, 'r', encoding='utf-8') as f:
 510             oldconfig = yaml.load(f)
 511         if config != oldconfig:
 512             logger.info("Server configuration has changed, rebuild from scratch is required")
 513             vm.destroy()
 514         else:
 515             logger.info("Re-provisioning existing server")
 516             writevf = False
 517     else:
 518         logger.info("No existing server - building from scratch")
 519     if writevf:
 520         with open(vf, 'w', encoding='utf-8') as f:
 521             yaml.dump(config, f)
 522
 523     if config['vm_provider'] == 'libvirt':
 524         found_basebox = False
 525         needs_mutate = False
 526         for box in v.box_list():
 527             if box.name == config['basebox']:
 528                 found_basebox = True
 529                 if box.provider != 'libvirt':
 530                     needs_mutate = True
 531                 continue
 532         if not found_basebox:
 533             if isinstance(config['baseboxurl'], str):
 534                 baseboxurl = config['baseboxurl']
 535             else:
 536                 baseboxurl = config['baseboxurl'][0]
 537             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 538             v.box_add(config['basebox'], baseboxurl)
 539             needs_mutate = True
 540         if needs_mutate:
 541             logger.info('Converting %s to libvirt format', config['basebox'])
 542             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 543             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 544             v.box_remove(config['basebox'], 'virtualbox')
 545
 546     logger.info("Configuring build server VM")
 547     debug_log_vagrant_vm(serverdir, config)
 548     try:
 549         v.up(provision=True)
 550     except subprocess.CalledProcessError:
 551         debug_log_vagrant_vm(serverdir, config)
 552         logger.error("'vagrant up' failed, is the base box missing?")
 553         sys.exit(1)
 554
 555     if config['copy_caches_from_host']:
 556         ssh_config = v.ssh_config()
 557         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 558         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 559         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 560         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 561
 562         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 563             fullpath = os.path.join(os.getenv('HOME'), d)
 564             if os.path.isdir(fullpath):
 565                 ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
 566                                         '-o StrictHostKeyChecking=no',
 567                                         '-o UserKnownHostsFile=/dev/null',
 568                                         '-o LogLevel=FATAL',
 569                                         '-o IdentitiesOnly=yes',
 570                                         '-o PasswordAuthentication=no'))
 571                 # TODO vagrant 1.5+ provides `vagrant rsync`
 572                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 573                 subprocess.call(['rsync', '-ax', '--delete', '-e',
 574                                  ssh_command,
 575                                  fullpath + '/',
 576                                  user + '@' + hostname + ':~/' + d + '/'])
 577
 578         # this file changes every time but should not be cached
 579         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 580         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 581
 582     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 583                          universal_newlines=True)
 584     buildserverid = p.communicate()[0].strip()
 585     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 586     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 587
 588     logger.info("Stopping build server VM")
 589     v.halt()
 590
 591     logger.info("Packaging")
 592     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 593     if os.path.exists(boxfile):
 594         os.remove(boxfile)
 595
 596     vm.package(output=boxfile)
 597
 598     logger.info("Adding box")
 599     vm.box_add('buildserver', boxfile, force=True)
 600
 601     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 602         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 603         sys.exit(1)
 604
 605     if not options.keep_box_file:
 606         logger.debug('box added to vagrant, ' +
 607                      'removing generated box file \'%s\'',
 608                      boxfile)
 609         os.remove(boxfile)
 610
 611
 612 if __name__ == '__main__':
 613     try:
 614         main()
 615     finally:
 616         if tail is not None:
 617             tail.stop()