makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     # Download platform-27 from F-Droid's server because google has two different files for that filename.
 182     # For reproducibility and general sanity we chose the newer one from Nov 17.
 183     # For reference the older one from Nov 8 has a sha256sum of cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac
 184     ('https://verification.f-droid.org/build-metadata/96ddff1a5034fcc4340f2d482635eeaccaa6707b6b0f82d26d1435476a2f52e5/platform-27_r01.zip',
 185      '96ddff1a5034fcc4340f2d482635eeaccaa6707b6b0f82d26d1435476a2f52e5'),
 186     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 187      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 188     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 189      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 190     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 191      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 192     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 193      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 194     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 195      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 196     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 197      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 198     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 199      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 200     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 201      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 202     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 203      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 204     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 205      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 206     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 207      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 209      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 210     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 211      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 212     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 213      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 214     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 215      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 216     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 217      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 218     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 219      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 220     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 221      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 222     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 223      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 224     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 225      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 226     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 227      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 228     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 229      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 230     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 231      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 232     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 233      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 234     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 235      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 236     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 237      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 238     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 239      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 240     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 241      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 242     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 243      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 244     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 245      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 246     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 247      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 248     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 249      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 250     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 251      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 252     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 253      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 254     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 255      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 256     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 257      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 258     ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
 259      'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
 260     ('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
 261      '5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
 262     # the binaries that Google uses are here:
 263     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 264     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 265      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 266     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 267      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 268     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 269      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 270     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 271      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 272     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 273      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 274     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 275      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 276     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 277      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 278     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 279      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 280     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 281      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 282     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 283      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 284     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 285      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 286     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 287      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 288     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 289      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 290     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 291      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 292     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 293      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 294     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 295      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 296     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 297      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 298     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 299      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 300     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 301      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 302     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 303      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 304     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 305      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 306     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 307      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 308     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 309      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 310     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 311      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 312     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 313      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 314     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 315      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 316     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 317      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 318     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 319      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 320     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 321      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 322     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 323      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 324     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 325      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 326     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 327      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 328     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 329      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 330     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 331      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 332     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 333      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 334     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 335      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 336     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 337      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 338     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 339      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 340     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 341      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 342     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 343      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 344     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 345      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 346     ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
 347      'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
 348     ('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
 349      'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
 350     ('https://downloads.gradle.org/distributions/gradle-4.5-bin.zip',
 351      '03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3'),
 352     ('https://downloads.gradle.org/distributions/gradle-4.5.1-bin.zip',
 353      '3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e'),
 354     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 355      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 356     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 357      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 358     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 359      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 360     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 361      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 362     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 363      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 364     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 365      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 366     ('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip',
 367      'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'),
 368 ]
 369
 370
 371 def sha256_for_file(path):
 372     with open(path, 'rb') as f:
 373         s = hashlib.sha256()
 374         while True:
 375             data = f.read(4096)
 376             if not data:
 377                 break
 378             s.update(data)
 379         return s.hexdigest()
 380
 381
 382 def run_via_vagrant_ssh(v, cmdlist):
 383     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 384         cmd = cmdlist
 385     else:
 386         cmd = ' '.join(cmdlist)
 387     v._run_vagrant_command(['ssh', '-c', cmd])
 388
 389
 390 def update_cache(cachedir, cachefiles):
 391     for srcurl, shasum in cachefiles:
 392         filename = os.path.basename(srcurl)
 393         local_filename = os.path.join(cachedir, filename)
 394
 395         if os.path.exists(local_filename):
 396             local_length = os.path.getsize(local_filename)
 397         else:
 398             local_length = -1
 399
 400         resume_header = {}
 401         download = True
 402
 403         try:
 404             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 405             if r.status_code == 200:
 406                 content_length = int(r.headers.get('content-length'))
 407             else:
 408                 content_length = local_length  # skip the download
 409         except requests.exceptions.RequestException as e:
 410             content_length = local_length  # skip the download
 411             logger.warn('%s', e)
 412
 413         if local_length == content_length:
 414             download = False
 415         elif local_length > content_length:
 416             logger.info('deleting corrupt file from cache: %s', local_filename)
 417             os.remove(local_filename)
 418             logger.info("Downloading %s to cache", filename)
 419         elif local_length > -1 and local_length < content_length:
 420             logger.info("Resuming download of %s", local_filename)
 421             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 422         else:
 423             logger.info("Downloading %s to cache", filename)
 424
 425         if download:
 426             r = requests.get(srcurl, headers=resume_header,
 427                              stream=True, allow_redirects=True)
 428             content_length = int(r.headers.get('content-length'))
 429             with open(local_filename, 'ab') as f:
 430                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 431                                           expected_size=(content_length / 65536) + 1):
 432                     if chunk:  # filter out keep-alive new chunks
 433                         f.write(chunk)
 434
 435         v = sha256_for_file(local_filename)
 436         if v == shasum:
 437             logger.info("\t...shasum verified for %s", local_filename)
 438         else:
 439             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 440             os.remove(local_filename)
 441             sys.exit(1)
 442
 443
 444 def debug_log_vagrant_vm(vm_dir, config):
 445     if options.verbosity >= 3:
 446         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 447         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 448         logger.debug('> vagrant status')
 449         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 450         logger.debug('> vagrant box list')
 451         subprocess.call(['vagrant', 'box', 'list'])
 452         if config['vm_provider'] == 'libvirt':
 453             logger.debug('> virsh -c qmeu:///system list --all')
 454             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 455             domain = 'buildserver_default'
 456             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 457             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 458
 459
 460 def main():
 461     global cachedir, cachefiles, config, tail
 462
 463     if options.skip_cache_update:
 464         logger.info('skipping cache update and verification...')
 465     else:
 466         update_cache(cachedir, cachefiles)
 467
 468     # use VirtualBox software virtualization if hardware is not available,
 469     # like if this is being run in kvm or some other VM platform, like
 470     # http://jenkins.debian.net, the values are 'on' or 'off'
 471     if sys.platform.startswith('darwin'):
 472         # all < 10 year old Macs work, and OSX servers as VM host are very
 473         # rare, but this could also be auto-detected if someone codes it
 474         config['hwvirtex'] = 'on'
 475         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 476     elif os.path.exists('/proc/cpuinfo'):
 477         with open('/proc/cpuinfo') as f:
 478             contents = f.read()
 479         if 'vmx' in contents or 'svm' in contents:
 480             config['hwvirtex'] = 'on'
 481             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 482
 483     serverdir = os.path.join(os.getcwd(), 'buildserver')
 484     logfilename = os.path.join(serverdir, 'up.log')
 485     if not os.path.exists(logfilename):
 486         open(logfilename, 'a').close()  # create blank file
 487     log_cm = vagrant.make_file_cm(logfilename)
 488     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 489
 490     if options.verbosity >= 2:
 491         tail = fdroidserver.tail.Tail(logfilename)
 492         tail.start()
 493
 494     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 495     if options.clean:
 496         vm.destroy()
 497
 498     # Check against the existing Vagrantfile.yaml, and if they differ, we
 499     # need to create a new box:
 500     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 501     writevf = True
 502     if os.path.exists(vf):
 503         logger.info('Halting %s', serverdir)
 504         v.halt()
 505         with open(vf, 'r', encoding='utf-8') as f:
 506             oldconfig = yaml.load(f)
 507         if config != oldconfig:
 508             logger.info("Server configuration has changed, rebuild from scratch is required")
 509             vm.destroy()
 510         else:
 511             logger.info("Re-provisioning existing server")
 512             writevf = False
 513     else:
 514         logger.info("No existing server - building from scratch")
 515     if writevf:
 516         with open(vf, 'w', encoding='utf-8') as f:
 517             yaml.dump(config, f)
 518
 519     if config['vm_provider'] == 'libvirt':
 520         found_basebox = False
 521         needs_mutate = False
 522         for box in v.box_list():
 523             if box.name == config['basebox']:
 524                 found_basebox = True
 525                 if box.provider != 'libvirt':
 526                     needs_mutate = True
 527                 continue
 528         if not found_basebox:
 529             if isinstance(config['baseboxurl'], str):
 530                 baseboxurl = config['baseboxurl']
 531             else:
 532                 baseboxurl = config['baseboxurl'][0]
 533             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 534             v.box_add(config['basebox'], baseboxurl)
 535             needs_mutate = True
 536         if needs_mutate:
 537             logger.info('Converting %s to libvirt format', config['basebox'])
 538             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 539             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 540             v.box_remove(config['basebox'], 'virtualbox')
 541
 542     logger.info("Configuring build server VM")
 543     debug_log_vagrant_vm(serverdir, config)
 544     try:
 545         v.up(provision=True)
 546     except subprocess.CalledProcessError:
 547         debug_log_vagrant_vm(serverdir, config)
 548         logger.error("'vagrant up' failed, is the base box missing?")
 549         sys.exit(1)
 550
 551     if config['copy_caches_from_host']:
 552         ssh_config = v.ssh_config()
 553         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 554         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 555         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 556         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 557
 558         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 559             fullpath = os.path.join(os.getenv('HOME'), d)
 560             if os.path.isdir(fullpath):
 561                 ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
 562                                         '-o StrictHostKeyChecking=no',
 563                                         '-o UserKnownHostsFile=/dev/null',
 564                                         '-o LogLevel=FATAL',
 565                                         '-o IdentitiesOnly=yes',
 566                                         '-o PasswordAuthentication=no'))
 567                 # TODO vagrant 1.5+ provides `vagrant rsync`
 568                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 569                 subprocess.call(['rsync', '-ax', '--delete', '-e',
 570                                  ssh_command,
 571                                  fullpath + '/',
 572                                  user + '@' + hostname + ':~/' + d + '/'])
 573
 574         # this file changes every time but should not be cached
 575         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 576         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 577
 578     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 579                          universal_newlines=True)
 580     buildserverid = p.communicate()[0].strip()
 581     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 582     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 583
 584     logger.info("Stopping build server VM")
 585     v.halt()
 586
 587     logger.info("Packaging")
 588     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 589     if os.path.exists(boxfile):
 590         os.remove(boxfile)
 591
 592     vm.package(output=boxfile)
 593
 594     logger.info("Adding box")
 595     vm.box_add('buildserver', boxfile, force=True)
 596
 597     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 598         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 599         sys.exit(1)
 600
 601     if not options.keep_box_file:
 602         logger.debug('box added to vagrant, ' +
 603                      'removing generated box file \'%s\'',
 604                      boxfile)
 605         os.remove(boxfile)
 606
 607
 608 if __name__ == '__main__':
 609     try:
 610         main()
 611     finally:
 612         if tail is not None:
 613             tail.stop()