makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     # Download platform-27 from F-Droid's server because google has two different files for that filename.
 182     # For reproducibility and general sanity we chose the newer one from Nov 17.
 183     # For reference the older one from Nov 8 has a sha256sum of cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac
 184     ('https://verification.f-droid.org/build-metadata/96ddff1a5034fcc4340f2d482635eeaccaa6707b6b0f82d26d1435476a2f52e5/platform-27_r01.zip',
 185      '96ddff1a5034fcc4340f2d482635eeaccaa6707b6b0f82d26d1435476a2f52e5'),
 186     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 187      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 188     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 189      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 190     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 191      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 192     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 193      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 194     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 195      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 196     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 197      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 198     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 199      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 200     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 201      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 202     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 203      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 204     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 205      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 206     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 207      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 209      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 210     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 211      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 212     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 213      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 214     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 215      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 216     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 217      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 218     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 219      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 220     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 221      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 222     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 223      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 224     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 225      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 226     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 227      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 228     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 229      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 230     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 231      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 232     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 233      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 234     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 235      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 236     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 237      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 238     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 239      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 240     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 241      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 242     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 243      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 244     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 245      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 246     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 247      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 248     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 249      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 250     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 251      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 252     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 253      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 254     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 255      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 256     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 257      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 258     ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
 259      'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
 260     ('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
 261      '5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
 262     # the binaries that Google uses are here:
 263     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 264     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 265      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 266     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 267      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 268     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 269      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 270     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 271      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 272     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 273      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 274     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 275      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 276     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 277      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 278     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 279      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 280     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 281      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 282     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 283      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 284     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 285      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 286     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 287      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 288     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 289      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 290     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 291      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 292     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 293      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 294     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 295      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 296     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 297      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 298     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 299      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 300     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 301      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 302     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 303      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 304     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 305      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 306     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 307      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 308     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 309      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 310     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 311      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 312     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 313      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 314     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 315      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 316     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 317      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 318     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 319      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 320     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 321      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 322     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 323      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 324     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 325      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 326     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 327      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 328     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 329      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 330     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 331      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 332     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 333      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 334     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 335      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 336     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 337      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 338     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 339      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 340     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 341      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 342     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 343      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 344     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 345      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 346     ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
 347      'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
 348     ('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
 349      'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
 350     ('https://downloads.gradle.org/distributions/gradle-4.5-bin.zip',
 351      '03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3'),
 352     ('https://downloads.gradle.org/distributions/gradle-4.5.1-bin.zip',
 353      '3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e'),
 354     ('https://downloads.gradle.org/distributions/gradle-4.6-bin.zip',
 355      '98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915'),
 356     ('https://downloads.gradle.org/distributions/gradle-4.7-bin.zip',
 357      'fca5087dc8b50c64655c000989635664a73b11b9bd3703c7d6cabd31b7dcdb04'),
 358     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 359      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 360     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 361      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 362     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 363      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 364     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 365      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 366     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 367      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 368     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 369      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 370     ('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip',
 371      'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'),
 372 ]
 373
 374
 375 def sha256_for_file(path):
 376     with open(path, 'rb') as f:
 377         s = hashlib.sha256()
 378         while True:
 379             data = f.read(4096)
 380             if not data:
 381                 break
 382             s.update(data)
 383         return s.hexdigest()
 384
 385
 386 def run_via_vagrant_ssh(v, cmdlist):
 387     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 388         cmd = cmdlist
 389     else:
 390         cmd = ' '.join(cmdlist)
 391     v._run_vagrant_command(['ssh', '-c', cmd])
 392
 393
 394 def update_cache(cachedir, cachefiles):
 395     for srcurl, shasum in cachefiles:
 396         filename = os.path.basename(srcurl)
 397         local_filename = os.path.join(cachedir, filename)
 398
 399         if os.path.exists(local_filename):
 400             local_length = os.path.getsize(local_filename)
 401         else:
 402             local_length = -1
 403
 404         resume_header = {}
 405         download = True
 406
 407         try:
 408             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 409             if r.status_code == 200:
 410                 content_length = int(r.headers.get('content-length'))
 411             else:
 412                 content_length = local_length  # skip the download
 413         except requests.exceptions.RequestException as e:
 414             content_length = local_length  # skip the download
 415             logger.warn('%s', e)
 416
 417         if local_length == content_length:
 418             download = False
 419         elif local_length > content_length:
 420             logger.info('deleting corrupt file from cache: %s', local_filename)
 421             os.remove(local_filename)
 422             logger.info("Downloading %s to cache", filename)
 423         elif local_length > -1 and local_length < content_length:
 424             logger.info("Resuming download of %s", local_filename)
 425             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 426         else:
 427             logger.info("Downloading %s to cache", filename)
 428
 429         if download:
 430             r = requests.get(srcurl, headers=resume_header,
 431                              stream=True, allow_redirects=True)
 432             content_length = int(r.headers.get('content-length'))
 433             with open(local_filename, 'ab') as f:
 434                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 435                                           expected_size=(content_length / 65536) + 1):
 436                     if chunk:  # filter out keep-alive new chunks
 437                         f.write(chunk)
 438
 439         v = sha256_for_file(local_filename)
 440         if v == shasum:
 441             logger.info("\t...shasum verified for %s", local_filename)
 442         else:
 443             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 444             os.remove(local_filename)
 445             sys.exit(1)
 446
 447
 448 def debug_log_vagrant_vm(vm_dir, config):
 449     if options.verbosity >= 3:
 450         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 451         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 452         logger.debug('> vagrant status')
 453         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 454         logger.debug('> vagrant box list')
 455         subprocess.call(['vagrant', 'box', 'list'])
 456         if config['vm_provider'] == 'libvirt':
 457             logger.debug('> virsh -c qmeu:///system list --all')
 458             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 459             domain = 'buildserver_default'
 460             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 461             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 462
 463
 464 def main():
 465     global cachedir, cachefiles, config, tail
 466
 467     if options.skip_cache_update:
 468         logger.info('skipping cache update and verification...')
 469     else:
 470         update_cache(cachedir, cachefiles)
 471
 472     # use VirtualBox software virtualization if hardware is not available,
 473     # like if this is being run in kvm or some other VM platform, like
 474     # http://jenkins.debian.net, the values are 'on' or 'off'
 475     if sys.platform.startswith('darwin'):
 476         # all < 10 year old Macs work, and OSX servers as VM host are very
 477         # rare, but this could also be auto-detected if someone codes it
 478         config['hwvirtex'] = 'on'
 479         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 480     elif os.path.exists('/proc/cpuinfo'):
 481         with open('/proc/cpuinfo') as f:
 482             contents = f.read()
 483         if 'vmx' in contents or 'svm' in contents:
 484             config['hwvirtex'] = 'on'
 485             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 486
 487     serverdir = os.path.join(os.getcwd(), 'buildserver')
 488     logfilename = os.path.join(serverdir, 'up.log')
 489     if not os.path.exists(logfilename):
 490         open(logfilename, 'a').close()  # create blank file
 491     log_cm = vagrant.make_file_cm(logfilename)
 492     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 493
 494     if options.verbosity >= 2:
 495         tail = fdroidserver.tail.Tail(logfilename)
 496         tail.start()
 497
 498     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 499     if options.clean:
 500         vm.destroy()
 501
 502     # Check against the existing Vagrantfile.yaml, and if they differ, we
 503     # need to create a new box:
 504     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 505     writevf = True
 506     if os.path.exists(vf):
 507         logger.info('Halting %s', serverdir)
 508         v.halt()
 509         with open(vf, 'r', encoding='utf-8') as f:
 510             oldconfig = yaml.load(f)
 511         if config != oldconfig:
 512             logger.info("Server configuration has changed, rebuild from scratch is required")
 513             vm.destroy()
 514         else:
 515             logger.info("Re-provisioning existing server")
 516             writevf = False
 517     else:
 518         logger.info("No existing server - building from scratch")
 519     if writevf:
 520         with open(vf, 'w', encoding='utf-8') as f:
 521             yaml.dump(config, f)
 522
 523     if config['vm_provider'] == 'libvirt':
 524         found_basebox = False
 525         needs_mutate = False
 526         for box in v.box_list():
 527             if box.name == config['basebox']:
 528                 found_basebox = True
 529                 if box.provider != 'libvirt':
 530                     logger.info('NEED TO MUTATE %s %s' % (box.name, box.provider))
 531                     needs_mutate = True
 532                 continue
 533         if not found_basebox:
 534             if isinstance(config['baseboxurl'], str):
 535                 baseboxurl = config['baseboxurl']
 536             else:
 537                 baseboxurl = config['baseboxurl'][0]
 538             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 539             v.box_add(config['basebox'], baseboxurl)
 540             #needs_mutate = True
 541         if needs_mutate:
 542             logger.info('Converting %s to libvirt format', config['basebox'])
 543             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 544             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 545             v.box_remove(config['basebox'], 'virtualbox')
 546
 547     logger.info("Configuring build server VM")
 548     debug_log_vagrant_vm(serverdir, config)
 549     try:
 550         v.up(provision=True)
 551     except subprocess.CalledProcessError:
 552         debug_log_vagrant_vm(serverdir, config)
 553         logger.error("'vagrant up' failed, is the base box missing?")
 554         sys.exit(1)
 555
 556     if config['copy_caches_from_host']:
 557         ssh_config = v.ssh_config()
 558         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 559         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 560         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 561         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 562
 563         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 564             fullpath = os.path.join(os.getenv('HOME'), d)
 565             if os.path.isdir(fullpath):
 566                 ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
 567                                         '-o StrictHostKeyChecking=no',
 568                                         '-o UserKnownHostsFile=/dev/null',
 569                                         '-o LogLevel=FATAL',
 570                                         '-o IdentitiesOnly=yes',
 571                                         '-o PasswordAuthentication=no'))
 572                 # TODO vagrant 1.5+ provides `vagrant rsync`
 573                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 574                 subprocess.call(['rsync', '-ax', '--delete', '-e',
 575                                  ssh_command,
 576                                  fullpath + '/',
 577                                  user + '@' + hostname + ':~/' + d + '/'])
 578
 579         # this file changes every time but should not be cached
 580         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 581         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 582
 583     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 584                          universal_newlines=True)
 585     buildserverid = p.communicate()[0].strip()
 586     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 587     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 588
 589     logger.info("Stopping build server VM")
 590     v.halt()
 591
 592     logger.info("Packaging")
 593     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 594     if os.path.exists(boxfile):
 595         os.remove(boxfile)
 596
 597     vm.package(output=boxfile)
 598
 599     logger.info("Adding box")
 600     vm.box_add('buildserver', boxfile, force=True)
 601
 602     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 603         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 604         sys.exit(1)
 605
 606     if not options.keep_box_file:
 607         logger.debug('box added to vagrant, ' +
 608                      'removing generated box file \'%s\'',
 609                      boxfile)
 610         os.remove(boxfile)
 611
 612
 613 if __name__ == '__main__':
 614     try:
 615         main()
 616     finally:
 617         if tail is not None:
 618             tail.stop()