makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     # Download platform-27 from F-Droid's server because google has two different files for that filename.
 182     # For reproducibility and general sanity we chose the newer one from Nov 17.
 183     # For reference the older one from Nov 8 has a sha256sum of cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac
 184     ('https://verification.f-droid.org/build-metadata/96ddff1a5034fcc4340f2d482635eeaccaa6707b6b0f82d26d1435476a2f52e5/platform-27_r01.zip',
 185      '96ddff1a5034fcc4340f2d482635eeaccaa6707b6b0f82d26d1435476a2f52e5'),
 186     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 187      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 188     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 189      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 190     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 191      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 192     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 193      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 194     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 195      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 196     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 197      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 198     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 199      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 200     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 201      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 202     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 203      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 204     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 205      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 206     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 207      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 209      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 210     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 211      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 212     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 213      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 214     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 215      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 216     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 217      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 218     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 219      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 220     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 221      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 222     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 223      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 224     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 225      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 226     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 227      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 228     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 229      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 230     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 231      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 232     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 233      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 234     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 235      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 236     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 237      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 238     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 239      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 240     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 241      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 242     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 243      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 244     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 245      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 246     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 247      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 248     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 249      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 250     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 251      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 252     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 253      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 254     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 255      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 256     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 257      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 258     ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
 259      'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
 260     ('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
 261      '5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
 262     # the binaries that Google uses are here:
 263     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 264     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 265      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 266     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 267      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 268     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 269      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 270     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 271      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 272     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 273      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 274     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 275      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 276     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 277      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 278     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 279      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 280     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 281      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 282     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 283      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 284     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 285      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 286     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 287      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 288     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 289      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 290     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 291      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 292     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 293      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 294     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 295      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 296     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 297      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 298     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 299      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 300     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 301      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 302     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 303      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 304     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 305      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 306     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 307      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 308     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 309      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 310     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 311      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 312     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 313      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 314     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 315      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 316     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 317      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 318     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 319      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 320     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 321      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 322     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 323      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 324     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 325      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 326     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 327      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 328     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 329      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 330     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 331      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 332     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 333      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 334     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 335      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 336     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 337      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 338     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 339      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 340     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 341      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 342     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 343      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 344     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 345      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 346     ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
 347      'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
 348     ('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
 349      'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
 350     ('https://downloads.gradle.org/distributions/gradle-4.5-bin.zip',
 351      '03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3'),
 352     ('https://downloads.gradle.org/distributions/gradle-4.5.1-bin.zip',
 353      '3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e'),
 354     ('https://downloads.gradle.org/distributions/gradle-4.6-bin.zip',
 355      '98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915'),
 356     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 357      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 358     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 359      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 360     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 361      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 362     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 363      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 364     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 365      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 366     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 367      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 368     ('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip',
 369      'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'),
 370 ]
 371
 372
 373 def sha256_for_file(path):
 374     with open(path, 'rb') as f:
 375         s = hashlib.sha256()
 376         while True:
 377             data = f.read(4096)
 378             if not data:
 379                 break
 380             s.update(data)
 381         return s.hexdigest()
 382
 383
 384 def run_via_vagrant_ssh(v, cmdlist):
 385     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 386         cmd = cmdlist
 387     else:
 388         cmd = ' '.join(cmdlist)
 389     v._run_vagrant_command(['ssh', '-c', cmd])
 390
 391
 392 def update_cache(cachedir, cachefiles):
 393     for srcurl, shasum in cachefiles:
 394         filename = os.path.basename(srcurl)
 395         local_filename = os.path.join(cachedir, filename)
 396
 397         if os.path.exists(local_filename):
 398             local_length = os.path.getsize(local_filename)
 399         else:
 400             local_length = -1
 401
 402         resume_header = {}
 403         download = True
 404
 405         try:
 406             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 407             if r.status_code == 200:
 408                 content_length = int(r.headers.get('content-length'))
 409             else:
 410                 content_length = local_length  # skip the download
 411         except requests.exceptions.RequestException as e:
 412             content_length = local_length  # skip the download
 413             logger.warn('%s', e)
 414
 415         if local_length == content_length:
 416             download = False
 417         elif local_length > content_length:
 418             logger.info('deleting corrupt file from cache: %s', local_filename)
 419             os.remove(local_filename)
 420             logger.info("Downloading %s to cache", filename)
 421         elif local_length > -1 and local_length < content_length:
 422             logger.info("Resuming download of %s", local_filename)
 423             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 424         else:
 425             logger.info("Downloading %s to cache", filename)
 426
 427         if download:
 428             r = requests.get(srcurl, headers=resume_header,
 429                              stream=True, allow_redirects=True)
 430             content_length = int(r.headers.get('content-length'))
 431             with open(local_filename, 'ab') as f:
 432                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 433                                           expected_size=(content_length / 65536) + 1):
 434                     if chunk:  # filter out keep-alive new chunks
 435                         f.write(chunk)
 436
 437         v = sha256_for_file(local_filename)
 438         if v == shasum:
 439             logger.info("\t...shasum verified for %s", local_filename)
 440         else:
 441             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 442             os.remove(local_filename)
 443             sys.exit(1)
 444
 445
 446 def debug_log_vagrant_vm(vm_dir, config):
 447     if options.verbosity >= 3:
 448         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 449         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 450         logger.debug('> vagrant status')
 451         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 452         logger.debug('> vagrant box list')
 453         subprocess.call(['vagrant', 'box', 'list'])
 454         if config['vm_provider'] == 'libvirt':
 455             logger.debug('> virsh -c qmeu:///system list --all')
 456             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 457             domain = 'buildserver_default'
 458             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 459             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 460
 461
 462 def main():
 463     global cachedir, cachefiles, config, tail
 464
 465     if options.skip_cache_update:
 466         logger.info('skipping cache update and verification...')
 467     else:
 468         update_cache(cachedir, cachefiles)
 469
 470     # use VirtualBox software virtualization if hardware is not available,
 471     # like if this is being run in kvm or some other VM platform, like
 472     # http://jenkins.debian.net, the values are 'on' or 'off'
 473     if sys.platform.startswith('darwin'):
 474         # all < 10 year old Macs work, and OSX servers as VM host are very
 475         # rare, but this could also be auto-detected if someone codes it
 476         config['hwvirtex'] = 'on'
 477         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 478     elif os.path.exists('/proc/cpuinfo'):
 479         with open('/proc/cpuinfo') as f:
 480             contents = f.read()
 481         if 'vmx' in contents or 'svm' in contents:
 482             config['hwvirtex'] = 'on'
 483             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 484
 485     serverdir = os.path.join(os.getcwd(), 'buildserver')
 486     logfilename = os.path.join(serverdir, 'up.log')
 487     if not os.path.exists(logfilename):
 488         open(logfilename, 'a').close()  # create blank file
 489     log_cm = vagrant.make_file_cm(logfilename)
 490     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 491
 492     if options.verbosity >= 2:
 493         tail = fdroidserver.tail.Tail(logfilename)
 494         tail.start()
 495
 496     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 497     if options.clean:
 498         vm.destroy()
 499
 500     # Check against the existing Vagrantfile.yaml, and if they differ, we
 501     # need to create a new box:
 502     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 503     writevf = True
 504     if os.path.exists(vf):
 505         logger.info('Halting %s', serverdir)
 506         v.halt()
 507         with open(vf, 'r', encoding='utf-8') as f:
 508             oldconfig = yaml.load(f)
 509         if config != oldconfig:
 510             logger.info("Server configuration has changed, rebuild from scratch is required")
 511             vm.destroy()
 512         else:
 513             logger.info("Re-provisioning existing server")
 514             writevf = False
 515     else:
 516         logger.info("No existing server - building from scratch")
 517     if writevf:
 518         with open(vf, 'w', encoding='utf-8') as f:
 519             yaml.dump(config, f)
 520
 521     if config['vm_provider'] == 'libvirt':
 522         found_basebox = False
 523         needs_mutate = False
 524         for box in v.box_list():
 525             if box.name == config['basebox']:
 526                 found_basebox = True
 527                 if box.provider != 'libvirt':
 528                     needs_mutate = True
 529                 continue
 530         if not found_basebox:
 531             if isinstance(config['baseboxurl'], str):
 532                 baseboxurl = config['baseboxurl']
 533             else:
 534                 baseboxurl = config['baseboxurl'][0]
 535             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 536             v.box_add(config['basebox'], baseboxurl)
 537             needs_mutate = True
 538         if needs_mutate:
 539             logger.info('Converting %s to libvirt format', config['basebox'])
 540             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 541             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 542             v.box_remove(config['basebox'], 'virtualbox')
 543
 544     logger.info("Configuring build server VM")
 545     debug_log_vagrant_vm(serverdir, config)
 546     try:
 547         v.up(provision=True)
 548     except subprocess.CalledProcessError:
 549         debug_log_vagrant_vm(serverdir, config)
 550         logger.error("'vagrant up' failed, is the base box missing?")
 551         sys.exit(1)
 552
 553     if config['copy_caches_from_host']:
 554         ssh_config = v.ssh_config()
 555         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 556         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 557         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 558         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 559
 560         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 561             fullpath = os.path.join(os.getenv('HOME'), d)
 562             if os.path.isdir(fullpath):
 563                 ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
 564                                         '-o StrictHostKeyChecking=no',
 565                                         '-o UserKnownHostsFile=/dev/null',
 566                                         '-o LogLevel=FATAL',
 567                                         '-o IdentitiesOnly=yes',
 568                                         '-o PasswordAuthentication=no'))
 569                 # TODO vagrant 1.5+ provides `vagrant rsync`
 570                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 571                 subprocess.call(['rsync', '-ax', '--delete', '-e',
 572                                  ssh_command,
 573                                  fullpath + '/',
 574                                  user + '@' + hostname + ':~/' + d + '/'])
 575
 576         # this file changes every time but should not be cached
 577         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 578         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 579
 580     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 581                          universal_newlines=True)
 582     buildserverid = p.communicate()[0].strip()
 583     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 584     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 585
 586     logger.info("Stopping build server VM")
 587     v.halt()
 588
 589     logger.info("Packaging")
 590     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 591     if os.path.exists(boxfile):
 592         os.remove(boxfile)
 593
 594     vm.package(output=boxfile)
 595
 596     logger.info("Adding box")
 597     vm.box_add('buildserver', boxfile, force=True)
 598
 599     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 600         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 601         sys.exit(1)
 602
 603     if not options.keep_box_file:
 604         logger.debug('box added to vagrant, ' +
 605                      'removing generated box file \'%s\'',
 606                      boxfile)
 607         os.remove(boxfile)
 608
 609
 610 if __name__ == '__main__':
 611     try:
 612         main()
 613     finally:
 614         if tail is not None:
 615             tail.stop()