makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 182      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 184      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 186      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 188      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 189     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 190      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 192      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 194      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 196      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 198      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 199     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 200      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 201     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 202      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 204      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 206      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 208      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 210      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 212      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 213     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 214      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 215     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 216      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 217     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 218      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 220      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 222      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 224      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 225     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 226      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 228      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 230      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 232      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 233     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 234      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 236      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 238      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 240      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 241     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 242      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 243     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 244      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 246      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 248      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 249     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 250      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 251     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 252      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 253     ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
 254      'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
 255     # the binaries that Google uses are here:
 256     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 257     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 258      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 259     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 260      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 261     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 262      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 263     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 264      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 265     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 266      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 267     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 268      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 269     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 270      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 271     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 272      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 273     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 274      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 275     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 276      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 277     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 278      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 279     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 280      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 281     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 282      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 283     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 284      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 285     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 286      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 287     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 288      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 289     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 290      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 291     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 292      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 293     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 294      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 295     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 296      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 297     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 298      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 299     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 300      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 301     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 302      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 303     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 304      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 305     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 306      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 307     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 308      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 309     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 310      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 311     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 312      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 313     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 314      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 315     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 316      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 317     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 318      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 319     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 320      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 321     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 322      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 323     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 324      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 325     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 326      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 327     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 328      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 329     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 330      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 331     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 332      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 333     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 334      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 335     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 336      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 337     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 338      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 339     ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
 340      'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
 341     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 342      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 343     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 344      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 345     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 346      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 347     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 348      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 349     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 350      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 351     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 352      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 353     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 354      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 355     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 356      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 357     ('https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip',
 358      'a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e'),
 359     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 360      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 361 ]
 362
 363
 364 def sha256_for_file(path):
 365     with open(path, 'rb') as f:
 366         s = hashlib.sha256()
 367         while True:
 368             data = f.read(4096)
 369             if not data:
 370                 break
 371             s.update(data)
 372         return s.hexdigest()
 373
 374
 375 def run_via_vagrant_ssh(v, cmdlist):
 376     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 377         cmd = cmdlist
 378     else:
 379         cmd = ' '.join(cmdlist)
 380     v._run_vagrant_command(['ssh', '-c', cmd])
 381
 382
 383 def update_cache(cachedir, cachefiles):
 384     for srcurl, shasum in cachefiles:
 385         filename = os.path.basename(srcurl)
 386         local_filename = os.path.join(cachedir, filename)
 387
 388         if os.path.exists(local_filename):
 389             local_length = os.path.getsize(local_filename)
 390         else:
 391             local_length = -1
 392
 393         resume_header = {}
 394         download = True
 395
 396         try:
 397             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 398             if r.status_code == 200:
 399                 content_length = int(r.headers.get('content-length'))
 400             else:
 401                 content_length = local_length  # skip the download
 402         except requests.exceptions.RequestException as e:
 403             content_length = local_length  # skip the download
 404             logger.warn('%s', e)
 405
 406         if local_length == content_length:
 407             download = False
 408         elif local_length > content_length:
 409             logger.info('deleting corrupt file from cache: %s', local_filename)
 410             os.remove(local_filename)
 411             logger.info("Downloading %s to cache", filename)
 412         elif local_length > -1 and local_length < content_length:
 413             logger.info("Resuming download of %s", local_filename)
 414             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 415         else:
 416             logger.info("Downloading %s to cache", filename)
 417
 418         if download:
 419             r = requests.get(srcurl, headers=resume_header,
 420                              stream=True, allow_redirects=True)
 421             content_length = int(r.headers.get('content-length'))
 422             with open(local_filename, 'ab') as f:
 423                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 424                                           expected_size=(content_length / 65536) + 1):
 425                     if chunk:  # filter out keep-alive new chunks
 426                         f.write(chunk)
 427
 428         v = sha256_for_file(local_filename)
 429         if v == shasum:
 430             logger.info("\t...shasum verified for %s", local_filename)
 431         else:
 432             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 433             os.remove(local_filename)
 434             sys.exit(1)
 435
 436
 437 def debug_log_vagrant_vm(vm_dir, config):
 438     if options.verbosity >= 3:
 439         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 440         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 441         logger.debug('> vagrant status')
 442         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 443         logger.debug('> vagrant box list')
 444         subprocess.call(['vagrant', 'box', 'list'])
 445         if config['vm_provider'] == 'libvirt':
 446             logger.debug('> virsh -c qmeu:///system list --all')
 447             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 448             domain = 'buildserver_default'
 449             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 450             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 451
 452
 453 def main():
 454     global cachedir, cachefiles, config, tail
 455
 456     if options.skip_cache_update:
 457         logger.info('skipping cache update and verification...')
 458     else:
 459         update_cache(cachedir, cachefiles)
 460
 461     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 462     logger.info("Setting executable bit for %s", local_qt_filename)
 463     os.chmod(local_qt_filename, 0o755)
 464
 465     # use VirtualBox software virtualization if hardware is not available,
 466     # like if this is being run in kvm or some other VM platform, like
 467     # http://jenkins.debian.net, the values are 'on' or 'off'
 468     if sys.platform.startswith('darwin'):
 469         # all < 10 year old Macs work, and OSX servers as VM host are very
 470         # rare, but this could also be auto-detected if someone codes it
 471         config['hwvirtex'] = 'on'
 472         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 473     elif os.path.exists('/proc/cpuinfo'):
 474         with open('/proc/cpuinfo') as f:
 475             contents = f.read()
 476         if 'vmx' in contents or 'svm' in contents:
 477             config['hwvirtex'] = 'on'
 478             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 479
 480     serverdir = os.path.join(os.getcwd(), 'buildserver')
 481     logfilename = os.path.join(serverdir, 'up.log')
 482     if not os.path.exists(logfilename):
 483         open(logfilename, 'a').close()  # create blank file
 484     log_cm = vagrant.make_file_cm(logfilename)
 485     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 486
 487     if options.verbosity >= 2:
 488         tail = fdroidserver.tail.Tail(logfilename)
 489         tail.start()
 490
 491     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 492     if options.clean:
 493         vm.destroy()
 494
 495     # Check against the existing Vagrantfile.yaml, and if they differ, we
 496     # need to create a new box:
 497     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 498     writevf = True
 499     if os.path.exists(vf):
 500         logger.info('Halting %s', serverdir)
 501         v.halt()
 502         with open(vf, 'r', encoding='utf-8') as f:
 503             oldconfig = yaml.load(f)
 504         if config != oldconfig:
 505             logger.info("Server configuration has changed, rebuild from scratch is required")
 506             vm.destroy()
 507         else:
 508             logger.info("Re-provisioning existing server")
 509             writevf = False
 510     else:
 511         logger.info("No existing server - building from scratch")
 512     if writevf:
 513         with open(vf, 'w', encoding='utf-8') as f:
 514             yaml.dump(config, f)
 515
 516     if config['vm_provider'] == 'libvirt':
 517         found_basebox = False
 518         needs_mutate = False
 519         for box in v.box_list():
 520             if box.name == config['basebox']:
 521                 found_basebox = True
 522                 if box.provider != 'libvirt':
 523                     needs_mutate = True
 524                 continue
 525         if not found_basebox:
 526             if isinstance(config['baseboxurl'], str):
 527                 baseboxurl = config['baseboxurl']
 528             else:
 529                 baseboxurl = config['baseboxurl'][0]
 530             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 531             v.box_add(config['basebox'], baseboxurl)
 532             needs_mutate = True
 533         if needs_mutate:
 534             logger.info('Converting %s to libvirt format', config['basebox'])
 535             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 536             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 537             v.box_remove(config['basebox'], 'virtualbox')
 538
 539     logger.info("Configuring build server VM")
 540     debug_log_vagrant_vm(serverdir, config)
 541     try:
 542         v.up(provision=True)
 543     except subprocess.CalledProcessError:
 544         debug_log_vagrant_vm(serverdir, config)
 545         logger.error("'vagrant up' failed, is the base box missing?")
 546         sys.exit(1)
 547
 548     if config['copy_caches_from_host']:
 549         ssh_config = v.ssh_config()
 550         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 551         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 552         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 553         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 554
 555         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 556             fullpath = os.path.join(os.getenv('HOME'), d)
 557             if os.path.isdir(fullpath):
 558                 ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
 559                                         '-o StrictHostKeyChecking=no',
 560                                         '-o UserKnownHostsFile=/dev/null',
 561                                         '-o LogLevel=FATAL',
 562                                         '-o IdentitiesOnly=yes',
 563                                         '-o PasswordAuthentication=no'))
 564                 # TODO vagrant 1.5+ provides `vagrant rsync`
 565                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 566                 subprocess.call(['rsync', '-ax', '--delete', '-e',
 567                                  ssh_command,
 568                                  fullpath + '/',
 569                                  user + '@' + hostname + ':~/' + d + '/'])
 570
 571         # this file changes every time but should not be cached
 572         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 573         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 574
 575     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 576                          universal_newlines=True)
 577     buildserverid = p.communicate()[0].strip()
 578     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 579     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 580
 581     logger.info("Stopping build server VM")
 582     v.halt()
 583
 584     logger.info("Packaging")
 585     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 586     if os.path.exists(boxfile):
 587         os.remove(boxfile)
 588
 589     vm.package(output=boxfile)
 590
 591     logger.info("Adding box")
 592     vm.box_add('buildserver', boxfile, force=True)
 593
 594     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 595         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 596         sys.exit(1)
 597
 598     if not options.keep_box_file:
 599         logger.debug('box added to vagrant, ' +
 600                      'removing generated box file \'%s\'',
 601                      boxfile)
 602         os.remove(boxfile)
 603
 604
 605 if __name__ == '__main__':
 606     try:
 607         main()
 608     finally:
 609         if tail is not None:
 610             tail.stop()