makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/platform-27_r01.zip',
 182      'cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac'),
 183     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 184      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 186      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 188      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 189     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 190      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 191     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 192      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 194      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 196      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 198      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 199     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 200      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 201     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 202      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 203     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 204      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 206      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 208      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 210      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 212      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 213     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 214      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 215     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 216      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 217     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 218      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 219     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 220      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 222      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 224      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 225     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 226      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 227     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 228      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 230      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 232      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 233     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 234      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 235     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 236      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 238      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 240      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 241     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 242      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 243     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 244      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 246      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 248      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 249     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 250      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 251     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 252      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 253     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 254      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 255     # the binaries that Google uses are here:
 256     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 257     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 258      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 259     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 260      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 261     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 262      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 263     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 264      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 265     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 266      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 267     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 268      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 269     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 270      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 271     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 272      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 273     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 274      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 275     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 276      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 277     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 278      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 279     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 280      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 281     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 282      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 283     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 284      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 285     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 286      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 287     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 288      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 289     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 290      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 291     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 292      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 293     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 294      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 295     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 296      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 297     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 298      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 299     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 300      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 301     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 302      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 303     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 304      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 305     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 306      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 307     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 308      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 309     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 310      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 311     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 312      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 313     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 314      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 315     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 316      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 317     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 318      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 319     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 320      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 321     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 322      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 323     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 324      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 325     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 326      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 327     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 328      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 329     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 330      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 331     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 332      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 333     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 334      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 335     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 336      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 337     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 338      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 339     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 340      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 341     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 342      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 343     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 344      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 345     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 346      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 347     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 348      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 349     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 350      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 351     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 352      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 353     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 354      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 355     ('https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip',
 356      'a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e'),
 357     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 358      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 359 ]
 360
 361
 362 def sha256_for_file(path):
 363     with open(path, 'rb') as f:
 364         s = hashlib.sha256()
 365         while True:
 366             data = f.read(4096)
 367             if not data:
 368                 break
 369             s.update(data)
 370         return s.hexdigest()
 371
 372
 373 def run_via_vagrant_ssh(v, cmdlist):
 374     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 375         cmd = cmdlist
 376     else:
 377         cmd = ' '.join(cmdlist)
 378     v._run_vagrant_command(['ssh', '-c', cmd])
 379
 380
 381 def update_cache(cachedir, cachefiles):
 382     for srcurl, shasum in cachefiles:
 383         filename = os.path.basename(srcurl)
 384         local_filename = os.path.join(cachedir, filename)
 385
 386         if os.path.exists(local_filename):
 387             local_length = os.path.getsize(local_filename)
 388         else:
 389             local_length = -1
 390
 391         resume_header = {}
 392         download = True
 393
 394         try:
 395             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 396             if r.status_code == 200:
 397                 content_length = int(r.headers.get('content-length'))
 398             else:
 399                 content_length = local_length  # skip the download
 400         except requests.exceptions.RequestException as e:
 401             content_length = local_length  # skip the download
 402             logger.warn('%s', e)
 403
 404         if local_length == content_length:
 405             download = False
 406         elif local_length > content_length:
 407             logger.info('deleting corrupt file from cache: %s', local_filename)
 408             os.remove(local_filename)
 409             logger.info("Downloading %s to cache", filename)
 410         elif local_length > -1 and local_length < content_length:
 411             logger.info("Resuming download of %s", local_filename)
 412             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 413         else:
 414             logger.info("Downloading %s to cache", filename)
 415
 416         if download:
 417             r = requests.get(srcurl, headers=resume_header,
 418                              stream=True, allow_redirects=True)
 419             content_length = int(r.headers.get('content-length'))
 420             with open(local_filename, 'ab') as f:
 421                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 422                                           expected_size=(content_length / 65536) + 1):
 423                     if chunk:  # filter out keep-alive new chunks
 424                         f.write(chunk)
 425
 426         v = sha256_for_file(local_filename)
 427         if v == shasum:
 428             logger.info("\t...shasum verified for %s", local_filename)
 429         else:
 430             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 431             os.remove(local_filename)
 432             sys.exit(1)
 433
 434
 435 def debug_log_vagrant_vm(vm_dir, config):
 436     if options.verbosity >= 3:
 437         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 438         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 439         logger.debug('> vagrant status')
 440         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 441         logger.debug('> vagrant box list')
 442         subprocess.call(['vagrant', 'box', 'list'])
 443         if config['vm_provider'] == 'libvirt':
 444             logger.debug('> virsh -c qmeu:///system list --all')
 445             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 446             domain = 'buildserver_default'
 447             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 448             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 449
 450
 451 def main():
 452     global cachedir, cachefiles, config, tail
 453
 454     if options.skip_cache_update:
 455         logger.info('skipping cache update and verification...')
 456     else:
 457         update_cache(cachedir, cachefiles)
 458
 459     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 460     logger.info("Setting executable bit for %s", local_qt_filename)
 461     os.chmod(local_qt_filename, 0o755)
 462
 463     # use VirtualBox software virtualization if hardware is not available,
 464     # like if this is being run in kvm or some other VM platform, like
 465     # http://jenkins.debian.net, the values are 'on' or 'off'
 466     if sys.platform.startswith('darwin'):
 467         # all < 10 year old Macs work, and OSX servers as VM host are very
 468         # rare, but this could also be auto-detected if someone codes it
 469         config['hwvirtex'] = 'on'
 470         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 471     elif os.path.exists('/proc/cpuinfo'):
 472         with open('/proc/cpuinfo') as f:
 473             contents = f.read()
 474         if 'vmx' in contents or 'svm' in contents:
 475             config['hwvirtex'] = 'on'
 476             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 477
 478     serverdir = os.path.join(os.getcwd(), 'buildserver')
 479     logfilename = os.path.join(serverdir, 'up.log')
 480     if not os.path.exists(logfilename):
 481         open(logfilename, 'a').close()  # create blank file
 482     log_cm = vagrant.make_file_cm(logfilename)
 483     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 484
 485     if options.verbosity >= 2:
 486         tail = fdroidserver.tail.Tail(logfilename)
 487         tail.start()
 488
 489     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 490     if options.clean:
 491         vm.destroy()
 492
 493     # Check against the existing Vagrantfile.yaml, and if they differ, we
 494     # need to create a new box:
 495     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 496     writevf = True
 497     if os.path.exists(vf):
 498         logger.info('Halting %s', serverdir)
 499         v.halt()
 500         with open(vf, 'r', encoding='utf-8') as f:
 501             oldconfig = yaml.load(f)
 502         if config != oldconfig:
 503             logger.info("Server configuration has changed, rebuild from scratch is required")
 504             vm.destroy()
 505         else:
 506             logger.info("Re-provisioning existing server")
 507             writevf = False
 508     else:
 509         logger.info("No existing server - building from scratch")
 510     if writevf:
 511         with open(vf, 'w', encoding='utf-8') as f:
 512             yaml.dump(config, f)
 513
 514     if config['vm_provider'] == 'libvirt':
 515         found_basebox = False
 516         needs_mutate = False
 517         for box in v.box_list():
 518             if box.name == config['basebox']:
 519                 found_basebox = True
 520                 if box.provider != 'libvirt':
 521                     needs_mutate = True
 522                 continue
 523         if not found_basebox:
 524             if isinstance(config['baseboxurl'], str):
 525                 baseboxurl = config['baseboxurl']
 526             else:
 527                 baseboxurl = config['baseboxurl'][0]
 528             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 529             v.box_add(config['basebox'], baseboxurl)
 530             needs_mutate = True
 531         if needs_mutate:
 532             logger.info('Converting %s to libvirt format', config['basebox'])
 533             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 534             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 535             v.box_remove(config['basebox'], 'virtualbox')
 536
 537     logger.info("Configuring build server VM")
 538     debug_log_vagrant_vm(serverdir, config)
 539     try:
 540         v.up(provision=True)
 541     except subprocess.CalledProcessError:
 542         debug_log_vagrant_vm(serverdir, config)
 543         logger.error("'vagrant up' failed, is the base box missing?")
 544         sys.exit(1)
 545
 546     if config['copy_caches_from_host']:
 547         ssh_config = v.ssh_config()
 548         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 549         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 550         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 551         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 552
 553         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 554             fullpath = os.path.join(os.getenv('HOME'), d)
 555             if os.path.isdir(fullpath):
 556                 # TODO newer versions of vagrant provide `vagrant rsync`
 557                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 558                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 559                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 560                                  fullpath + '/',
 561                                  user + '@' + hostname + ':~/' + d + '/'])
 562
 563         # this file changes every time but should not be cached
 564         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 565         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 566
 567     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 568                          universal_newlines=True)
 569     buildserverid = p.communicate()[0].strip()
 570     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 571     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 572
 573     logger.info("Stopping build server VM")
 574     v.halt()
 575
 576     logger.info("Packaging")
 577     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 578     if os.path.exists(boxfile):
 579         os.remove(boxfile)
 580
 581     vm.package(output=boxfile)
 582
 583     logger.info("Adding box")
 584     vm.box_add('buildserver', boxfile, force=True)
 585
 586     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 587         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 588         sys.exit(1)
 589
 590     if not options.keep_box_file:
 591         logger.debug('box added to vagrant, ' +
 592                      'removing generated box file \'%s\'',
 593                      boxfile)
 594         os.remove(boxfile)
 595
 596
 597 if __name__ == '__main__':
 598     try:
 599         main()
 600     finally:
 601         if tail is not None:
 602             tail.stop()