makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/platform-27_r01.zip',
 182      'cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac'),
 183     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 184      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 186      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 188      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 189     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 190      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 191     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 192      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 194      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 196      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 198      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 199     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 200      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 201     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 202      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 203     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 204      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 206      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 208      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 210      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 212      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 213     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 214      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 215     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 216      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 217     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 218      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 219     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 220      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 222      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 224      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 225     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 226      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 227     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 228      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 230      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 232      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 233     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 234      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 235     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 236      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 238      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 240      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 241     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 242      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 243     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 244      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 246      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 248      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 249     # the binaries that Google uses are here:
 250     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 251     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 252      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 253     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 254      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 255     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 256      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 257     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 258      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 259     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 260      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 261     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 262      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 263     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 264      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 265     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 266      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 267     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 268      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 269     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 270      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 271     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 272      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 273     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 274      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 275     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 276      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 277     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 278      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 279     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 280      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 281     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 282      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 283     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 284      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 285     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 286      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 287     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 288      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 289     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 290      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 291     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 292      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 293     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 294      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 295     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 296      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 297     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 298      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 299     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 300      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 301     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 302      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 303     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 304      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 305     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 306      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 307     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 308      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 309     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 310      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 311     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 312      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 313     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 314      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 315     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 316      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 317     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 318      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 319     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 320      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 321     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 322      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 323     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 324      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 325     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 326      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 327     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 328      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 329     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 330      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 331     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 332      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 333     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 334      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 335     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 336      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 337     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 338      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 339     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 340      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 341     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 342      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 343     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 344      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 345     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 346      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 347     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 348      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 349 ]
 350
 351
 352 def sha256_for_file(path):
 353     with open(path, 'rb') as f:
 354         s = hashlib.sha256()
 355         while True:
 356             data = f.read(4096)
 357             if not data:
 358                 break
 359             s.update(data)
 360         return s.hexdigest()
 361
 362
 363 def run_via_vagrant_ssh(v, cmdlist):
 364     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 365         cmd = cmdlist
 366     else:
 367         cmd = ' '.join(cmdlist)
 368     v._run_vagrant_command(['ssh', '-c', cmd])
 369
 370
 371 def update_cache(cachedir, cachefiles):
 372     for srcurl, shasum in cachefiles:
 373         filename = os.path.basename(srcurl)
 374         local_filename = os.path.join(cachedir, filename)
 375
 376         if os.path.exists(local_filename):
 377             local_length = os.path.getsize(local_filename)
 378         else:
 379             local_length = -1
 380
 381         resume_header = {}
 382         download = True
 383
 384         try:
 385             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 386             if r.status_code == 200:
 387                 content_length = int(r.headers.get('content-length'))
 388             else:
 389                 content_length = local_length  # skip the download
 390         except requests.exceptions.RequestException as e:
 391             content_length = local_length  # skip the download
 392             logger.warn('%s', e)
 393
 394         if local_length == content_length:
 395             download = False
 396         elif local_length > content_length:
 397             logger.info('deleting corrupt file from cache: %s', local_filename)
 398             os.remove(local_filename)
 399             logger.info("Downloading %s to cache", filename)
 400         elif local_length > -1 and local_length < content_length:
 401             logger.info("Resuming download of %s", local_filename)
 402             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 403         else:
 404             logger.info("Downloading %s to cache", filename)
 405
 406         if download:
 407             r = requests.get(srcurl, headers=resume_header,
 408                              stream=True, allow_redirects=True)
 409             content_length = int(r.headers.get('content-length'))
 410             with open(local_filename, 'ab') as f:
 411                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 412                                           expected_size=(content_length / 65536) + 1):
 413                     if chunk:  # filter out keep-alive new chunks
 414                         f.write(chunk)
 415
 416         v = sha256_for_file(local_filename)
 417         if v == shasum:
 418             logger.info("\t...shasum verified for %s", local_filename)
 419         else:
 420             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 421             os.remove(local_filename)
 422             sys.exit(1)
 423
 424
 425 def debug_log_vagrant_vm(vm_dir, config):
 426     if options.verbosity >= 3:
 427         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 428         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 429         logger.debug('> vagrant status')
 430         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 431         logger.debug('> vagrant box list')
 432         subprocess.call(['vagrant', 'box', 'list'])
 433         if config['vm_provider'] == 'libvirt':
 434             logger.debug('> virsh -c qmeu:///system list --all')
 435             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 436             domain = 'buildserver_default'
 437             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 438             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 439
 440
 441 def main():
 442     global cachedir, cachefiles, config, tail
 443
 444     if options.skip_cache_update:
 445         logger.info('skipping cache update and verification...')
 446     else:
 447         update_cache(cachedir, cachefiles)
 448
 449     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 450     logger.info("Setting executable bit for %s", local_qt_filename)
 451     os.chmod(local_qt_filename, 0o755)
 452
 453     # use VirtualBox software virtualization if hardware is not available,
 454     # like if this is being run in kvm or some other VM platform, like
 455     # http://jenkins.debian.net, the values are 'on' or 'off'
 456     if sys.platform.startswith('darwin'):
 457         # all < 10 year old Macs work, and OSX servers as VM host are very
 458         # rare, but this could also be auto-detected if someone codes it
 459         config['hwvirtex'] = 'on'
 460         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 461     elif os.path.exists('/proc/cpuinfo'):
 462         with open('/proc/cpuinfo') as f:
 463             contents = f.read()
 464         if 'vmx' in contents or 'svm' in contents:
 465             config['hwvirtex'] = 'on'
 466             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 467
 468     serverdir = os.path.join(os.getcwd(), 'buildserver')
 469     logfilename = os.path.join(serverdir, 'up.log')
 470     if not os.path.exists(logfilename):
 471         open(logfilename, 'a').close()  # create blank file
 472     log_cm = vagrant.make_file_cm(logfilename)
 473     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 474
 475     if options.verbosity >= 2:
 476         tail = fdroidserver.tail.Tail(logfilename)
 477         tail.start()
 478
 479     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 480     if options.clean:
 481         vm.destroy()
 482
 483     # Check against the existing Vagrantfile.yaml, and if they differ, we
 484     # need to create a new box:
 485     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 486     writevf = True
 487     if os.path.exists(vf):
 488         logger.info('Halting %s', serverdir)
 489         v.halt()
 490         with open(vf, 'r', encoding='utf-8') as f:
 491             oldconfig = yaml.load(f)
 492         if config != oldconfig:
 493             logger.info("Server configuration has changed, rebuild from scratch is required")
 494             vm.destroy()
 495         else:
 496             logger.info("Re-provisioning existing server")
 497             writevf = False
 498     else:
 499         logger.info("No existing server - building from scratch")
 500     if writevf:
 501         with open(vf, 'w', encoding='utf-8') as f:
 502             yaml.dump(config, f)
 503
 504     if config['vm_provider'] == 'libvirt':
 505         found_basebox = False
 506         needs_mutate = False
 507         for box in v.box_list():
 508             if box.name == config['basebox']:
 509                 found_basebox = True
 510                 if box.provider != 'libvirt':
 511                     needs_mutate = True
 512                 continue
 513         if not found_basebox:
 514             if isinstance(config['baseboxurl'], str):
 515                 baseboxurl = config['baseboxurl']
 516             else:
 517                 baseboxurl = config['baseboxurl'][0]
 518             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 519             v.box_add(config['basebox'], baseboxurl)
 520             needs_mutate = True
 521         if needs_mutate:
 522             logger.info('Converting %s to libvirt format', config['basebox'])
 523             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 524             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 525             v.box_remove(config['basebox'], 'virtualbox')
 526
 527     logger.info("Configuring build server VM")
 528     debug_log_vagrant_vm(serverdir, config)
 529     try:
 530         v.up(provision=True)
 531     except subprocess.CalledProcessError:
 532         debug_log_vagrant_vm(serverdir, config)
 533         logger.error("'vagrant up' failed, is the base box missing?")
 534         sys.exit(1)
 535
 536     if config['copy_caches_from_host']:
 537         ssh_config = v.ssh_config()
 538         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 539         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 540         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 541         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 542
 543         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 544             fullpath = os.path.join(os.getenv('HOME'), d)
 545             if os.path.isdir(fullpath):
 546                 # TODO newer versions of vagrant provide `vagrant rsync`
 547                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 548                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 549                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 550                                  fullpath + '/',
 551                                  user + '@' + hostname + ':~/' + d + '/'])
 552
 553         # this file changes every time but should not be cached
 554         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 555         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 556
 557     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 558                          universal_newlines=True)
 559     buildserverid = p.communicate()[0].strip()
 560     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 561     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 562
 563     logger.info("Stopping build server VM")
 564     v.halt()
 565
 566     logger.info("Packaging")
 567     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 568     if os.path.exists(boxfile):
 569         os.remove(boxfile)
 570
 571     vm.package(output=boxfile)
 572
 573     logger.info("Adding box")
 574     vm.box_add('buildserver', boxfile, force=True)
 575
 576     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 577         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 578         sys.exit(1)
 579
 580     if not options.keep_box_file:
 581         logger.debug('box added to vagrant, ' +
 582                      'removing generated box file \'%s\'',
 583                      boxfile)
 584         os.remove(boxfile)
 585
 586
 587 if __name__ == '__main__':
 588     try:
 589         main()
 590     finally:
 591         if tail is not None:
 592             tail.stop()