makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/platform-27_r01.zip',
 182      'cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac'),
 183     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 184      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 186      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 188      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 189     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 190      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 191     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 192      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 194      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 196      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 198      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 199     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 200      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 201     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 202      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 203     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 204      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 206      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 208      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 210      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 212      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 213     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 214      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 215     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 216      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 217     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 218      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 219     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 220      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 222      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 224      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 225     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 226      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 227     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 228      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 230      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 232      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 233     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 234      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 235     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 236      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 238      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 240      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 241     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 242      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 243     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 244      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 246      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 248      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 249     # the binaries that Google uses are here:
 250     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 251     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 252      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 253     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 254      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 255     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 256      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 257     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 258      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 259     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 260      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 261     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 262      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 263     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 264      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 265     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 266      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 267     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 268      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 269     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 270      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 271     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 272      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 273     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 274      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 275     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 276      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 277     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 278      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 279     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 280      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 281     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 282      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 283     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 284      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 285     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 286      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 287     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 288      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 289     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 290      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 291     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 292      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 293     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 294      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 295     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 296      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 297     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 298      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 299     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 300      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 301     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 302      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 303     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 304      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 305     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 306      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 307     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 308      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 309     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 310      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 311     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 312      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 313     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 314      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 315     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 316      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 317     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 318      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 319     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 320      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 321     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 322      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 323     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 324      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 325     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 326      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 327     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 328      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 329     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 330      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 331     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 332      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 333     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 334      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 335     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 336      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 337     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 338      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 339     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 340      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 341     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 342      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 343     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 344      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 345     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 346      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 347 ]
 348
 349
 350 def sha256_for_file(path):
 351     with open(path, 'rb') as f:
 352         s = hashlib.sha256()
 353         while True:
 354             data = f.read(4096)
 355             if not data:
 356                 break
 357             s.update(data)
 358         return s.hexdigest()
 359
 360
 361 def run_via_vagrant_ssh(v, cmdlist):
 362     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 363         cmd = cmdlist
 364     else:
 365         cmd = ' '.join(cmdlist)
 366     v._run_vagrant_command(['ssh', '-c', cmd])
 367
 368
 369 def update_cache(cachedir, cachefiles):
 370     for srcurl, shasum in cachefiles:
 371         filename = os.path.basename(srcurl)
 372         local_filename = os.path.join(cachedir, filename)
 373
 374         if os.path.exists(local_filename):
 375             local_length = os.path.getsize(local_filename)
 376         else:
 377             local_length = -1
 378
 379         resume_header = {}
 380         download = True
 381
 382         try:
 383             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 384             if r.status_code == 200:
 385                 content_length = int(r.headers.get('content-length'))
 386             else:
 387                 content_length = local_length  # skip the download
 388         except requests.exceptions.RequestException as e:
 389             content_length = local_length  # skip the download
 390             logger.warn('%s', e)
 391
 392         if local_length == content_length:
 393             download = False
 394         elif local_length > content_length:
 395             logger.info('deleting corrupt file from cache: %s', local_filename)
 396             os.remove(local_filename)
 397             logger.info("Downloading %s to cache", filename)
 398         elif local_length > -1 and local_length < content_length:
 399             logger.info("Resuming download of %s", local_filename)
 400             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 401         else:
 402             logger.info("Downloading %s to cache", filename)
 403
 404         if download:
 405             r = requests.get(srcurl, headers=resume_header,
 406                              stream=True, allow_redirects=True)
 407             content_length = int(r.headers.get('content-length'))
 408             with open(local_filename, 'ab') as f:
 409                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 410                                           expected_size=(content_length / 65536) + 1):
 411                     if chunk:  # filter out keep-alive new chunks
 412                         f.write(chunk)
 413
 414         v = sha256_for_file(local_filename)
 415         if v == shasum:
 416             logger.info("\t...shasum verified for %s", local_filename)
 417         else:
 418             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 419             os.remove(local_filename)
 420             sys.exit(1)
 421
 422
 423 def debug_log_vagrant_vm(vm_dir, config):
 424     if options.verbosity >= 3:
 425         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 426         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 427         logger.debug('> vagrant status')
 428         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 429         logger.debug('> vagrant box list')
 430         subprocess.call(['vagrant', 'box', 'list'])
 431         if config['vm_provider'] == 'libvirt':
 432             logger.debug('> virsh -c qmeu:///system list --all')
 433             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 434             domain = 'buildserver_default'
 435             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 436             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 437
 438
 439 def main():
 440     global cachedir, cachefiles, config, tail
 441
 442     if options.skip_cache_update:
 443         logger.info('skipping cache update and verification...')
 444     else:
 445         update_cache(cachedir, cachefiles)
 446
 447     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 448     logger.info("Setting executable bit for %s", local_qt_filename)
 449     os.chmod(local_qt_filename, 0o755)
 450
 451     # use VirtualBox software virtualization if hardware is not available,
 452     # like if this is being run in kvm or some other VM platform, like
 453     # http://jenkins.debian.net, the values are 'on' or 'off'
 454     if sys.platform.startswith('darwin'):
 455         # all < 10 year old Macs work, and OSX servers as VM host are very
 456         # rare, but this could also be auto-detected if someone codes it
 457         config['hwvirtex'] = 'on'
 458         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 459     elif os.path.exists('/proc/cpuinfo'):
 460         with open('/proc/cpuinfo') as f:
 461             contents = f.read()
 462         if 'vmx' in contents or 'svm' in contents:
 463             config['hwvirtex'] = 'on'
 464             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 465
 466     serverdir = os.path.join(os.getcwd(), 'buildserver')
 467     logfilename = os.path.join(serverdir, 'up.log')
 468     if not os.path.exists(logfilename):
 469         open(logfilename, 'a').close()  # create blank file
 470     log_cm = vagrant.make_file_cm(logfilename)
 471     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 472
 473     if options.verbosity >= 2:
 474         tail = fdroidserver.tail.Tail(logfilename)
 475         tail.start()
 476
 477     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 478     if options.clean:
 479         vm.destroy()
 480
 481     # Check against the existing Vagrantfile.yaml, and if they differ, we
 482     # need to create a new box:
 483     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 484     writevf = True
 485     if os.path.exists(vf):
 486         logger.info('Halting %s', serverdir)
 487         v.halt()
 488         with open(vf, 'r', encoding='utf-8') as f:
 489             oldconfig = yaml.load(f)
 490         if config != oldconfig:
 491             logger.info("Server configuration has changed, rebuild from scratch is required")
 492             vm.destroy()
 493         else:
 494             logger.info("Re-provisioning existing server")
 495             writevf = False
 496     else:
 497         logger.info("No existing server - building from scratch")
 498     if writevf:
 499         with open(vf, 'w', encoding='utf-8') as f:
 500             yaml.dump(config, f)
 501
 502     if config['vm_provider'] == 'libvirt':
 503         found_basebox = False
 504         needs_mutate = False
 505         for box in v.box_list():
 506             if box.name == config['basebox']:
 507                 found_basebox = True
 508                 if box.provider != 'libvirt':
 509                     needs_mutate = True
 510                 continue
 511         if not found_basebox:
 512             if isinstance(config['baseboxurl'], str):
 513                 baseboxurl = config['baseboxurl']
 514             else:
 515                 baseboxurl = config['baseboxurl'][0]
 516             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 517             v.box_add(config['basebox'], baseboxurl)
 518             needs_mutate = True
 519         if needs_mutate:
 520             logger.info('Converting %s to libvirt format', config['basebox'])
 521             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 522             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 523             v.box_remove(config['basebox'], 'virtualbox')
 524
 525     logger.info("Configuring build server VM")
 526     debug_log_vagrant_vm(serverdir, config)
 527     try:
 528         v.up(provision=True)
 529     except subprocess.CalledProcessError:
 530         debug_log_vagrant_vm(serverdir, config)
 531         logger.error("'vagrant up' failed, is the base box missing?")
 532         sys.exit(1)
 533
 534     if config['copy_caches_from_host']:
 535         ssh_config = v.ssh_config()
 536         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 537         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 538         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 539         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 540
 541         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 542             fullpath = os.path.join(os.getenv('HOME'), d)
 543             if os.path.isdir(fullpath):
 544                 # TODO newer versions of vagrant provide `vagrant rsync`
 545                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 546                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 547                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 548                                  fullpath + '/',
 549                                  user + '@' + hostname + ':~/' + d + '/'])
 550
 551         # this file changes every time but should not be cached
 552         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 553         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 554
 555     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 556                          universal_newlines=True)
 557     buildserverid = p.communicate()[0].strip()
 558     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 559     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 560
 561     logger.info("Stopping build server VM")
 562     v.halt()
 563
 564     logger.info("Packaging")
 565     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 566     if os.path.exists(boxfile):
 567         os.remove(boxfile)
 568
 569     vm.package(output=boxfile)
 570
 571     logger.info("Adding box")
 572     vm.box_add('buildserver', boxfile, force=True)
 573
 574     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 575         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 576         sys.exit(1)
 577
 578     if not options.keep_box_file:
 579         logger.debug('box added to vagrant, ' +
 580                      'removing generated box file \'%s\'',
 581                      boxfile)
 582         os.remove(boxfile)
 583
 584
 585 if __name__ == '__main__':
 586     try:
 587         main()
 588     finally:
 589         if tail is not None:
 590             tail.stop()