makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/platform-27_r01.zip',
 182      'cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac'),
 183     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 184      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 186      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 188      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 189     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 190      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 191     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 192      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 194      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 196      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 198      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 199     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 200      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 201     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 202      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 203     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 204      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 206      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 208      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 210      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 212      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 213     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 214      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 215     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 216      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 217     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 218      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 219     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 220      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 222      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 224      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 225     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 226      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 227     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 228      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 230      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 232      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 233     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 234      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 235     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 236      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 238      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 240      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 241     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 242      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 243     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 244      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 246      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 248      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 249     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 250      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 251     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 252      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 253     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 254      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 255     # the binaries that Google uses are here:
 256     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 257     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 258      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 259     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 260      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 261     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 262      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 263     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 264      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 265     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 266      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 267     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 268      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 269     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 270      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 271     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 272      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 273     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 274      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 275     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 276      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 277     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 278      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 279     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 280      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 281     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 282      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 283     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 284      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 285     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 286      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 287     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 288      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 289     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 290      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 291     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 292      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 293     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 294      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 295     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 296      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 297     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 298      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 299     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 300      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 301     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 302      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 303     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 304      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 305     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 306      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 307     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 308      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 309     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 310      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 311     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 312      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 313     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 314      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 315     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 316      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 317     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 318      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 319     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 320      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 321     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 322      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 323     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 324      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 325     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 326      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 327     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 328      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 329     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 330      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 331     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 332      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 333     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 334      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 335     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 336      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 337     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 338      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 339     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 340      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 341     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 342      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 343     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 344      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 345     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 346      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 347     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 348      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 349     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 350      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 351     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 352      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 353     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 354      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 355 ]
 356
 357
 358 def sha256_for_file(path):
 359     with open(path, 'rb') as f:
 360         s = hashlib.sha256()
 361         while True:
 362             data = f.read(4096)
 363             if not data:
 364                 break
 365             s.update(data)
 366         return s.hexdigest()
 367
 368
 369 def run_via_vagrant_ssh(v, cmdlist):
 370     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 371         cmd = cmdlist
 372     else:
 373         cmd = ' '.join(cmdlist)
 374     v._run_vagrant_command(['ssh', '-c', cmd])
 375
 376
 377 def update_cache(cachedir, cachefiles):
 378     for srcurl, shasum in cachefiles:
 379         filename = os.path.basename(srcurl)
 380         local_filename = os.path.join(cachedir, filename)
 381
 382         if os.path.exists(local_filename):
 383             local_length = os.path.getsize(local_filename)
 384         else:
 385             local_length = -1
 386
 387         resume_header = {}
 388         download = True
 389
 390         try:
 391             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 392             if r.status_code == 200:
 393                 content_length = int(r.headers.get('content-length'))
 394             else:
 395                 content_length = local_length  # skip the download
 396         except requests.exceptions.RequestException as e:
 397             content_length = local_length  # skip the download
 398             logger.warn('%s', e)
 399
 400         if local_length == content_length:
 401             download = False
 402         elif local_length > content_length:
 403             logger.info('deleting corrupt file from cache: %s', local_filename)
 404             os.remove(local_filename)
 405             logger.info("Downloading %s to cache", filename)
 406         elif local_length > -1 and local_length < content_length:
 407             logger.info("Resuming download of %s", local_filename)
 408             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 409         else:
 410             logger.info("Downloading %s to cache", filename)
 411
 412         if download:
 413             r = requests.get(srcurl, headers=resume_header,
 414                              stream=True, allow_redirects=True)
 415             content_length = int(r.headers.get('content-length'))
 416             with open(local_filename, 'ab') as f:
 417                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 418                                           expected_size=(content_length / 65536) + 1):
 419                     if chunk:  # filter out keep-alive new chunks
 420                         f.write(chunk)
 421
 422         v = sha256_for_file(local_filename)
 423         if v == shasum:
 424             logger.info("\t...shasum verified for %s", local_filename)
 425         else:
 426             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 427             os.remove(local_filename)
 428             sys.exit(1)
 429
 430
 431 def debug_log_vagrant_vm(vm_dir, config):
 432     if options.verbosity >= 3:
 433         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 434         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 435         logger.debug('> vagrant status')
 436         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 437         logger.debug('> vagrant box list')
 438         subprocess.call(['vagrant', 'box', 'list'])
 439         if config['vm_provider'] == 'libvirt':
 440             logger.debug('> virsh -c qmeu:///system list --all')
 441             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 442             domain = 'buildserver_default'
 443             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 444             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 445
 446
 447 def main():
 448     global cachedir, cachefiles, config, tail
 449
 450     if options.skip_cache_update:
 451         logger.info('skipping cache update and verification...')
 452     else:
 453         update_cache(cachedir, cachefiles)
 454
 455     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 456     logger.info("Setting executable bit for %s", local_qt_filename)
 457     os.chmod(local_qt_filename, 0o755)
 458
 459     # use VirtualBox software virtualization if hardware is not available,
 460     # like if this is being run in kvm or some other VM platform, like
 461     # http://jenkins.debian.net, the values are 'on' or 'off'
 462     if sys.platform.startswith('darwin'):
 463         # all < 10 year old Macs work, and OSX servers as VM host are very
 464         # rare, but this could also be auto-detected if someone codes it
 465         config['hwvirtex'] = 'on'
 466         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 467     elif os.path.exists('/proc/cpuinfo'):
 468         with open('/proc/cpuinfo') as f:
 469             contents = f.read()
 470         if 'vmx' in contents or 'svm' in contents:
 471             config['hwvirtex'] = 'on'
 472             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 473
 474     serverdir = os.path.join(os.getcwd(), 'buildserver')
 475     logfilename = os.path.join(serverdir, 'up.log')
 476     if not os.path.exists(logfilename):
 477         open(logfilename, 'a').close()  # create blank file
 478     log_cm = vagrant.make_file_cm(logfilename)
 479     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 480
 481     if options.verbosity >= 2:
 482         tail = fdroidserver.tail.Tail(logfilename)
 483         tail.start()
 484
 485     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 486     if options.clean:
 487         vm.destroy()
 488
 489     # Check against the existing Vagrantfile.yaml, and if they differ, we
 490     # need to create a new box:
 491     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 492     writevf = True
 493     if os.path.exists(vf):
 494         logger.info('Halting %s', serverdir)
 495         v.halt()
 496         with open(vf, 'r', encoding='utf-8') as f:
 497             oldconfig = yaml.load(f)
 498         if config != oldconfig:
 499             logger.info("Server configuration has changed, rebuild from scratch is required")
 500             vm.destroy()
 501         else:
 502             logger.info("Re-provisioning existing server")
 503             writevf = False
 504     else:
 505         logger.info("No existing server - building from scratch")
 506     if writevf:
 507         with open(vf, 'w', encoding='utf-8') as f:
 508             yaml.dump(config, f)
 509
 510     if config['vm_provider'] == 'libvirt':
 511         found_basebox = False
 512         needs_mutate = False
 513         for box in v.box_list():
 514             if box.name == config['basebox']:
 515                 found_basebox = True
 516                 if box.provider != 'libvirt':
 517                     needs_mutate = True
 518                 continue
 519         if not found_basebox:
 520             if isinstance(config['baseboxurl'], str):
 521                 baseboxurl = config['baseboxurl']
 522             else:
 523                 baseboxurl = config['baseboxurl'][0]
 524             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 525             v.box_add(config['basebox'], baseboxurl)
 526             needs_mutate = True
 527         if needs_mutate:
 528             logger.info('Converting %s to libvirt format', config['basebox'])
 529             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 530             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 531             v.box_remove(config['basebox'], 'virtualbox')
 532
 533     logger.info("Configuring build server VM")
 534     debug_log_vagrant_vm(serverdir, config)
 535     try:
 536         v.up(provision=True)
 537     except subprocess.CalledProcessError:
 538         debug_log_vagrant_vm(serverdir, config)
 539         logger.error("'vagrant up' failed, is the base box missing?")
 540         sys.exit(1)
 541
 542     if config['copy_caches_from_host']:
 543         ssh_config = v.ssh_config()
 544         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 545         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 546         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 547         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 548
 549         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 550             fullpath = os.path.join(os.getenv('HOME'), d)
 551             if os.path.isdir(fullpath):
 552                 # TODO newer versions of vagrant provide `vagrant rsync`
 553                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 554                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 555                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 556                                  fullpath + '/',
 557                                  user + '@' + hostname + ':~/' + d + '/'])
 558
 559         # this file changes every time but should not be cached
 560         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 561         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 562
 563     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 564                          universal_newlines=True)
 565     buildserverid = p.communicate()[0].strip()
 566     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 567     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 568
 569     logger.info("Stopping build server VM")
 570     v.halt()
 571
 572     logger.info("Packaging")
 573     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 574     if os.path.exists(boxfile):
 575         os.remove(boxfile)
 576
 577     vm.package(output=boxfile)
 578
 579     logger.info("Adding box")
 580     vm.box_add('buildserver', boxfile, force=True)
 581
 582     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 583         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 584         sys.exit(1)
 585
 586     if not options.keep_box_file:
 587         logger.debug('box added to vagrant, ' +
 588                      'removing generated box file \'%s\'',
 589                      boxfile)
 590         os.remove(boxfile)
 591
 592
 593 if __name__ == '__main__':
 594     try:
 595         main()
 596     finally:
 597         if tail is not None:
 598             tail.stop()