makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/platform-27_r01.zip',
 182      'cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac'),
 183     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 184      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 186      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 188      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 189     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 190      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 191     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 192      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 194      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 196      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 198      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 199     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 200      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 201     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 202      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 203     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 204      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 206      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 208      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 210      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 212      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 213     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 214      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 215     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 216      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 217     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 218      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 219     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 220      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 222      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 224      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 225     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 226      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 227     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 228      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 230      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 232      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 233     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 234      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 235     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 236      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 238      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 240      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 241     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 242      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 243     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 244      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 246      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 248      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 249     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 250      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 251     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 252      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 253     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 254      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 255     # the binaries that Google uses are here:
 256     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 257     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 258      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 259     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 260      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 261     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 262      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 263     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 264      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 265     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 266      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 267     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 268      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 269     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 270      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 271     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 272      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 273     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 274      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 275     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 276      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 277     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 278      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 279     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 280      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 281     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 282      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 283     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 284      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 285     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 286      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 287     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 288      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 289     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 290      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 291     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 292      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 293     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 294      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 295     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 296      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 297     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 298      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 299     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 300      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 301     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 302      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 303     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 304      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 305     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 306      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 307     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 308      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 309     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 310      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 311     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 312      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 313     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 314      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 315     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 316      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 317     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 318      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 319     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 320      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 321     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 322      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 323     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 324      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 325     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 326      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 327     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 328      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 329     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 330      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 331     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 332      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 333     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 334      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 335     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 336      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 337     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 338      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 339     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 340      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 341     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 342      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 343     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 344      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 345     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 346      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 347     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 348      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 349     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 350      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 351     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 352      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 353     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 354      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 355     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 356      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 357 ]
 358
 359
 360 def sha256_for_file(path):
 361     with open(path, 'rb') as f:
 362         s = hashlib.sha256()
 363         while True:
 364             data = f.read(4096)
 365             if not data:
 366                 break
 367             s.update(data)
 368         return s.hexdigest()
 369
 370
 371 def run_via_vagrant_ssh(v, cmdlist):
 372     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 373         cmd = cmdlist
 374     else:
 375         cmd = ' '.join(cmdlist)
 376     v._run_vagrant_command(['ssh', '-c', cmd])
 377
 378
 379 def update_cache(cachedir, cachefiles):
 380     for srcurl, shasum in cachefiles:
 381         filename = os.path.basename(srcurl)
 382         local_filename = os.path.join(cachedir, filename)
 383
 384         if os.path.exists(local_filename):
 385             local_length = os.path.getsize(local_filename)
 386         else:
 387             local_length = -1
 388
 389         resume_header = {}
 390         download = True
 391
 392         try:
 393             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 394             if r.status_code == 200:
 395                 content_length = int(r.headers.get('content-length'))
 396             else:
 397                 content_length = local_length  # skip the download
 398         except requests.exceptions.RequestException as e:
 399             content_length = local_length  # skip the download
 400             logger.warn('%s', e)
 401
 402         if local_length == content_length:
 403             download = False
 404         elif local_length > content_length:
 405             logger.info('deleting corrupt file from cache: %s', local_filename)
 406             os.remove(local_filename)
 407             logger.info("Downloading %s to cache", filename)
 408         elif local_length > -1 and local_length < content_length:
 409             logger.info("Resuming download of %s", local_filename)
 410             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 411         else:
 412             logger.info("Downloading %s to cache", filename)
 413
 414         if download:
 415             r = requests.get(srcurl, headers=resume_header,
 416                              stream=True, allow_redirects=True)
 417             content_length = int(r.headers.get('content-length'))
 418             with open(local_filename, 'ab') as f:
 419                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 420                                           expected_size=(content_length / 65536) + 1):
 421                     if chunk:  # filter out keep-alive new chunks
 422                         f.write(chunk)
 423
 424         v = sha256_for_file(local_filename)
 425         if v == shasum:
 426             logger.info("\t...shasum verified for %s", local_filename)
 427         else:
 428             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 429             os.remove(local_filename)
 430             sys.exit(1)
 431
 432
 433 def debug_log_vagrant_vm(vm_dir, config):
 434     if options.verbosity >= 3:
 435         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 436         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 437         logger.debug('> vagrant status')
 438         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 439         logger.debug('> vagrant box list')
 440         subprocess.call(['vagrant', 'box', 'list'])
 441         if config['vm_provider'] == 'libvirt':
 442             logger.debug('> virsh -c qmeu:///system list --all')
 443             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 444             domain = 'buildserver_default'
 445             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 446             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 447
 448
 449 def main():
 450     global cachedir, cachefiles, config, tail
 451
 452     if options.skip_cache_update:
 453         logger.info('skipping cache update and verification...')
 454     else:
 455         update_cache(cachedir, cachefiles)
 456
 457     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 458     logger.info("Setting executable bit for %s", local_qt_filename)
 459     os.chmod(local_qt_filename, 0o755)
 460
 461     # use VirtualBox software virtualization if hardware is not available,
 462     # like if this is being run in kvm or some other VM platform, like
 463     # http://jenkins.debian.net, the values are 'on' or 'off'
 464     if sys.platform.startswith('darwin'):
 465         # all < 10 year old Macs work, and OSX servers as VM host are very
 466         # rare, but this could also be auto-detected if someone codes it
 467         config['hwvirtex'] = 'on'
 468         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 469     elif os.path.exists('/proc/cpuinfo'):
 470         with open('/proc/cpuinfo') as f:
 471             contents = f.read()
 472         if 'vmx' in contents or 'svm' in contents:
 473             config['hwvirtex'] = 'on'
 474             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 475
 476     serverdir = os.path.join(os.getcwd(), 'buildserver')
 477     logfilename = os.path.join(serverdir, 'up.log')
 478     if not os.path.exists(logfilename):
 479         open(logfilename, 'a').close()  # create blank file
 480     log_cm = vagrant.make_file_cm(logfilename)
 481     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 482
 483     if options.verbosity >= 2:
 484         tail = fdroidserver.tail.Tail(logfilename)
 485         tail.start()
 486
 487     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 488     if options.clean:
 489         vm.destroy()
 490
 491     # Check against the existing Vagrantfile.yaml, and if they differ, we
 492     # need to create a new box:
 493     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 494     writevf = True
 495     if os.path.exists(vf):
 496         logger.info('Halting %s', serverdir)
 497         v.halt()
 498         with open(vf, 'r', encoding='utf-8') as f:
 499             oldconfig = yaml.load(f)
 500         if config != oldconfig:
 501             logger.info("Server configuration has changed, rebuild from scratch is required")
 502             vm.destroy()
 503         else:
 504             logger.info("Re-provisioning existing server")
 505             writevf = False
 506     else:
 507         logger.info("No existing server - building from scratch")
 508     if writevf:
 509         with open(vf, 'w', encoding='utf-8') as f:
 510             yaml.dump(config, f)
 511
 512     if config['vm_provider'] == 'libvirt':
 513         found_basebox = False
 514         needs_mutate = False
 515         for box in v.box_list():
 516             if box.name == config['basebox']:
 517                 found_basebox = True
 518                 if box.provider != 'libvirt':
 519                     needs_mutate = True
 520                 continue
 521         if not found_basebox:
 522             if isinstance(config['baseboxurl'], str):
 523                 baseboxurl = config['baseboxurl']
 524             else:
 525                 baseboxurl = config['baseboxurl'][0]
 526             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 527             v.box_add(config['basebox'], baseboxurl)
 528             needs_mutate = True
 529         if needs_mutate:
 530             logger.info('Converting %s to libvirt format', config['basebox'])
 531             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 532             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 533             v.box_remove(config['basebox'], 'virtualbox')
 534
 535     logger.info("Configuring build server VM")
 536     debug_log_vagrant_vm(serverdir, config)
 537     try:
 538         v.up(provision=True)
 539     except subprocess.CalledProcessError:
 540         debug_log_vagrant_vm(serverdir, config)
 541         logger.error("'vagrant up' failed, is the base box missing?")
 542         sys.exit(1)
 543
 544     if config['copy_caches_from_host']:
 545         ssh_config = v.ssh_config()
 546         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 547         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 548         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 549         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 550
 551         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 552             fullpath = os.path.join(os.getenv('HOME'), d)
 553             if os.path.isdir(fullpath):
 554                 # TODO newer versions of vagrant provide `vagrant rsync`
 555                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 556                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 557                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 558                                  fullpath + '/',
 559                                  user + '@' + hostname + ':~/' + d + '/'])
 560
 561         # this file changes every time but should not be cached
 562         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 563         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 564
 565     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 566                          universal_newlines=True)
 567     buildserverid = p.communicate()[0].strip()
 568     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 569     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 570
 571     logger.info("Stopping build server VM")
 572     v.halt()
 573
 574     logger.info("Packaging")
 575     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 576     if os.path.exists(boxfile):
 577         os.remove(boxfile)
 578
 579     vm.package(output=boxfile)
 580
 581     logger.info("Adding box")
 582     vm.box_add('buildserver', boxfile, force=True)
 583
 584     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 585         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 586         sys.exit(1)
 587
 588     if not options.keep_box_file:
 589         logger.debug('box added to vagrant, ' +
 590                      'removing generated box file \'%s\'',
 591                      boxfile)
 592         os.remove(boxfile)
 593
 594
 595 if __name__ == '__main__':
 596     try:
 597         main()
 598     finally:
 599         if tail is not None:
 600             tail.stop()