makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/platform-27_r01.zip',
 182      'cbba6f8fcf025e1b533326746763aa1d6e2cf4001b1b441602bb44d253bc49ac'),
 183     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 184      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 186      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 188      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 189     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 190      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 191     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 192      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 194      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 196      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 198      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 199     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 200      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 201     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 202      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 203     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 204      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 206      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 208      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 210      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 212      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 213     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 214      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 215     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 216      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 217     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 218      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 219     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 220      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 222      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 224      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 225     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 226      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 227     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 228      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 230      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 232      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 233     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 234      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 235     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 236      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 238      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 240      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 241     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 242      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 243     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 244      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 246      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 248      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 249     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 250      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 251     # the binaries that Google uses are here:
 252     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 253     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 254      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 255     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 256      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 257     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 258      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 259     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 260      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 261     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 262      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 263     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 264      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 265     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 266      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 267     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 268      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 269     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 270      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 271     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 272      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 273     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 274      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 275     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 276      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 277     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 278      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 279     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 280      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 281     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 282      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 283     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 284      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 285     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 286      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 287     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 288      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 289     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 290      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 291     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 292      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 293     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 294      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 295     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 296      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 297     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 298      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 299     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 300      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 301     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 302      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 303     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 304      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 305     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 306      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 307     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 308      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 309     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 310      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 311     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 312      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 313     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 314      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 315     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 316      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 317     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 318      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 319     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 320      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 321     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 322      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 323     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 324      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 325     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 326      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 327     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 328      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 329     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 330      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 331     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 332      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 333     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 334      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 335     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 336      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 337     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 338      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 339     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 340      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 341     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 342      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 343     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 344      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 345     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 346      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 347     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 348      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 349     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 350      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 351 ]
 352
 353
 354 def sha256_for_file(path):
 355     with open(path, 'rb') as f:
 356         s = hashlib.sha256()
 357         while True:
 358             data = f.read(4096)
 359             if not data:
 360                 break
 361             s.update(data)
 362         return s.hexdigest()
 363
 364
 365 def run_via_vagrant_ssh(v, cmdlist):
 366     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 367         cmd = cmdlist
 368     else:
 369         cmd = ' '.join(cmdlist)
 370     v._run_vagrant_command(['ssh', '-c', cmd])
 371
 372
 373 def update_cache(cachedir, cachefiles):
 374     for srcurl, shasum in cachefiles:
 375         filename = os.path.basename(srcurl)
 376         local_filename = os.path.join(cachedir, filename)
 377
 378         if os.path.exists(local_filename):
 379             local_length = os.path.getsize(local_filename)
 380         else:
 381             local_length = -1
 382
 383         resume_header = {}
 384         download = True
 385
 386         try:
 387             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 388             if r.status_code == 200:
 389                 content_length = int(r.headers.get('content-length'))
 390             else:
 391                 content_length = local_length  # skip the download
 392         except requests.exceptions.RequestException as e:
 393             content_length = local_length  # skip the download
 394             logger.warn('%s', e)
 395
 396         if local_length == content_length:
 397             download = False
 398         elif local_length > content_length:
 399             logger.info('deleting corrupt file from cache: %s', local_filename)
 400             os.remove(local_filename)
 401             logger.info("Downloading %s to cache", filename)
 402         elif local_length > -1 and local_length < content_length:
 403             logger.info("Resuming download of %s", local_filename)
 404             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 405         else:
 406             logger.info("Downloading %s to cache", filename)
 407
 408         if download:
 409             r = requests.get(srcurl, headers=resume_header,
 410                              stream=True, allow_redirects=True)
 411             content_length = int(r.headers.get('content-length'))
 412             with open(local_filename, 'ab') as f:
 413                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 414                                           expected_size=(content_length / 65536) + 1):
 415                     if chunk:  # filter out keep-alive new chunks
 416                         f.write(chunk)
 417
 418         v = sha256_for_file(local_filename)
 419         if v == shasum:
 420             logger.info("\t...shasum verified for %s", local_filename)
 421         else:
 422             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 423             os.remove(local_filename)
 424             sys.exit(1)
 425
 426
 427 def debug_log_vagrant_vm(vm_dir, config):
 428     if options.verbosity >= 3:
 429         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 430         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 431         logger.debug('> vagrant status')
 432         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 433         logger.debug('> vagrant box list')
 434         subprocess.call(['vagrant', 'box', 'list'])
 435         if config['vm_provider'] == 'libvirt':
 436             logger.debug('> virsh -c qmeu:///system list --all')
 437             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 438             domain = 'buildserver_default'
 439             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 440             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 441
 442
 443 def main():
 444     global cachedir, cachefiles, config, tail
 445
 446     if options.skip_cache_update:
 447         logger.info('skipping cache update and verification...')
 448     else:
 449         update_cache(cachedir, cachefiles)
 450
 451     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 452     logger.info("Setting executable bit for %s", local_qt_filename)
 453     os.chmod(local_qt_filename, 0o755)
 454
 455     # use VirtualBox software virtualization if hardware is not available,
 456     # like if this is being run in kvm or some other VM platform, like
 457     # http://jenkins.debian.net, the values are 'on' or 'off'
 458     if sys.platform.startswith('darwin'):
 459         # all < 10 year old Macs work, and OSX servers as VM host are very
 460         # rare, but this could also be auto-detected if someone codes it
 461         config['hwvirtex'] = 'on'
 462         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 463     elif os.path.exists('/proc/cpuinfo'):
 464         with open('/proc/cpuinfo') as f:
 465             contents = f.read()
 466         if 'vmx' in contents or 'svm' in contents:
 467             config['hwvirtex'] = 'on'
 468             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 469
 470     serverdir = os.path.join(os.getcwd(), 'buildserver')
 471     logfilename = os.path.join(serverdir, 'up.log')
 472     if not os.path.exists(logfilename):
 473         open(logfilename, 'a').close()  # create blank file
 474     log_cm = vagrant.make_file_cm(logfilename)
 475     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 476
 477     if options.verbosity >= 2:
 478         tail = fdroidserver.tail.Tail(logfilename)
 479         tail.start()
 480
 481     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 482     if options.clean:
 483         vm.destroy()
 484
 485     # Check against the existing Vagrantfile.yaml, and if they differ, we
 486     # need to create a new box:
 487     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 488     writevf = True
 489     if os.path.exists(vf):
 490         logger.info('Halting %s', serverdir)
 491         v.halt()
 492         with open(vf, 'r', encoding='utf-8') as f:
 493             oldconfig = yaml.load(f)
 494         if config != oldconfig:
 495             logger.info("Server configuration has changed, rebuild from scratch is required")
 496             vm.destroy()
 497         else:
 498             logger.info("Re-provisioning existing server")
 499             writevf = False
 500     else:
 501         logger.info("No existing server - building from scratch")
 502     if writevf:
 503         with open(vf, 'w', encoding='utf-8') as f:
 504             yaml.dump(config, f)
 505
 506     if config['vm_provider'] == 'libvirt':
 507         found_basebox = False
 508         needs_mutate = False
 509         for box in v.box_list():
 510             if box.name == config['basebox']:
 511                 found_basebox = True
 512                 if box.provider != 'libvirt':
 513                     needs_mutate = True
 514                 continue
 515         if not found_basebox:
 516             if isinstance(config['baseboxurl'], str):
 517                 baseboxurl = config['baseboxurl']
 518             else:
 519                 baseboxurl = config['baseboxurl'][0]
 520             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 521             v.box_add(config['basebox'], baseboxurl)
 522             needs_mutate = True
 523         if needs_mutate:
 524             logger.info('Converting %s to libvirt format', config['basebox'])
 525             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 526             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 527             v.box_remove(config['basebox'], 'virtualbox')
 528
 529     logger.info("Configuring build server VM")
 530     debug_log_vagrant_vm(serverdir, config)
 531     try:
 532         v.up(provision=True)
 533     except subprocess.CalledProcessError:
 534         debug_log_vagrant_vm(serverdir, config)
 535         logger.error("'vagrant up' failed, is the base box missing?")
 536         sys.exit(1)
 537
 538     if config['copy_caches_from_host']:
 539         ssh_config = v.ssh_config()
 540         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 541         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 542         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 543         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 544
 545         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 546             fullpath = os.path.join(os.getenv('HOME'), d)
 547             if os.path.isdir(fullpath):
 548                 # TODO newer versions of vagrant provide `vagrant rsync`
 549                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 550                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 551                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 552                                  fullpath + '/',
 553                                  user + '@' + hostname + ':~/' + d + '/'])
 554
 555         # this file changes every time but should not be cached
 556         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 557         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 558
 559     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 560                          universal_newlines=True)
 561     buildserverid = p.communicate()[0].strip()
 562     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 563     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 564
 565     logger.info("Stopping build server VM")
 566     v.halt()
 567
 568     logger.info("Packaging")
 569     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 570     if os.path.exists(boxfile):
 571         os.remove(boxfile)
 572
 573     vm.package(output=boxfile)
 574
 575     logger.info("Adding box")
 576     vm.box_add('buildserver', boxfile, force=True)
 577
 578     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 579         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 580         sys.exit(1)
 581
 582     if not options.keep_box_file:
 583         logger.debug('box added to vagrant, ' +
 584                      'removing generated box file \'%s\'',
 585                      boxfile)
 586         os.remove(boxfile)
 587
 588
 589 if __name__ == '__main__':
 590     try:
 591         main()
 592     finally:
 593         if tail is not None:
 594             tail.stop()