makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 1024,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 182      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 184      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 186      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 188      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 189     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 190      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 192      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 194      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 196      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 198      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 199     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 200      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 201     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 202      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 204      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 206      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 208      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 210      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 212      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 213     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 214      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 215     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 216      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 217     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 218      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 220      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 222      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 224      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 225     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 226      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 228      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 230      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 232      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 233     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 234      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 236      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 238      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 240      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 241     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 242      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 243     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 244      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 246      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 247     # the binaries that Google uses are here:
 248     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 249     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 250      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 251     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 252      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 253     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 254      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 255     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 256      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 257     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 258      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 259     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 260      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 261     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 262      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 263     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 264      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 265     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 266      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 267     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 268      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 269     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 270      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 271     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 272      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 273     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 274      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 275     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 276      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 277     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 278      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 279     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 280      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 281     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 282      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 283     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 284      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 285     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 286      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 287     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 288      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 289     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 290      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 291     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 292      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 293     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 294      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 295     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 296      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 297     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 298      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 299     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 300      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 301     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 302      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 303     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 304      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 305     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 306      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 307     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 308      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 309     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 310      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 311     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 312      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 313     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 314      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 315     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 316      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 317     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 318      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 319     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 320      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 321     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 322      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 323     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 324      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 325     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 326      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 327     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 328      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 329     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 330      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 331     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 332      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 333     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 334      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 335     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 336      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 337     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 338      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 339     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 340      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 341     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 342      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 343     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 344      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 345 ]
 346
 347
 348 def sha256_for_file(path):
 349     with open(path, 'rb') as f:
 350         s = hashlib.sha256()
 351         while True:
 352             data = f.read(4096)
 353             if not data:
 354                 break
 355             s.update(data)
 356         return s.hexdigest()
 357
 358
 359 def run_via_vagrant_ssh(v, cmdlist):
 360     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 361         cmd = cmdlist
 362     else:
 363         cmd = ' '.join(cmdlist)
 364     v._run_vagrant_command(['ssh', '-c', cmd])
 365
 366
 367 def update_cache(cachedir, cachefiles):
 368     for srcurl, shasum in cachefiles:
 369         filename = os.path.basename(srcurl)
 370         local_filename = os.path.join(cachedir, filename)
 371
 372         if os.path.exists(local_filename):
 373             local_length = os.path.getsize(local_filename)
 374         else:
 375             local_length = -1
 376
 377         resume_header = {}
 378         download = True
 379
 380         try:
 381             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 382             if r.status_code == 200:
 383                 content_length = int(r.headers.get('content-length'))
 384             else:
 385                 content_length = local_length  # skip the download
 386         except requests.exceptions.RequestException as e:
 387             content_length = local_length  # skip the download
 388             logger.warn('%s', e)
 389
 390         if local_length == content_length:
 391             download = False
 392         elif local_length > content_length:
 393             logger.info('deleting corrupt file from cache: %s', local_filename)
 394             os.remove(local_filename)
 395             logger.info("Downloading %s to cache", filename)
 396         elif local_length > -1 and local_length < content_length:
 397             logger.info("Resuming download of %s", local_filename)
 398             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 399         else:
 400             logger.info("Downloading %s to cache", filename)
 401
 402         if download:
 403             r = requests.get(srcurl, headers=resume_header,
 404                              stream=True, allow_redirects=True)
 405             content_length = int(r.headers.get('content-length'))
 406             with open(local_filename, 'ab') as f:
 407                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 408                                           expected_size=(content_length / 65536) + 1):
 409                     if chunk:  # filter out keep-alive new chunks
 410                         f.write(chunk)
 411
 412         v = sha256_for_file(local_filename)
 413         if v == shasum:
 414             logger.info("\t...shasum verified for %s", local_filename)
 415         else:
 416             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 417             os.remove(local_filename)
 418             sys.exit(1)
 419
 420
 421 def debug_log_vagrant_vm(vm_dir, config):
 422     if options.verbosity >= 3:
 423         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 424         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 425         logger.debug('> vagrant status')
 426         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 427         logger.debug('> vagrant box list')
 428         subprocess.call(['vagrant', 'box', 'list'])
 429         if config['vm_provider'] == 'libvirt':
 430             logger.debug('> virsh -c qmeu:///system list --all')
 431             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 432             domain = 'buildserver_default'
 433             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 434             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 435
 436
 437 def main():
 438     global cachedir, cachefiles, config, tail
 439
 440     if options.skip_cache_update:
 441         logger.info('skipping cache update and verification...')
 442     else:
 443         update_cache(cachedir, cachefiles)
 444
 445     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 446     logger.info("Setting executable bit for %s", local_qt_filename)
 447     os.chmod(local_qt_filename, 0o755)
 448
 449     # use VirtualBox software virtualization if hardware is not available,
 450     # like if this is being run in kvm or some other VM platform, like
 451     # http://jenkins.debian.net, the values are 'on' or 'off'
 452     if sys.platform.startswith('darwin'):
 453         # all < 10 year old Macs work, and OSX servers as VM host are very
 454         # rare, but this could also be auto-detected if someone codes it
 455         config['hwvirtex'] = 'on'
 456         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 457     elif os.path.exists('/proc/cpuinfo'):
 458         with open('/proc/cpuinfo') as f:
 459             contents = f.read()
 460         if 'vmx' in contents or 'svm' in contents:
 461             config['hwvirtex'] = 'on'
 462             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 463
 464     serverdir = os.path.join(os.getcwd(), 'buildserver')
 465     logfilename = os.path.join(serverdir, 'up.log')
 466     if not os.path.exists(logfilename):
 467         open(logfilename, 'a').close()  # create blank file
 468     log_cm = vagrant.make_file_cm(logfilename)
 469     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 470
 471     if options.verbosity >= 2:
 472         tail = fdroidserver.tail.Tail(logfilename)
 473         tail.start()
 474
 475     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 476     if options.clean:
 477         vm.destroy()
 478
 479     # Check against the existing Vagrantfile.yaml, and if they differ, we
 480     # need to create a new box:
 481     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 482     writevf = True
 483     if os.path.exists(vf):
 484         logger.info('Halting %s', serverdir)
 485         v.halt()
 486         with open(vf, 'r', encoding='utf-8') as f:
 487             oldconfig = yaml.load(f)
 488         if config != oldconfig:
 489             logger.info("Server configuration has changed, rebuild from scratch is required")
 490             vm.destroy()
 491         else:
 492             logger.info("Re-provisioning existing server")
 493             writevf = False
 494     else:
 495         logger.info("No existing server - building from scratch")
 496     if writevf:
 497         with open(vf, 'w', encoding='utf-8') as f:
 498             yaml.dump(config, f)
 499
 500     if config['vm_provider'] == 'libvirt':
 501         found_basebox = False
 502         needs_mutate = False
 503         for box in v.box_list():
 504             if box.name == config['basebox']:
 505                 found_basebox = True
 506                 if box.provider != 'libvirt':
 507                     needs_mutate = True
 508                 continue
 509         if not found_basebox:
 510             if isinstance(config['baseboxurl'], str):
 511                 baseboxurl = config['baseboxurl']
 512             else:
 513                 baseboxurl = config['baseboxurl'][0]
 514             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 515             v.box_add(config['basebox'], baseboxurl)
 516             needs_mutate = True
 517         if needs_mutate:
 518             logger.info('Converting %s to libvirt format', config['basebox'])
 519             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 520             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 521             v.box_remove(config['basebox'], 'virtualbox')
 522
 523     logger.info("Configuring build server VM")
 524     debug_log_vagrant_vm(serverdir, config)
 525     try:
 526         v.up(provision=True)
 527     except subprocess.CalledProcessError:
 528         debug_log_vagrant_vm(serverdir, config)
 529         logger.error("'vagrant up' failed, is the base box missing?")
 530         sys.exit(1)
 531
 532     if config['copy_caches_from_host']:
 533         ssh_config = v.ssh_config()
 534         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 535         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 536         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 537         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 538
 539         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 540             fullpath = os.path.join(os.getenv('HOME'), d)
 541             if os.path.isdir(fullpath):
 542                 # TODO newer versions of vagrant provide `vagrant rsync`
 543                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 544                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 545                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 546                                  fullpath + '/',
 547                                  user + '@' + hostname + ':~/' + d + '/'])
 548
 549         # this file changes every time but should not be cached
 550         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 551         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 552
 553     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 554                          universal_newlines=True)
 555     buildserverid = p.communicate()[0].strip()
 556     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 557     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 558
 559     logger.info("Stopping build server VM")
 560     v.halt()
 561
 562     logger.info("Packaging")
 563     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 564     if os.path.exists(boxfile):
 565         os.remove(boxfile)
 566
 567     vm.package(output=boxfile)
 568
 569     logger.info("Adding box")
 570     vm.box_add('buildserver', boxfile, force=True)
 571
 572     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 573         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 574         sys.exit(1)
 575
 576     if not options.keep_box_file:
 577         logger.debug('box added to vagrant, ' +
 578                      'removing generated box file \'%s\'',
 579                      boxfile)
 580         os.remove(boxfile)
 581
 582
 583 if __name__ == '__main__':
 584     try:
 585         main()
 586     finally:
 587         if tail is not None:
 588             tail.stop()