makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 2048,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     # Don't update sdk tools beyond 25.2.5.
 128     # Support for android update project has been removed and there is no replacement.
 129     # Until we find a solution for that we need to stay at this revision.
 130     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 131      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 132     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 133      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 134     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 135      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 136     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 137      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 138     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 139      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 140     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 141      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 142     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 143      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 144     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 145      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 146     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 147      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 148     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 149      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 150     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 151      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 152     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 153      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 154     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 155      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 156     ('https://dl.google.com/android/repository/android-14_r04.zip',
 157      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 158     ('https://dl.google.com/android/repository/android-15_r05.zip',
 159      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 160     ('https://dl.google.com/android/repository/android-16_r05.zip',
 161      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 162     ('https://dl.google.com/android/repository/android-17_r03.zip',
 163      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 164     ('https://dl.google.com/android/repository/android-18_r03.zip',
 165      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 166     ('https://dl.google.com/android/repository/android-19_r04.zip',
 167      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 168     ('https://dl.google.com/android/repository/android-20_r02.zip',
 169      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 170     ('https://dl.google.com/android/repository/android-21_r02.zip',
 171      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 172     ('https://dl.google.com/android/repository/android-22_r02.zip',
 173      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 174     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 175      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 176     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 177      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 178     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 179      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 180     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 181      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 182     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 183      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 184     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 185      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 186     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 187      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 188     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 189      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 190     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 191      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 192     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 193      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 194     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 195      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 196     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 197      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 198     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 199      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 200     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 201      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 202     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 203      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 204     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 205      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 206     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 207      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 209      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 210     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 211      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 212     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 213      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 214     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 215      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 216     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 217      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 218     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 219      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 220     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 221      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 222     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 223      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 224     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 225      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 226     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 227      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 228     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 229      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 230     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 231      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 232     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 233      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 234     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 235      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 236     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 237      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 238     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 239      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 240     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 241      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 242     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 243      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 244     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 245      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 246     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 247      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 248     # the binaries that Google uses are here:
 249     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 250     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 251      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 252     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 253      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 254     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 255      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 256     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 257      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 258     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 259      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 260     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 261      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 262     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 263      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 264     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 265      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 266     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 267      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 268     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 269      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 270     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 271      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 272     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 273      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 274     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 275      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 276     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 277      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 278     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 279      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 280     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 281      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 282     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 283      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 284     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 285      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 286     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 287      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 288     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 289      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 290     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 291      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 292     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 293      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 294     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 295      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 296     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 297      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 298     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 299      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 300     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 301      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 302     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 303      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 304     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 305      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 306     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 307      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 308     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 309      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 310     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 311      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 312     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 313      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 314     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 315      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 316     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 317      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 318     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 319      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 320     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 321      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 322     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 323      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 324     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 325      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 326     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 327      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 328     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 329      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 330     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 331      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 332     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 333      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 334     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 335      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 336     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 337      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 338     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 339      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 340     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 341      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 342     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 343      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 344     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 345      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 346 ]
 347
 348
 349 def sha256_for_file(path):
 350     with open(path, 'rb') as f:
 351         s = hashlib.sha256()
 352         while True:
 353             data = f.read(4096)
 354             if not data:
 355                 break
 356             s.update(data)
 357         return s.hexdigest()
 358
 359
 360 def run_via_vagrant_ssh(v, cmdlist):
 361     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 362         cmd = cmdlist
 363     else:
 364         cmd = ' '.join(cmdlist)
 365     v._run_vagrant_command(['ssh', '-c', cmd])
 366
 367
 368 def update_cache(cachedir, cachefiles):
 369     for srcurl, shasum in cachefiles:
 370         filename = os.path.basename(srcurl)
 371         local_filename = os.path.join(cachedir, filename)
 372
 373         if os.path.exists(local_filename):
 374             local_length = os.path.getsize(local_filename)
 375         else:
 376             local_length = -1
 377
 378         resume_header = {}
 379         download = True
 380
 381         try:
 382             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 383             if r.status_code == 200:
 384                 content_length = int(r.headers.get('content-length'))
 385             else:
 386                 content_length = local_length  # skip the download
 387         except requests.exceptions.RequestException as e:
 388             content_length = local_length  # skip the download
 389             logger.warn('%s', e)
 390
 391         if local_length == content_length:
 392             download = False
 393         elif local_length > content_length:
 394             logger.info('deleting corrupt file from cache: %s', local_filename)
 395             os.remove(local_filename)
 396             logger.info("Downloading %s to cache", filename)
 397         elif local_length > -1 and local_length < content_length:
 398             logger.info("Resuming download of %s", local_filename)
 399             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 400         else:
 401             logger.info("Downloading %s to cache", filename)
 402
 403         if download:
 404             r = requests.get(srcurl, headers=resume_header,
 405                              stream=True, allow_redirects=True)
 406             content_length = int(r.headers.get('content-length'))
 407             with open(local_filename, 'ab') as f:
 408                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 409                                           expected_size=(content_length / 65536) + 1):
 410                     if chunk:  # filter out keep-alive new chunks
 411                         f.write(chunk)
 412
 413         v = sha256_for_file(local_filename)
 414         if v == shasum:
 415             logger.info("\t...shasum verified for %s", local_filename)
 416         else:
 417             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 418             os.remove(local_filename)
 419             sys.exit(1)
 420
 421
 422 def debug_log_vagrant_vm(vm_dir, config):
 423     if options.verbosity >= 3:
 424         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 425         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 426         logger.debug('> vagrant status')
 427         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 428         logger.debug('> vagrant box list')
 429         subprocess.call(['vagrant', 'box', 'list'])
 430         if config['vm_provider'] == 'libvirt':
 431             logger.debug('> virsh -c qmeu:///system list --all')
 432             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 433             domain = 'buildserver_default'
 434             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 435             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 436
 437
 438 def main():
 439     global cachedir, cachefiles, config, tail
 440
 441     if options.skip_cache_update:
 442         logger.info('skipping cache update and verification...')
 443     else:
 444         update_cache(cachedir, cachefiles)
 445
 446     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 447     logger.info("Setting executable bit for %s", local_qt_filename)
 448     os.chmod(local_qt_filename, 0o755)
 449
 450     # use VirtualBox software virtualization if hardware is not available,
 451     # like if this is being run in kvm or some other VM platform, like
 452     # http://jenkins.debian.net, the values are 'on' or 'off'
 453     if sys.platform.startswith('darwin'):
 454         # all < 10 year old Macs work, and OSX servers as VM host are very
 455         # rare, but this could also be auto-detected if someone codes it
 456         config['hwvirtex'] = 'on'
 457         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 458     elif os.path.exists('/proc/cpuinfo'):
 459         with open('/proc/cpuinfo') as f:
 460             contents = f.read()
 461         if 'vmx' in contents or 'svm' in contents:
 462             config['hwvirtex'] = 'on'
 463             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 464
 465     serverdir = os.path.join(os.getcwd(), 'buildserver')
 466     logfilename = os.path.join(serverdir, 'up.log')
 467     if not os.path.exists(logfilename):
 468         open(logfilename, 'a').close()  # create blank file
 469     log_cm = vagrant.make_file_cm(logfilename)
 470     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 471
 472     if options.verbosity >= 2:
 473         tail = fdroidserver.tail.Tail(logfilename)
 474         tail.start()
 475
 476     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 477     if options.clean:
 478         vm.destroy()
 479
 480     # Check against the existing Vagrantfile.yaml, and if they differ, we
 481     # need to create a new box:
 482     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 483     writevf = True
 484     if os.path.exists(vf):
 485         logger.info('Halting %s', serverdir)
 486         v.halt()
 487         with open(vf, 'r', encoding='utf-8') as f:
 488             oldconfig = yaml.load(f)
 489         if config != oldconfig:
 490             logger.info("Server configuration has changed, rebuild from scratch is required")
 491             vm.destroy()
 492         else:
 493             logger.info("Re-provisioning existing server")
 494             writevf = False
 495     else:
 496         logger.info("No existing server - building from scratch")
 497     if writevf:
 498         with open(vf, 'w', encoding='utf-8') as f:
 499             yaml.dump(config, f)
 500
 501     if config['vm_provider'] == 'libvirt':
 502         found_basebox = False
 503         needs_mutate = False
 504         for box in v.box_list():
 505             if box.name == config['basebox']:
 506                 found_basebox = True
 507                 if box.provider != 'libvirt':
 508                     needs_mutate = True
 509                 continue
 510         if not found_basebox:
 511             if isinstance(config['baseboxurl'], str):
 512                 baseboxurl = config['baseboxurl']
 513             else:
 514                 baseboxurl = config['baseboxurl'][0]
 515             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 516             v.box_add(config['basebox'], baseboxurl)
 517             needs_mutate = True
 518         if needs_mutate:
 519             logger.info('Converting %s to libvirt format', config['basebox'])
 520             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 521             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 522             v.box_remove(config['basebox'], 'virtualbox')
 523
 524     logger.info("Configuring build server VM")
 525     debug_log_vagrant_vm(serverdir, config)
 526     try:
 527         v.up(provision=True)
 528     except fdroidserver.vmtools.FDroidBuildVmException as e:
 529         debug_log_vagrant_vm(serverdir, config)
 530         logger.exception('could not bring buildserver vm up. %s', e)
 531         sys.exit(1)
 532
 533     if config['copy_caches_from_host']:
 534         ssh_config = v.ssh_config()
 535         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 536         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 537         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 538         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 539
 540         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 541             fullpath = os.path.join(os.getenv('HOME'), d)
 542             if os.path.isdir(fullpath):
 543                 # TODO newer versions of vagrant provide `vagrant rsync`
 544                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 545                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 546                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 547                                  fullpath + '/',
 548                                  user + '@' + hostname + ':~/' + d + '/'])
 549
 550         # this file changes every time but should not be cached
 551         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 552         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 553
 554     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 555                          universal_newlines=True)
 556     buildserverid = p.communicate()[0].strip()
 557     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 558     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 559
 560     logger.info("Stopping build server VM")
 561     v.halt()
 562
 563     logger.info("Packaging")
 564     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 565     if os.path.exists(boxfile):
 566         os.remove(boxfile)
 567
 568     vm.package(output=boxfile)
 569
 570     logger.info("Adding box")
 571     vm.box_add('buildserver', boxfile, force=True)
 572
 573     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 574         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 575         sys.exit(1)
 576
 577     if not options.keep_box_file:
 578         logger.debug('box added to vagrant, ' +
 579                      'removing generated box file \'%s\'',
 580                      boxfile)
 581         os.remove(boxfile)
 582
 583
 584 if __name__ == '__main__':
 585     try:
 586         main()
 587     finally:
 588         if tail is not None:
 589             tail.stop()