makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     # Don't update sdk tools beyond 25.2.5.
 128     # Support for android update project has been removed and there is no replacement.
 129     # Until we find a solution for that we need to stay at this revision.
 130     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 131      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 132     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 133      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 134     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 135      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 136     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 137      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 138     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 139      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 140     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 141      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 142     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 143      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 144     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 145      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 146     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 147      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 148     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 149      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 150     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 151      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 152     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 153      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 154     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 155      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 156     ('https://dl.google.com/android/repository/android-14_r04.zip',
 157      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 158     ('https://dl.google.com/android/repository/android-15_r05.zip',
 159      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 160     ('https://dl.google.com/android/repository/android-16_r05.zip',
 161      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 162     ('https://dl.google.com/android/repository/android-17_r03.zip',
 163      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 164     ('https://dl.google.com/android/repository/android-18_r03.zip',
 165      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 166     ('https://dl.google.com/android/repository/android-19_r04.zip',
 167      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 168     ('https://dl.google.com/android/repository/android-20_r02.zip',
 169      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 170     ('https://dl.google.com/android/repository/android-21_r02.zip',
 171      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 172     ('https://dl.google.com/android/repository/android-22_r02.zip',
 173      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 174     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 175      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 176     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 177      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 178     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 179      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 180     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 181      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 182     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 183      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 184     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 185      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 186     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 187      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 188     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 189      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 190     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 191      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 192     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 193      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 194     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 195      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 196     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 197      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 198     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 199      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 200     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 201      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 202     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 203      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 204     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 205      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 206     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 207      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 209      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 210     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 211      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 212     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 213      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 214     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 215      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 216     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 217      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 218     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 219      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 220     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 221      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 222     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 223      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 224     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 225      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 226     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 227      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 228     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 229      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 230     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 231      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 232     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 233      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 234     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 235      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 236     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 237      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 238     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 239      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 240     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 241      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 242     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 243      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 244     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 245      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 246     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 247      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 248     # the binaries that Google uses are here:
 249     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 250     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 251      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 252     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 253      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 254     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 255      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 256     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 257      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 258     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 259      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 260     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 261      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 262     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 263      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 264     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 265      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 266     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 267      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 268     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 269      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 270     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 271      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 272     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 273      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 274     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 275      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 276     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 277      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 278     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 279      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 280     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 281      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 282     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 283      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 284     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 285      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 286     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 287      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 288     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 289      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 290     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 291      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 292     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 293      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 294     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 295      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 296     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 297      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 298     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 299      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 300     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 301      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 302     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 303      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 304     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 305      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 306     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 307      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 308     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 309      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 310     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 311      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 312     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 313      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 314     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 315      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 316     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 317      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 318     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 319      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 320     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 321      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 322     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 323      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 324     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 325      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 326     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 327      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 328     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 329      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 330     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 331      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 332     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 333      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 334     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 335      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 336     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 337      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 338     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 339      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 340     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 341      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 342     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 343      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 344 ]
 345
 346
 347 def sha256_for_file(path):
 348     with open(path, 'rb') as f:
 349         s = hashlib.sha256()
 350         while True:
 351             data = f.read(4096)
 352             if not data:
 353                 break
 354             s.update(data)
 355         return s.hexdigest()
 356
 357
 358 def run_via_vagrant_ssh(v, cmdlist):
 359     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 360         cmd = cmdlist
 361     else:
 362         cmd = ' '.join(cmdlist)
 363     v._run_vagrant_command(['ssh', '-c', cmd])
 364
 365
 366 def update_cache(cachedir, cachefiles):
 367     for srcurl, shasum in cachefiles:
 368         filename = os.path.basename(srcurl)
 369         local_filename = os.path.join(cachedir, filename)
 370
 371         if os.path.exists(local_filename):
 372             local_length = os.path.getsize(local_filename)
 373         else:
 374             local_length = -1
 375
 376         resume_header = {}
 377         download = True
 378
 379         try:
 380             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 381             if r.status_code == 200:
 382                 content_length = int(r.headers.get('content-length'))
 383             else:
 384                 content_length = local_length  # skip the download
 385         except requests.exceptions.RequestException as e:
 386             content_length = local_length  # skip the download
 387             logger.warn('%s', e)
 388
 389         if local_length == content_length:
 390             download = False
 391         elif local_length > content_length:
 392             logger.info('deleting corrupt file from cache: %s', local_filename)
 393             os.remove(local_filename)
 394             logger.info("Downloading %s to cache", filename)
 395         elif local_length > -1 and local_length < content_length:
 396             logger.info("Resuming download of %s", local_filename)
 397             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 398         else:
 399             logger.info("Downloading %s to cache", filename)
 400
 401         if download:
 402             r = requests.get(srcurl, headers=resume_header,
 403                              stream=True, allow_redirects=True)
 404             content_length = int(r.headers.get('content-length'))
 405             with open(local_filename, 'ab') as f:
 406                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 407                                           expected_size=(content_length / 65536) + 1):
 408                     if chunk:  # filter out keep-alive new chunks
 409                         f.write(chunk)
 410
 411         v = sha256_for_file(local_filename)
 412         if v == shasum:
 413             logger.info("\t...shasum verified for %s", local_filename)
 414         else:
 415             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 416             os.remove(local_filename)
 417             sys.exit(1)
 418
 419
 420 def debug_log_vagrant_vm(vm_dir, config):
 421     if options.verbosity >= 3:
 422         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 423         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 424         logger.debug('> vagrant status')
 425         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 426         logger.debug('> vagrant box list')
 427         subprocess.call(['vagrant', 'box', 'list'])
 428         if config['vm_provider'] == 'libvirt':
 429             logger.debug('> virsh -c qmeu:///system list --all')
 430             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 431             domain = 'buildserver_default'
 432             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 433             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 434
 435
 436 def main():
 437     global cachedir, cachefiles, config, tail
 438
 439     if options.skip_cache_update:
 440         logger.info('skipping cache update and verification...')
 441     else:
 442         update_cache(cachedir, cachefiles)
 443
 444     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 445     logger.info("Setting executable bit for %s", local_qt_filename)
 446     os.chmod(local_qt_filename, 0o755)
 447
 448     # use VirtualBox software virtualization if hardware is not available,
 449     # like if this is being run in kvm or some other VM platform, like
 450     # http://jenkins.debian.net, the values are 'on' or 'off'
 451     if sys.platform.startswith('darwin'):
 452         # all < 10 year old Macs work, and OSX servers as VM host are very
 453         # rare, but this could also be auto-detected if someone codes it
 454         config['hwvirtex'] = 'on'
 455         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 456     elif os.path.exists('/proc/cpuinfo'):
 457         with open('/proc/cpuinfo') as f:
 458             contents = f.read()
 459         if 'vmx' in contents or 'svm' in contents:
 460             config['hwvirtex'] = 'on'
 461             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 462
 463     serverdir = os.path.join(os.getcwd(), 'buildserver')
 464     logfilename = os.path.join(serverdir, 'up.log')
 465     if not os.path.exists(logfilename):
 466         open(logfilename, 'a').close()  # create blank file
 467     log_cm = vagrant.make_file_cm(logfilename)
 468     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 469
 470     if options.verbosity >= 2:
 471         tail = fdroidserver.tail.Tail(logfilename)
 472         tail.start()
 473
 474     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 475     if options.clean:
 476         vm.destroy()
 477
 478     # Check against the existing Vagrantfile.yaml, and if they differ, we
 479     # need to create a new box:
 480     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 481     writevf = True
 482     if os.path.exists(vf):
 483         logger.info('Halting %s', serverdir)
 484         v.halt()
 485         with open(vf, 'r', encoding='utf-8') as f:
 486             oldconfig = yaml.load(f)
 487         if config != oldconfig:
 488             logger.info("Server configuration has changed, rebuild from scratch is required")
 489             vm.destroy()
 490         else:
 491             logger.info("Re-provisioning existing server")
 492             writevf = False
 493     else:
 494         logger.info("No existing server - building from scratch")
 495     if writevf:
 496         with open(vf, 'w', encoding='utf-8') as f:
 497             yaml.dump(config, f)
 498
 499     if config['vm_provider'] == 'libvirt':
 500         found_basebox = False
 501         needs_mutate = False
 502         for box in v.box_list():
 503             if box.name == config['basebox']:
 504                 found_basebox = True
 505                 if box.provider != 'libvirt':
 506                     needs_mutate = True
 507                 continue
 508         if not found_basebox:
 509             if isinstance(config['baseboxurl'], str):
 510                 baseboxurl = config['baseboxurl']
 511             else:
 512                 baseboxurl = config['baseboxurl'][0]
 513             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 514             v.box_add(config['basebox'], baseboxurl)
 515             needs_mutate = True
 516         if needs_mutate:
 517             logger.info('Converting %s to libvirt format', config['basebox'])
 518             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 519             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 520             v.box_remove(config['basebox'], 'virtualbox')
 521
 522     logger.info("Configuring build server VM")
 523     debug_log_vagrant_vm(serverdir, config)
 524     try:
 525         v.up(provision=True)
 526     except fdroidserver.vmtools.FDroidBuildVmException as e:
 527         debug_log_vagrant_vm(serverdir, config)
 528         logger.exception('could not bring buildserver vm up. %s', e)
 529         sys.exit(1)
 530
 531     if config['copy_caches_from_host']:
 532         ssh_config = v.ssh_config()
 533         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 534         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 535         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 536         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 537
 538         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 539             fullpath = os.path.join(os.getenv('HOME'), d)
 540             if os.path.isdir(fullpath):
 541                 # TODO newer versions of vagrant provide `vagrant rsync`
 542                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 543                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 544                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 545                                  fullpath + '/',
 546                                  user + '@' + hostname + ':~/' + d + '/'])
 547
 548         # this file changes every time but should not be cached
 549         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 550         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 551
 552     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 553                          universal_newlines=True)
 554     buildserverid = p.communicate()[0].strip()
 555     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 556     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 557
 558     logger.info("Stopping build server VM")
 559     v.halt()
 560
 561     logger.info("Packaging")
 562     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 563     if os.path.exists(boxfile):
 564         os.remove(boxfile)
 565
 566     vm.package(output=boxfile)
 567
 568     logger.info("Adding box")
 569     vm.box_add('buildserver', boxfile, force=True)
 570
 571     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 572         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 573         sys.exit(1)
 574
 575     if not options.keep_box_file:
 576         logger.debug('box added to vagrant, ' +
 577                      'removing generated box file \'%s\'',
 578                      boxfile)
 579         os.remove(boxfile)
 580
 581
 582 if __name__ == '__main__':
 583     try:
 584         main()
 585     finally:
 586         if tail is not None:
 587             tail.stop()