makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     # Don't update sdk tools beyond 25.2.5.
 128     # Support for android update project has been removed and there is no replacement.
 129     # Until we find a solution for that we need to stay at this revision.
 130     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 131      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 132     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 133      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 134     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 135      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 136     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 137      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 138     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 139      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 140     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 141      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 142     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 143      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 144     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 145      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 146     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 147      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 148     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 149      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 150     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 151      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 152     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 153      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 154     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 155      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 156     ('https://dl.google.com/android/repository/android-14_r04.zip',
 157      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 158     ('https://dl.google.com/android/repository/android-15_r05.zip',
 159      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 160     ('https://dl.google.com/android/repository/android-16_r05.zip',
 161      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 162     ('https://dl.google.com/android/repository/android-17_r03.zip',
 163      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 164     ('https://dl.google.com/android/repository/android-18_r03.zip',
 165      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 166     ('https://dl.google.com/android/repository/android-19_r04.zip',
 167      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 168     ('https://dl.google.com/android/repository/android-20_r02.zip',
 169      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 170     ('https://dl.google.com/android/repository/android-21_r02.zip',
 171      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 172     ('https://dl.google.com/android/repository/android-22_r02.zip',
 173      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 174     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 175      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 176     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 177      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 178     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 179      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 180     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 181      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 182     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 183      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 184     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 185      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 186     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 187      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 188     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 189      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 190     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 191      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 192     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 193      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 194     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 195      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 196     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 197      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 198     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 199      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 200     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 201      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 202     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 203      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 204     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 205      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 206     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 207      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 209      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 210     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 211      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 212     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 213      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 214     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 215      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 216     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 217      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 218     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 219      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 220     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 221      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 222     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 223      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 224     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 225      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 226     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 227      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 228     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 229      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 230     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 231      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 232     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 233      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 234     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 235      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 236     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 237      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 238     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 239      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 240     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 241      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 242     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 243      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 244     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 245      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 246     # the binaries that Google uses are here:
 247     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 248     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 249      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 250     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 251      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 252     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 253      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 254     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 255      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 256     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 257      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 258     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 259      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 260     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 261      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 262     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 263      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 264     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 265      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 266     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 267      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 268     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 269      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 270     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 271      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 272     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 273      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 274     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 275      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 276     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 277      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 278     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 279      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 280     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 281      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 282     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 283      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 284     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 285      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 286     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 287      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 288     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 289      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 290     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 291      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 292     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 293      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 294     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 295      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 296     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 297      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 298     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 299      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 300     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 301      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 302     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 303      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 304     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 305      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 306     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 307      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 308     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 309      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 310     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 311      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 312     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 313      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 314     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 315      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 316     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 317      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 318     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 319      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 320     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 321      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 322     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 323      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 324     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 325      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 326     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 327      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 328     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 329      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 330     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 331      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 332     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 333      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 334     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 335      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 336     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 337      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 338     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 339      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 340     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 341      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 342 ]
 343
 344
 345 def sha256_for_file(path):
 346     with open(path, 'rb') as f:
 347         s = hashlib.sha256()
 348         while True:
 349             data = f.read(4096)
 350             if not data:
 351                 break
 352             s.update(data)
 353         return s.hexdigest()
 354
 355
 356 def run_via_vagrant_ssh(v, cmdlist):
 357     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 358         cmd = cmdlist
 359     else:
 360         cmd = ' '.join(cmdlist)
 361     v._run_vagrant_command(['ssh', '-c', cmd])
 362
 363
 364 def update_cache(cachedir, cachefiles):
 365     for srcurl, shasum in cachefiles:
 366         filename = os.path.basename(srcurl)
 367         local_filename = os.path.join(cachedir, filename)
 368
 369         if os.path.exists(local_filename):
 370             local_length = os.path.getsize(local_filename)
 371         else:
 372             local_length = -1
 373
 374         resume_header = {}
 375         download = True
 376
 377         try:
 378             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 379             if r.status_code == 200:
 380                 content_length = int(r.headers.get('content-length'))
 381             else:
 382                 content_length = local_length  # skip the download
 383         except requests.exceptions.RequestException as e:
 384             content_length = local_length  # skip the download
 385             logger.warn('%s', e)
 386
 387         if local_length == content_length:
 388             download = False
 389         elif local_length > content_length:
 390             logger.info('deleting corrupt file from cache: %s', local_filename)
 391             os.remove(local_filename)
 392             logger.info("Downloading %s to cache", filename)
 393         elif local_length > -1 and local_length < content_length:
 394             logger.info("Resuming download of %s", local_filename)
 395             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 396         else:
 397             logger.info("Downloading %s to cache", filename)
 398
 399         if download:
 400             r = requests.get(srcurl, headers=resume_header,
 401                              stream=True, allow_redirects=True)
 402             content_length = int(r.headers.get('content-length'))
 403             with open(local_filename, 'ab') as f:
 404                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 405                                           expected_size=(content_length / 65536) + 1):
 406                     if chunk:  # filter out keep-alive new chunks
 407                         f.write(chunk)
 408
 409         v = sha256_for_file(local_filename)
 410         if v == shasum:
 411             logger.info("\t...shasum verified for %s", local_filename)
 412         else:
 413             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 414             os.remove(local_filename)
 415             sys.exit(1)
 416
 417
 418 def debug_log_vagrant_vm(vm_dir, config):
 419     if options.verbosity >= 3:
 420         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 421         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 422         logger.debug('> vagrant status')
 423         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 424         logger.debug('> vagrant box list')
 425         subprocess.call(['vagrant', 'box', 'list'])
 426         if config['vm_provider'] == 'libvirt':
 427             logger.debug('> virsh -c qmeu:///system list --all')
 428             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 429             domain = 'buildserver_default'
 430             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 431             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 432
 433
 434 def main():
 435     global cachedir, cachefiles, config, tail
 436
 437     if options.skip_cache_update:
 438         logger.info('skipping cache update and verification...')
 439     else:
 440         update_cache(cachedir, cachefiles)
 441
 442     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 443     logger.info("Setting executable bit for %s", local_qt_filename)
 444     os.chmod(local_qt_filename, 0o755)
 445
 446     # use VirtualBox software virtualization if hardware is not available,
 447     # like if this is being run in kvm or some other VM platform, like
 448     # http://jenkins.debian.net, the values are 'on' or 'off'
 449     if sys.platform.startswith('darwin'):
 450         # all < 10 year old Macs work, and OSX servers as VM host are very
 451         # rare, but this could also be auto-detected if someone codes it
 452         config['hwvirtex'] = 'on'
 453         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 454     elif os.path.exists('/proc/cpuinfo'):
 455         with open('/proc/cpuinfo') as f:
 456             contents = f.read()
 457         if 'vmx' in contents or 'svm' in contents:
 458             config['hwvirtex'] = 'on'
 459             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 460
 461     serverdir = os.path.join(os.getcwd(), 'buildserver')
 462     logfilename = os.path.join(serverdir, 'up.log')
 463     if not os.path.exists(logfilename):
 464         open(logfilename, 'a').close()  # create blank file
 465     log_cm = vagrant.make_file_cm(logfilename)
 466     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 467
 468     if options.verbosity >= 2:
 469         tail = fdroidserver.tail.Tail(logfilename)
 470         tail.start()
 471
 472     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 473     if options.clean:
 474         vm.destroy()
 475
 476     # Check against the existing Vagrantfile.yaml, and if they differ, we
 477     # need to create a new box:
 478     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 479     writevf = True
 480     if os.path.exists(vf):
 481         logger.info('Halting %s', serverdir)
 482         v.halt()
 483         with open(vf, 'r', encoding='utf-8') as f:
 484             oldconfig = yaml.load(f)
 485         if config != oldconfig:
 486             logger.info("Server configuration has changed, rebuild from scratch is required")
 487             vm.destroy()
 488         else:
 489             logger.info("Re-provisioning existing server")
 490             writevf = False
 491     else:
 492         logger.info("No existing server - building from scratch")
 493     if writevf:
 494         with open(vf, 'w', encoding='utf-8') as f:
 495             yaml.dump(config, f)
 496
 497     if config['vm_provider'] == 'libvirt':
 498         found_basebox = False
 499         needs_mutate = False
 500         for box in v.box_list():
 501             if box.name == config['basebox']:
 502                 found_basebox = True
 503                 if box.provider != 'libvirt':
 504                     needs_mutate = True
 505                 continue
 506         if not found_basebox:
 507             if isinstance(config['baseboxurl'], str):
 508                 baseboxurl = config['baseboxurl']
 509             else:
 510                 baseboxurl = config['baseboxurl'][0]
 511             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 512             v.box_add(config['basebox'], baseboxurl)
 513             needs_mutate = True
 514         if needs_mutate:
 515             logger.info('Converting %s to libvirt format', config['basebox'])
 516             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 517             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 518             v.box_remove(config['basebox'], 'virtualbox')
 519
 520     logger.info("Configuring build server VM")
 521     debug_log_vagrant_vm(serverdir, config)
 522     try:
 523         v.up(provision=True)
 524     except fdroidserver.vmtools.FDroidBuildVmException as e:
 525         debug_log_vagrant_vm(serverdir, config)
 526         logger.exception('could not bring buildserver vm up. %s', e)
 527         sys.exit(1)
 528
 529     if config['copy_caches_from_host']:
 530         ssh_config = v.ssh_config()
 531         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 532         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 533         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 534         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 535
 536         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 537             fullpath = os.path.join(os.getenv('HOME'), d)
 538             if os.path.isdir(fullpath):
 539                 # TODO newer versions of vagrant provide `vagrant rsync`
 540                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 541                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 542                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 543                                  fullpath + '/',
 544                                  user + '@' + hostname + ':~/' + d + '/'])
 545
 546         # this file changes every time but should not be cached
 547         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 548         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 549
 550     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 551                          universal_newlines=True)
 552     buildserverid = p.communicate()[0].strip()
 553     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 554     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 555
 556     logger.info("Stopping build server VM")
 557     v.halt()
 558
 559     logger.info("Packaging")
 560     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 561     if os.path.exists(boxfile):
 562         os.remove(boxfile)
 563
 564     vm.package(output=boxfile)
 565
 566     logger.info("Adding box")
 567     vm.box_add('buildserver', boxfile, force=True)
 568
 569     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 570         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 571         sys.exit(1)
 572
 573     if not options.keep_box_file:
 574         logger.debug('box added to vagrant, ' +
 575                      'removing generated box file \'%s\'',
 576                      boxfile)
 577         os.remove(boxfile)
 578
 579
 580 if __name__ == '__main__':
 581     try:
 582         main()
 583     finally:
 584         if tail is not None:
 585             tail.stop()