makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     # Don't update sdk tools beyond 25.2.5.
 128     # Support for android update project has been removed and there is no replacement.
 129     # Until we find a solution for that we need to stay at this revision.
 130     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 131      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 132     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 133      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 134     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 135      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 136     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 137      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 138     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 139      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 140     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 141      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 142     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 143      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 144     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 145      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 146     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 147      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 148     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 149      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 150     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 151      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 152     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 153      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 154     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 155      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 156     ('https://dl.google.com/android/repository/android-14_r04.zip',
 157      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 158     ('https://dl.google.com/android/repository/android-15_r05.zip',
 159      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 160     ('https://dl.google.com/android/repository/android-16_r05.zip',
 161      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 162     ('https://dl.google.com/android/repository/android-17_r03.zip',
 163      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 164     ('https://dl.google.com/android/repository/android-18_r03.zip',
 165      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 166     ('https://dl.google.com/android/repository/android-19_r04.zip',
 167      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 168     ('https://dl.google.com/android/repository/android-20_r02.zip',
 169      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 170     ('https://dl.google.com/android/repository/android-21_r02.zip',
 171      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 172     ('https://dl.google.com/android/repository/android-22_r02.zip',
 173      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 174     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 175      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 176     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 177      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 178     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 179      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 180     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 181      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 182     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 183      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 184     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 185      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 186     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 187      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 188     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 189      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 190     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 191      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 192     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 193      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 194     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 195      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 196     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 197      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 198     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 199      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 200     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 201      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 202     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 203      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 204     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 205      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 206     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 207      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 209      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 210     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 211      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 212     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 213      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 214     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 215      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 216     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 217      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 218     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 219      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 220     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 221      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 222     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 223      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 224     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 225      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 226     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 227      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 228     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 229      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 230     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 231      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 232     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 233      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 234     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 235      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 236     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 237      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 238     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 239      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 240     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 241      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 242     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 243      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 244     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 245      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 246     # the binaries that Google uses are here:
 247     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 248     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 249      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 250     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 251      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 252     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 253      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 254     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 255      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 256     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 257      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 258     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 259      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 260     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 261      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 262     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 263      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 264     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 265      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 266     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 267      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 268     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 269      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 270     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 271      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 272     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 273      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 274     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 275      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 276     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 277      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 278     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 279      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 280     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 281      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 282     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 283      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 284     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 285      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 286     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 287      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 288     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 289      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 290     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 291      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 292     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 293      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 294     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 295      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 296     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 297      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 298     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 299      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 300     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 301      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 302     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 303      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 304     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 305      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 306     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 307      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 308     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 309      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 310     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 311      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 312     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 313      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 314     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 315      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 316     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 317      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 318     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 319      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 320     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 321      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 322     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 323      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 324     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 325      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 326     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 327      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 328     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 329      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 330     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 331      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 332     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 333      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 334     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 335      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 336     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 337      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 338     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 339      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 340 ]
 341
 342
 343 def sha256_for_file(path):
 344     with open(path, 'rb') as f:
 345         s = hashlib.sha256()
 346         while True:
 347             data = f.read(4096)
 348             if not data:
 349                 break
 350             s.update(data)
 351         return s.hexdigest()
 352
 353
 354 def run_via_vagrant_ssh(v, cmdlist):
 355     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 356         cmd = cmdlist
 357     else:
 358         cmd = ' '.join(cmdlist)
 359     v._run_vagrant_command(['ssh', '-c', cmd])
 360
 361
 362 def update_cache(cachedir, cachefiles):
 363     for srcurl, shasum in cachefiles:
 364         filename = os.path.basename(srcurl)
 365         local_filename = os.path.join(cachedir, filename)
 366
 367         if os.path.exists(local_filename):
 368             local_length = os.path.getsize(local_filename)
 369         else:
 370             local_length = -1
 371
 372         resume_header = {}
 373         download = True
 374
 375         try:
 376             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 377             if r.status_code == 200:
 378                 content_length = int(r.headers.get('content-length'))
 379             else:
 380                 content_length = local_length  # skip the download
 381         except requests.exceptions.RequestException as e:
 382             content_length = local_length  # skip the download
 383             logger.warn('%s', e)
 384
 385         if local_length == content_length:
 386             download = False
 387         elif local_length > content_length:
 388             logger.info('deleting corrupt file from cache: %s', local_filename)
 389             os.remove(local_filename)
 390             logger.info("Downloading %s to cache", filename)
 391         elif local_length > -1 and local_length < content_length:
 392             logger.info("Resuming download of %s", local_filename)
 393             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 394         else:
 395             logger.info("Downloading %s to cache", filename)
 396
 397         if download:
 398             r = requests.get(srcurl, headers=resume_header,
 399                              stream=True, allow_redirects=True)
 400             content_length = int(r.headers.get('content-length'))
 401             with open(local_filename, 'ab') as f:
 402                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 403                                           expected_size=(content_length / 65536) + 1):
 404                     if chunk:  # filter out keep-alive new chunks
 405                         f.write(chunk)
 406
 407         v = sha256_for_file(local_filename)
 408         if v == shasum:
 409             logger.info("\t...shasum verified for %s", local_filename)
 410         else:
 411             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 412             os.remove(local_filename)
 413             sys.exit(1)
 414
 415
 416 def debug_log_vagrant_vm(vm_dir, config):
 417     if options.verbosity >= 3:
 418         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 419         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 420         logger.debug('> vagrant status')
 421         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 422         logger.debug('> vagrant box list')
 423         subprocess.call(['vagrant', 'box', 'list'])
 424         if config['vm_provider'] == 'libvirt':
 425             logger.debug('> virsh -c qmeu:///system list --all')
 426             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 427             domain = 'buildserver_default'
 428             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 429             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 430
 431
 432 def main():
 433     global cachedir, cachefiles, config, tail
 434
 435     if options.skip_cache_update:
 436         logger.info('skipping cache update and verification...')
 437     else:
 438         update_cache(cachedir, cachefiles)
 439
 440     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 441     logger.info("Setting executable bit for %s", local_qt_filename)
 442     os.chmod(local_qt_filename, 0o755)
 443
 444     # use VirtualBox software virtualization if hardware is not available,
 445     # like if this is being run in kvm or some other VM platform, like
 446     # http://jenkins.debian.net, the values are 'on' or 'off'
 447     if sys.platform.startswith('darwin'):
 448         # all < 10 year old Macs work, and OSX servers as VM host are very
 449         # rare, but this could also be auto-detected if someone codes it
 450         config['hwvirtex'] = 'on'
 451         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 452     elif os.path.exists('/proc/cpuinfo'):
 453         with open('/proc/cpuinfo') as f:
 454             contents = f.read()
 455         if 'vmx' in contents or 'svm' in contents:
 456             config['hwvirtex'] = 'on'
 457             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 458
 459     serverdir = os.path.join(os.getcwd(), 'buildserver')
 460     logfilename = os.path.join(serverdir, 'up.log')
 461     if not os.path.exists(logfilename):
 462         open(logfilename, 'a').close()  # create blank file
 463     log_cm = vagrant.make_file_cm(logfilename)
 464     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 465
 466     if options.verbosity >= 2:
 467         tail = fdroidserver.tail.Tail(logfilename)
 468         tail.start()
 469
 470     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 471     if options.clean:
 472         vm.destroy()
 473
 474     # Check against the existing Vagrantfile.yaml, and if they differ, we
 475     # need to create a new box:
 476     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 477     writevf = True
 478     if os.path.exists(vf):
 479         logger.info('Halting %s', serverdir)
 480         v.halt()
 481         with open(vf, 'r', encoding='utf-8') as f:
 482             oldconfig = yaml.load(f)
 483         if config != oldconfig:
 484             logger.info("Server configuration has changed, rebuild from scratch is required")
 485             vm.destroy()
 486         else:
 487             logger.info("Re-provisioning existing server")
 488             writevf = False
 489     else:
 490         logger.info("No existing server - building from scratch")
 491     if writevf:
 492         with open(vf, 'w', encoding='utf-8') as f:
 493             yaml.dump(config, f)
 494
 495     if config['vm_provider'] == 'libvirt':
 496         found_basebox = False
 497         needs_mutate = False
 498         for box in v.box_list():
 499             if box.name == config['basebox']:
 500                 found_basebox = True
 501                 if box.provider != 'libvirt':
 502                     needs_mutate = True
 503                 continue
 504         if not found_basebox:
 505             if isinstance(config['baseboxurl'], str):
 506                 baseboxurl = config['baseboxurl']
 507             else:
 508                 baseboxurl = config['baseboxurl'][0]
 509             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 510             v.box_add(config['basebox'], baseboxurl)
 511             needs_mutate = True
 512         if needs_mutate:
 513             logger.info('Converting %s to libvirt format', config['basebox'])
 514             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 515             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 516             v.box_remove(config['basebox'], 'virtualbox')
 517
 518     logger.info("Configuring build server VM")
 519     debug_log_vagrant_vm(serverdir, config)
 520     try:
 521         v.up(provision=True)
 522     except fdroidserver.vmtools.FDroidBuildVmException as e:
 523         debug_log_vagrant_vm(serverdir, config)
 524         logger.exception('could not bring buildserver vm up. %s', e)
 525         sys.exit(1)
 526
 527     if config['copy_caches_from_host']:
 528         ssh_config = v.ssh_config()
 529         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 530         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 531         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 532         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 533
 534         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 535             fullpath = os.path.join(os.getenv('HOME'), d)
 536             if os.path.isdir(fullpath):
 537                 # TODO newer versions of vagrant provide `vagrant rsync`
 538                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 539                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 540                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 541                                  fullpath + '/',
 542                                  user + '@' + hostname + ':~/' + d + '/'])
 543
 544         # this file changes every time but should not be cached
 545         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 546         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 547
 548     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 549                          universal_newlines=True)
 550     buildserverid = p.communicate()[0].strip()
 551     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 552     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 553
 554     logger.info("Stopping build server VM")
 555     v.halt()
 556
 557     logger.info("Packaging")
 558     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 559     if os.path.exists(boxfile):
 560         os.remove(boxfile)
 561
 562     vm.package(output=boxfile)
 563
 564     logger.info("Adding box")
 565     vm.box_add('buildserver', boxfile, force=True)
 566
 567     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 568         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 569         sys.exit(1)
 570
 571     if not options.keep_box_file:
 572         logger.debug('box added to vagrant, ' +
 573                      'removing generated box file \'%s\'',
 574                      boxfile)
 575         os.remove(boxfile)
 576
 577
 578 if __name__ == '__main__':
 579     try:
 580         main()
 581     finally:
 582         if tail is not None:
 583             tail.stop()