makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 128      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 129     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 130      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 131     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 132      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 133     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 134      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 135     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 136      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 137     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 138      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 139     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 140      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 141     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 142      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 143     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 144      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 145     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 146      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 147     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 148      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 149     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 150      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 151     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 152      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 153     ('https://dl.google.com/android/repository/android-14_r04.zip',
 154      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 155     ('https://dl.google.com/android/repository/android-15_r05.zip',
 156      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 157     ('https://dl.google.com/android/repository/android-16_r05.zip',
 158      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 159     ('https://dl.google.com/android/repository/android-17_r03.zip',
 160      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 161     ('https://dl.google.com/android/repository/android-18_r03.zip',
 162      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 163     ('https://dl.google.com/android/repository/android-19_r04.zip',
 164      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 165     ('https://dl.google.com/android/repository/android-20_r02.zip',
 166      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 167     ('https://dl.google.com/android/repository/android-21_r02.zip',
 168      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 169     ('https://dl.google.com/android/repository/android-22_r02.zip',
 170      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 171     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 172      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 173     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 174      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 175     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 176      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 177     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 178      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 179     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 180      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 181     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 182      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 184      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 186      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 187     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 188      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 189     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 190      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 192      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 194      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 196      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 197     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 198      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 199     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 200      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 201     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 202      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 204      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 206      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 208      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 210      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 211     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 212      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 213     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 214      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 215     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 216      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 217     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 218      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 220      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 222      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 223     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 224      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 225     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 226      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 228      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 230      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 231     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 232      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 233     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 234      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 236      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 238      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 239     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 240      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 241     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 242      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 243     # the binaries that Google uses are here:
 244     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 245     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 246      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 247     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 248      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 249     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 250      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 251     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 252      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 253     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 254      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 255     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 256      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 257     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 258      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 259     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 260      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 261     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 262      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 263     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 264      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 265     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 266      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 267     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 268      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 269     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 270      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 271     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 272      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 273     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 274      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 275     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 276      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 277     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 278      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 279     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 280      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 281     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 282      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 283     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 284      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 285     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 286      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 287     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 288      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 289     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 290      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 291     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 292      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 293     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 294      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 295     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 296      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 297     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 298      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 299     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 300      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 301     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 302      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 303     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 304      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 305     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 306      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 307     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 308      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 309     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 310      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 311     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 312      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 313     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 314      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 315     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 316      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 317     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 318      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 319     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 320      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 321     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 322      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 323     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 324      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 325     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 326      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 327     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 328      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 329     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 330      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 331     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 332      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 333     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 334      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 335     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 336      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 337 ]
 338
 339
 340 def sha256_for_file(path):
 341     with open(path, 'rb') as f:
 342         s = hashlib.sha256()
 343         while True:
 344             data = f.read(4096)
 345             if not data:
 346                 break
 347             s.update(data)
 348         return s.hexdigest()
 349
 350
 351 def run_via_vagrant_ssh(v, cmdlist):
 352     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 353         cmd = cmdlist
 354     else:
 355         cmd = ' '.join(cmdlist)
 356     v._run_vagrant_command(['ssh', '-c', cmd])
 357
 358
 359 def update_cache(cachedir, cachefiles):
 360     for srcurl, shasum in cachefiles:
 361         filename = os.path.basename(srcurl)
 362         local_filename = os.path.join(cachedir, filename)
 363
 364         if os.path.exists(local_filename):
 365             local_length = os.path.getsize(local_filename)
 366         else:
 367             local_length = -1
 368
 369         resume_header = {}
 370         download = True
 371
 372         try:
 373             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 374             if r.status_code == 200:
 375                 content_length = int(r.headers.get('content-length'))
 376             else:
 377                 content_length = local_length  # skip the download
 378         except requests.exceptions.RequestException as e:
 379             content_length = local_length  # skip the download
 380             logger.warn('%s', e)
 381
 382         if local_length == content_length:
 383             download = False
 384         elif local_length > content_length:
 385             logger.info('deleting corrupt file from cache: %s', local_filename)
 386             os.remove(local_filename)
 387             logger.info("Downloading %s to cache", filename)
 388         elif local_length > -1 and local_length < content_length:
 389             logger.info("Resuming download of %s", local_filename)
 390             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 391         else:
 392             logger.info("Downloading %s to cache", filename)
 393
 394         if download:
 395             r = requests.get(srcurl, headers=resume_header,
 396                              stream=True, verify=False, allow_redirects=True)
 397             content_length = int(r.headers.get('content-length'))
 398             with open(local_filename, 'ab') as f:
 399                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 400                                           expected_size=(content_length / 65536) + 1):
 401                     if chunk:  # filter out keep-alive new chunks
 402                         f.write(chunk)
 403
 404         v = sha256_for_file(local_filename)
 405         if v == shasum:
 406             logger.info("\t...shasum verified for %s", local_filename)
 407         else:
 408             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 409             os.remove(local_filename)
 410             sys.exit(1)
 411
 412
 413 def debug_log_vagrant_vm(vm_dir, config):
 414     if options.verbosity >= 3:
 415         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 416         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 417         logger.debug('> vagrant status')
 418         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 419         logger.debug('> vagrant box list')
 420         subprocess.call(['vagrant', 'box', 'list'])
 421         if config['vm_provider'] == 'libvirt':
 422             logger.debug('> virsh -c qmeu:///system list --all')
 423             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 424             domain = 'buildserver_default'
 425             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 426             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 427
 428
 429 def main():
 430     global cachedir, cachefiles, config, tail
 431
 432     if options.skip_cache_update:
 433         logger.info('skipping cache update and verification...')
 434     else:
 435         update_cache(cachedir, cachefiles)
 436
 437     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 438     logger.info("Setting executable bit for %s", local_qt_filename)
 439     os.chmod(local_qt_filename, 0o755)
 440
 441     # use VirtualBox software virtualization if hardware is not available,
 442     # like if this is being run in kvm or some other VM platform, like
 443     # http://jenkins.debian.net, the values are 'on' or 'off'
 444     if sys.platform.startswith('darwin'):
 445         # all < 10 year old Macs work, and OSX servers as VM host are very
 446         # rare, but this could also be auto-detected if someone codes it
 447         config['hwvirtex'] = 'on'
 448         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 449     elif os.path.exists('/proc/cpuinfo'):
 450         with open('/proc/cpuinfo') as f:
 451             contents = f.read()
 452         if 'vmx' in contents or 'svm' in contents:
 453             config['hwvirtex'] = 'on'
 454             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 455
 456     serverdir = os.path.join(os.getcwd(), 'buildserver')
 457     logfilename = os.path.join(serverdir, 'up.log')
 458     if not os.path.exists(logfilename):
 459         open(logfilename, 'a').close()  # create blank file
 460     log_cm = vagrant.make_file_cm(logfilename)
 461     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 462
 463     if options.verbosity >= 2:
 464         tail = fdroidserver.tail.Tail(logfilename)
 465         tail.start()
 466
 467     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 468     if options.clean:
 469         vm.destroy()
 470
 471     # Check against the existing Vagrantfile.yaml, and if they differ, we
 472     # need to create a new box:
 473     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 474     writevf = True
 475     if os.path.exists(vf):
 476         logger.info('Halting %s', serverdir)
 477         v.halt()
 478         with open(vf, 'r', encoding='utf-8') as f:
 479             oldconfig = yaml.load(f)
 480         if config != oldconfig:
 481             logger.info("Server configuration has changed, rebuild from scratch is required")
 482             vm.destroy()
 483         else:
 484             logger.info("Re-provisioning existing server")
 485             writevf = False
 486     else:
 487         logger.info("No existing server - building from scratch")
 488     if writevf:
 489         with open(vf, 'w', encoding='utf-8') as f:
 490             yaml.dump(config, f)
 491
 492     if config['vm_provider'] == 'libvirt':
 493         found_basebox = False
 494         needs_mutate = False
 495         for box in v.box_list():
 496             if box.name == config['basebox']:
 497                 found_basebox = True
 498                 if box.provider != 'libvirt':
 499                     needs_mutate = True
 500                 continue
 501         if not found_basebox:
 502             if isinstance(config['baseboxurl'], str):
 503                 baseboxurl = config['baseboxurl']
 504             else:
 505                 baseboxurl = config['baseboxurl'][0]
 506             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 507             v.box_add(config['basebox'], baseboxurl)
 508             needs_mutate = True
 509         if needs_mutate:
 510             logger.info('Converting %s to libvirt format', config['basebox'])
 511             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 512             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 513             v.box_remove(config['basebox'], 'virtualbox')
 514
 515     logger.info("Configuring build server VM")
 516     debug_log_vagrant_vm(serverdir, config)
 517     try:
 518         v.up(provision=True)
 519     except fdroidserver.vmtools.FDroidBuildVmException as e:
 520         debug_log_vagrant_vm(serverdir, config)
 521         logger.exception('could not bring buildserver vm up. %s', e)
 522         sys.exit(1)
 523
 524     if config['copy_caches_from_host']:
 525         ssh_config = v.ssh_config()
 526         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 527         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 528         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 529         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 530
 531         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 532             fullpath = os.path.join(os.getenv('HOME'), d)
 533             if os.path.isdir(fullpath):
 534                 # TODO newer versions of vagrant provide `vagrant rsync`
 535                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 536                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 537                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 538                                  fullpath + '/',
 539                                  user + '@' + hostname + ':~/' + d + '/'])
 540
 541         # this file changes every time but should not be cached
 542         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 543         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 544
 545     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 546                          universal_newlines=True)
 547     buildserverid = p.communicate()[0].strip()
 548     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 549     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 550
 551     logger.info("Stopping build server VM")
 552     v.halt()
 553
 554     logger.info("Packaging")
 555     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 556     if os.path.exists(boxfile):
 557         os.remove(boxfile)
 558
 559     vm.package(output=boxfile)
 560
 561     logger.info("Adding box")
 562     vm.box_add('buildserver', boxfile, force=True)
 563
 564     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 565         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 566         sys.exit(1)
 567
 568     if not options.keep_box_file:
 569         logger.debug('box added to vagrant, ' +
 570                      'removing generated box file \'%s\'',
 571                      boxfile)
 572         os.remove(boxfile)
 573
 574
 575 if __name__ == '__main__':
 576     try:
 577         main()
 578     finally:
 579         if tail is not None:
 580             tail.stop()