makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri()
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 2048,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84     logger.debug('detected virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 117     if os.path.isfile(aptcachelock):
 118         logger.info('apt cache dir is locked, removing lock')
 119         os.remove(aptcachelock)
 120     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 121     if os.path.isdir(aptcachepartial):
 122         logger.info('removing partial downloads from apt cache dir')
 123         shutil.rmtree(aptcachepartial)
 124
 125 cachefiles = [
 126     # Don't update sdk tools beyond 25.2.5.
 127     # Support for android update project has been removed and there is no replacement.
 128     # Until we find a solution for that we need to stay at this revision.
 129     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 130      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r03.zip',
 178      '9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
 179     ('https://dl.google.com/android/repository/platform-26_r02.zip',
 180      '2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
 181     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 182      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 184      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 186      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 187     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 188      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 189     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 190      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 192      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 194      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 196      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 197     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 198      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 199     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 200      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 201     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 202      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 204      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 206      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 208      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 210      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 211     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 212      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 213     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 214      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 215     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 216      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 217     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 218      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 220      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 222      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 223     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 224      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 225     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 226      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 228      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 230      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 231     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 232      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 233     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 234      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 236      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 238      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 239     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 240      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 241     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 242      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 243     ('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
 244      'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
 245     ('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
 246      'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
 247     ('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
 248      '5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
 249     ('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
 250      '53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
 251     ('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
 252      '2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
 253     ('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
 254      'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
 255     # the binaries that Google uses are here:
 256     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 257     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 258      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 259     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 260      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 261     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 262      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 263     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 264      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 265     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 266      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 267     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 268      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 269     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 270      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 271     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 272      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 273     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 274      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 275     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 276      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 277     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 278      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 279     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 280      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 281     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 282      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 283     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 284      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 285     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 286      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 287     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 288      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 289     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 290      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 291     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 292      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 293     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 294      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 295     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 296      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 297     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 298      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 299     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 300      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 301     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 302      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 303     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 304      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 305     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 306      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 307     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 308      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 309     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 310      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 311     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 312      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 313     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 314      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 315     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 316      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 317     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 318      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 319     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 320      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 321     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 322      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 323     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 324      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 325     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 326      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 327     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 328      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 329     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 330      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 331     ('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
 332      '515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
 333     ('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
 334      'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
 335     ('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
 336      '8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
 337     ('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
 338      '15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
 339     ('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
 340      'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
 341     ('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
 342      'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
 343     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 344      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 345     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 346      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 347     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 348      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 349     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 350      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 351     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 352      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 353     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 354      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 355     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 356      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 357     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 358      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 359     ('https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip',
 360      'a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e'),
 361 ]
 362
 363
 364 def sha256_for_file(path):
 365     with open(path, 'rb') as f:
 366         s = hashlib.sha256()
 367         while True:
 368             data = f.read(4096)
 369             if not data:
 370                 break
 371             s.update(data)
 372         return s.hexdigest()
 373
 374
 375 def run_via_vagrant_ssh(v, cmdlist):
 376     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 377         cmd = cmdlist
 378     else:
 379         cmd = ' '.join(cmdlist)
 380     v._run_vagrant_command(['ssh', '-c', cmd])
 381
 382
 383 def update_cache(cachedir, cachefiles):
 384     for srcurl, shasum in cachefiles:
 385         filename = os.path.basename(srcurl)
 386         local_filename = os.path.join(cachedir, filename)
 387
 388         if os.path.exists(local_filename):
 389             local_length = os.path.getsize(local_filename)
 390         else:
 391             local_length = -1
 392
 393         resume_header = {}
 394         download = True
 395
 396         try:
 397             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 398             if r.status_code == 200:
 399                 content_length = int(r.headers.get('content-length'))
 400             else:
 401                 content_length = local_length  # skip the download
 402         except requests.exceptions.RequestException as e:
 403             content_length = local_length  # skip the download
 404             logger.warn('%s', e)
 405
 406         if local_length == content_length:
 407             download = False
 408         elif local_length > content_length:
 409             logger.info('deleting corrupt file from cache: %s', local_filename)
 410             os.remove(local_filename)
 411             logger.info("Downloading %s to cache", filename)
 412         elif local_length > -1 and local_length < content_length:
 413             logger.info("Resuming download of %s", local_filename)
 414             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 415         else:
 416             logger.info("Downloading %s to cache", filename)
 417
 418         if download:
 419             r = requests.get(srcurl, headers=resume_header,
 420                              stream=True, allow_redirects=True)
 421             content_length = int(r.headers.get('content-length'))
 422             with open(local_filename, 'ab') as f:
 423                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 424                                           expected_size=(content_length / 65536) + 1):
 425                     if chunk:  # filter out keep-alive new chunks
 426                         f.write(chunk)
 427
 428         v = sha256_for_file(local_filename)
 429         if v == shasum:
 430             logger.info("\t...shasum verified for %s", local_filename)
 431         else:
 432             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 433             os.remove(local_filename)
 434             sys.exit(1)
 435
 436
 437 def debug_log_vagrant_vm(vm_dir, config):
 438     if options.verbosity >= 3:
 439         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 440         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 441         logger.debug('> vagrant status')
 442         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 443         logger.debug('> vagrant box list')
 444         subprocess.call(['vagrant', 'box', 'list'])
 445         if config['vm_provider'] == 'libvirt':
 446             logger.debug('> virsh -c qmeu:///system list --all')
 447             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 448             domain = 'buildserver_default'
 449             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 450             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 451
 452
 453 def main():
 454     global cachedir, cachefiles, config, tail
 455
 456     if options.skip_cache_update:
 457         logger.info('skipping cache update and verification...')
 458     else:
 459         update_cache(cachedir, cachefiles)
 460
 461     # use VirtualBox software virtualization if hardware is not available,
 462     # like if this is being run in kvm or some other VM platform, like
 463     # http://jenkins.debian.net, the values are 'on' or 'off'
 464     if sys.platform.startswith('darwin'):
 465         # all < 10 year old Macs work, and OSX servers as VM host are very
 466         # rare, but this could also be auto-detected if someone codes it
 467         config['hwvirtex'] = 'on'
 468         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 469     elif os.path.exists('/proc/cpuinfo'):
 470         with open('/proc/cpuinfo') as f:
 471             contents = f.read()
 472         if 'vmx' in contents or 'svm' in contents:
 473             config['hwvirtex'] = 'on'
 474             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 475
 476     serverdir = os.path.join(os.getcwd(), 'buildserver')
 477     logfilename = os.path.join(serverdir, 'up.log')
 478     if not os.path.exists(logfilename):
 479         open(logfilename, 'a').close()  # create blank file
 480     log_cm = vagrant.make_file_cm(logfilename)
 481     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 482
 483     if options.verbosity >= 2:
 484         tail = fdroidserver.tail.Tail(logfilename)
 485         tail.start()
 486
 487     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 488     if options.clean:
 489         vm.destroy()
 490
 491     # Check against the existing Vagrantfile.yaml, and if they differ, we
 492     # need to create a new box:
 493     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 494     writevf = True
 495     if os.path.exists(vf):
 496         logger.info('Halting %s', serverdir)
 497         v.halt()
 498         with open(vf, 'r', encoding='utf-8') as f:
 499             oldconfig = yaml.load(f)
 500         if config != oldconfig:
 501             logger.info("Server configuration has changed, rebuild from scratch is required")
 502             vm.destroy()
 503         else:
 504             logger.info("Re-provisioning existing server")
 505             writevf = False
 506     else:
 507         logger.info("No existing server - building from scratch")
 508     if writevf:
 509         with open(vf, 'w', encoding='utf-8') as f:
 510             yaml.dump(config, f)
 511
 512     if config['vm_provider'] == 'libvirt':
 513         found_basebox = False
 514         needs_mutate = False
 515         for box in v.box_list():
 516             if box.name == config['basebox']:
 517                 found_basebox = True
 518                 if box.provider != 'libvirt':
 519                     needs_mutate = True
 520                 continue
 521         if not found_basebox:
 522             if isinstance(config['baseboxurl'], str):
 523                 baseboxurl = config['baseboxurl']
 524             else:
 525                 baseboxurl = config['baseboxurl'][0]
 526             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 527             v.box_add(config['basebox'], baseboxurl)
 528             needs_mutate = True
 529         if needs_mutate:
 530             logger.info('Converting %s to libvirt format', config['basebox'])
 531             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 532             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 533             v.box_remove(config['basebox'], 'virtualbox')
 534
 535     logger.info("Configuring build server VM")
 536     debug_log_vagrant_vm(serverdir, config)
 537     try:
 538         v.up(provision=True)
 539     except subprocess.CalledProcessError:
 540         debug_log_vagrant_vm(serverdir, config)
 541         logger.error("'vagrant up' failed, is the base box missing?")
 542         sys.exit(1)
 543
 544     if config['copy_caches_from_host']:
 545         ssh_config = v.ssh_config()
 546         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 547         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 548         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 549         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 550
 551         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 552             fullpath = os.path.join(os.getenv('HOME'), d)
 553             if os.path.isdir(fullpath):
 554                 ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
 555                                         '-o StrictHostKeyChecking=no',
 556                                         '-o UserKnownHostsFile=/dev/null',
 557                                         '-o LogLevel=FATAL',
 558                                         '-o IdentitiesOnly=yes',
 559                                         '-o PasswordAuthentication=no'))
 560                 # TODO vagrant 1.5+ provides `vagrant rsync`
 561                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 562                 subprocess.call(['rsync', '-ax', '--delete', '-e',
 563                                  ssh_command,
 564                                  fullpath + '/',
 565                                  user + '@' + hostname + ':~/' + d + '/'])
 566
 567         # this file changes every time but should not be cached
 568         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 569         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 570
 571     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 572                          universal_newlines=True)
 573     buildserverid = p.communicate()[0].strip()
 574     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 575     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 576
 577     logger.info("Stopping build server VM")
 578     v.halt()
 579
 580     logger.info("Packaging")
 581     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 582     if os.path.exists(boxfile):
 583         os.remove(boxfile)
 584
 585     vm.package(output=boxfile)
 586
 587     logger.info("Adding box")
 588     vm.box_add('buildserver', boxfile, force=True)
 589
 590     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 591         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 592         sys.exit(1)
 593
 594     if not options.keep_box_file:
 595         logger.debug('box added to vagrant, ' +
 596                      'removing generated box file \'%s\'',
 597                      boxfile)
 598         os.remove(boxfile)
 599
 600
 601 if __name__ == '__main__':
 602     try:
 603         main()
 604     finally:
 605         if tail is not None:
 606             tail.stop()