makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 128      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 129     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 130      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 131     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 132      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 133     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 134      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 135     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 136      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 137     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 138      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 139     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 140      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 141     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 142      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 143     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 144      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 145     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 146      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 147     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 148      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 149     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 150      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 151     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 152      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 153     ('https://dl.google.com/android/repository/android-14_r04.zip',
 154      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 155     ('https://dl.google.com/android/repository/android-15_r05.zip',
 156      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 157     ('https://dl.google.com/android/repository/android-16_r05.zip',
 158      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 159     ('https://dl.google.com/android/repository/android-17_r03.zip',
 160      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 161     ('https://dl.google.com/android/repository/android-18_r03.zip',
 162      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 163     ('https://dl.google.com/android/repository/android-19_r04.zip',
 164      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 165     ('https://dl.google.com/android/repository/android-20_r02.zip',
 166      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 167     ('https://dl.google.com/android/repository/android-21_r02.zip',
 168      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 169     ('https://dl.google.com/android/repository/android-22_r02.zip',
 170      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 171     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 172      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 173     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 174      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 175     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 176      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 177     ('https://dl.google.com/android/repository/platform-26_r01.zip',
 178      '44e7eca5923320db1abb422aa7a3d0ac9ee199a379af6f5c7603e714d7717561'),
 179     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 180      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 181     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 182      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 184      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 186      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 187     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 188      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 189     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 190      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 192      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 194      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 196      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 197     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 198      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 199     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 200      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 201     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 202      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 204      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 206      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 208      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 210      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 211     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 212      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 213     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 214      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 215     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 216      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 217     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 218      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 220      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 222      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 223     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 224      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 225     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 226      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 228      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 230      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 231     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 232      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 233     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 234      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 236      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 238      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 239     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 240      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 241     # the binaries that Google uses are here:
 242     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 243     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 244      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 245     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 246      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 247     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 248      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 249     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 250      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 251     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 252      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 253     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 254      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 255     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 256      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 257     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 258      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 259     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 260      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 261     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 262      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 263     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 264      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 265     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 266      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 267     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 268      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 269     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 270      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 271     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 272      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 273     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 274      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 275     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 276      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 277     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 278      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 279     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 280      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 281     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 282      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 283     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 284      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 285     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 286      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 287     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 288      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 289     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 290      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 291     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 292      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 293     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 294      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 295     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 296      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 297     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 298      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 299     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 300      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 301     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 302      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 303     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 304      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 305     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 306      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 307     ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
 308      '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
 309     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 310      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 311     ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
 312      'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
 313     ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
 314      '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
 315     ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
 316      'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
 317     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 318      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 319     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 320      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 321     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 322      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 323     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 324      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 325     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 326      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 327     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 328      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 329     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 330      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 331     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 332      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 333     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 334      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 335 ]
 336
 337
 338 def sha256_for_file(path):
 339     with open(path, 'rb') as f:
 340         s = hashlib.sha256()
 341         while True:
 342             data = f.read(4096)
 343             if not data:
 344                 break
 345             s.update(data)
 346         return s.hexdigest()
 347
 348
 349 def run_via_vagrant_ssh(v, cmdlist):
 350     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 351         cmd = cmdlist
 352     else:
 353         cmd = ' '.join(cmdlist)
 354     v._run_vagrant_command(['ssh', '-c', cmd])
 355
 356
 357 def update_cache(cachedir, cachefiles):
 358     for srcurl, shasum in cachefiles:
 359         filename = os.path.basename(srcurl)
 360         local_filename = os.path.join(cachedir, filename)
 361
 362         if os.path.exists(local_filename):
 363             local_length = os.path.getsize(local_filename)
 364         else:
 365             local_length = -1
 366
 367         resume_header = {}
 368         download = True
 369
 370         try:
 371             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 372             if r.status_code == 200:
 373                 content_length = int(r.headers.get('content-length'))
 374             else:
 375                 content_length = local_length  # skip the download
 376         except requests.exceptions.RequestException as e:
 377             content_length = local_length  # skip the download
 378             logger.warn('%s', e)
 379
 380         if local_length == content_length:
 381             download = False
 382         elif local_length > content_length:
 383             logger.info('deleting corrupt file from cache: %s', local_filename)
 384             os.remove(local_filename)
 385             logger.info("Downloading %s to cache", filename)
 386         elif local_length > -1 and local_length < content_length:
 387             logger.info("Resuming download of %s", local_filename)
 388             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 389         else:
 390             logger.info("Downloading %s to cache", filename)
 391
 392         if download:
 393             r = requests.get(srcurl, headers=resume_header,
 394                              stream=True, verify=False, allow_redirects=True)
 395             content_length = int(r.headers.get('content-length'))
 396             with open(local_filename, 'ab') as f:
 397                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 398                                           expected_size=(content_length / 65536) + 1):
 399                     if chunk:  # filter out keep-alive new chunks
 400                         f.write(chunk)
 401
 402         v = sha256_for_file(local_filename)
 403         if v == shasum:
 404             logger.info("\t...shasum verified for %s", local_filename)
 405         else:
 406             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 407             os.remove(local_filename)
 408             sys.exit(1)
 409
 410
 411 def debug_log_vagrant_vm(vm_dir, config):
 412     if options.verbosity >= 3:
 413         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 414         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 415         logger.debug('> vagrant status')
 416         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 417         logger.debug('> vagrant box list')
 418         subprocess.call(['vagrant', 'box', 'list'])
 419         if config['vm_provider'] == 'libvirt':
 420             logger.debug('> virsh -c qmeu:///system list --all')
 421             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 422             domain = 'buildserver_default'
 423             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 424             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 425
 426
 427 def main():
 428     global cachedir, cachefiles, config, tail
 429
 430     if options.skip_cache_update:
 431         logger.info('skipping cache update and verification...')
 432     else:
 433         update_cache(cachedir, cachefiles)
 434
 435     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 436     logger.info("Setting executable bit for %s", local_qt_filename)
 437     os.chmod(local_qt_filename, 0o755)
 438
 439     # use VirtualBox software virtualization if hardware is not available,
 440     # like if this is being run in kvm or some other VM platform, like
 441     # http://jenkins.debian.net, the values are 'on' or 'off'
 442     if sys.platform.startswith('darwin'):
 443         # all < 10 year old Macs work, and OSX servers as VM host are very
 444         # rare, but this could also be auto-detected if someone codes it
 445         config['hwvirtex'] = 'on'
 446         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 447     elif os.path.exists('/proc/cpuinfo'):
 448         with open('/proc/cpuinfo') as f:
 449             contents = f.read()
 450         if 'vmx' in contents or 'svm' in contents:
 451             config['hwvirtex'] = 'on'
 452             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 453
 454     serverdir = os.path.join(os.getcwd(), 'buildserver')
 455     logfilename = os.path.join(serverdir, 'up.log')
 456     if not os.path.exists(logfilename):
 457         open(logfilename, 'a').close()  # create blank file
 458     log_cm = vagrant.make_file_cm(logfilename)
 459     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 460
 461     if options.verbosity >= 2:
 462         tail = fdroidserver.tail.Tail(logfilename)
 463         tail.start()
 464
 465     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 466     if options.clean:
 467         vm.destroy()
 468
 469     # Check against the existing Vagrantfile.yaml, and if they differ, we
 470     # need to create a new box:
 471     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 472     writevf = True
 473     if os.path.exists(vf):
 474         logger.info('Halting %s', serverdir)
 475         v.halt()
 476         with open(vf, 'r', encoding='utf-8') as f:
 477             oldconfig = yaml.load(f)
 478         if config != oldconfig:
 479             logger.info("Server configuration has changed, rebuild from scratch is required")
 480             vm.destroy()
 481         else:
 482             logger.info("Re-provisioning existing server")
 483             writevf = False
 484     else:
 485         logger.info("No existing server - building from scratch")
 486     if writevf:
 487         with open(vf, 'w', encoding='utf-8') as f:
 488             yaml.dump(config, f)
 489
 490     if config['vm_provider'] == 'libvirt':
 491         found_basebox = False
 492         needs_mutate = False
 493         for box in v.box_list():
 494             if box.name == config['basebox']:
 495                 found_basebox = True
 496                 if box.provider != 'libvirt':
 497                     needs_mutate = True
 498                 continue
 499         if not found_basebox:
 500             if isinstance(config['baseboxurl'], str):
 501                 baseboxurl = config['baseboxurl']
 502             else:
 503                 baseboxurl = config['baseboxurl'][0]
 504             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 505             v.box_add(config['basebox'], baseboxurl)
 506             needs_mutate = True
 507         if needs_mutate:
 508             logger.info('Converting %s to libvirt format', config['basebox'])
 509             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 510             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 511             v.box_remove(config['basebox'], 'virtualbox')
 512
 513     logger.info("Configuring build server VM")
 514     debug_log_vagrant_vm(serverdir, config)
 515     try:
 516         v.up(provision=True)
 517     except fdroidserver.vmtools.FDroidBuildVmException as e:
 518         debug_log_vagrant_vm(serverdir, config)
 519         logger.exception('could not bring buildserver vm up. %s', e)
 520         sys.exit(1)
 521
 522     if config['copy_caches_from_host']:
 523         ssh_config = v.ssh_config()
 524         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 525         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 526         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 527         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 528
 529         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 530             fullpath = os.path.join(os.getenv('HOME'), d)
 531             if os.path.isdir(fullpath):
 532                 # TODO newer versions of vagrant provide `vagrant rsync`
 533                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 534                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 535                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 536                                  fullpath + '/',
 537                                  user + '@' + hostname + ':~/' + d + '/'])
 538
 539         # this file changes every time but should not be cached
 540         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 541         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 542
 543     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 544                          universal_newlines=True)
 545     buildserverid = p.communicate()[0].strip()
 546     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 547     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 548
 549     logger.info("Stopping build server VM")
 550     v.halt()
 551
 552     logger.info("Packaging")
 553     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 554     if os.path.exists(boxfile):
 555         os.remove(boxfile)
 556
 557     vm.package(output=boxfile)
 558
 559     logger.info("Adding box")
 560     vm.box_add('buildserver', boxfile, force=True)
 561
 562     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 563         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 564         sys.exit(1)
 565
 566     if not options.keep_box_file:
 567         logger.debug('box added to vagrant, ' +
 568                      'removing generated box file \'%s\'',
 569                      boxfile)
 570         os.remove(boxfile)
 571
 572
 573 if __name__ == '__main__':
 574     try:
 575         main()
 576     finally:
 577         if tail is not None:
 578             tail.stop()