makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 128      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 129     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 130      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 131     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 132      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 133     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 134      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 135     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 136      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 137     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 138      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 139     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 140      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 141     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 142      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 143     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 144      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 145     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 146      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 147     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 148      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 149     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 150      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 151     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 152      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 153     ('https://dl.google.com/android/repository/android-14_r04.zip',
 154      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 155     ('https://dl.google.com/android/repository/android-15_r05.zip',
 156      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 157     ('https://dl.google.com/android/repository/android-16_r05.zip',
 158      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 159     ('https://dl.google.com/android/repository/android-17_r03.zip',
 160      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 161     ('https://dl.google.com/android/repository/android-18_r03.zip',
 162      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 163     ('https://dl.google.com/android/repository/android-19_r04.zip',
 164      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 165     ('https://dl.google.com/android/repository/android-20_r02.zip',
 166      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 167     ('https://dl.google.com/android/repository/android-21_r02.zip',
 168      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 169     ('https://dl.google.com/android/repository/android-22_r02.zip',
 170      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 171     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 172      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 173     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 174      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 175     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 176      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 177     ('https://dl.google.com/android/repository/platform-26_r01.zip',
 178      '44e7eca5923320db1abb422aa7a3d0ac9ee199a379af6f5c7603e714d7717561'),
 179     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 180      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 181     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 182      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 184      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 186      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 187     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 188      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 189     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 190      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 192      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 194      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 196      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 197     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 198      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 199     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 200      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 201     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 202      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 204      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 206      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 208      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 210      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 211     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 212      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 213     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 214      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 215     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 216      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 217     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 218      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 220      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 222      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 223     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 224      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 225     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 226      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 228      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 230      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 231     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 232      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 233     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 234      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 236      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 238      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 239     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 240      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 241     # the binaries that Google uses are here:
 242     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 243     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 244      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 245     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 246      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 247     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 248      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 249     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 250      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 251     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 252      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 253     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 254      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 255     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 256      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 257     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 258      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 259     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 260      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 261     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 262      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 263     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 264      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 265     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 266      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 267     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 268      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 269     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 270      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 271     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 272      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 273     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 274      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 275     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 276      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 277     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 278      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 279     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 280      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 281     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 282      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 283     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 284      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 285     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 286      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 287     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 288      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 289     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 290      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 291     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 292      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 293     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 294      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 295     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 296      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 297     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 298      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 299     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 300      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 301     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 302      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 303     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 304      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 305     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 306      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 307     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 308      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 309     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 310      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 311     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 312      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 313     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 314      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 315     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 316      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 317     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 318      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 319     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 320      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 321     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 322      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 323 ]
 324
 325
 326 def sha256_for_file(path):
 327     with open(path, 'rb') as f:
 328         s = hashlib.sha256()
 329         while True:
 330             data = f.read(4096)
 331             if not data:
 332                 break
 333             s.update(data)
 334         return s.hexdigest()
 335
 336
 337 def run_via_vagrant_ssh(v, cmdlist):
 338     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 339         cmd = cmdlist
 340     else:
 341         cmd = ' '.join(cmdlist)
 342     v._run_vagrant_command(['ssh', '-c', cmd])
 343
 344
 345 def update_cache(cachedir, cachefiles):
 346     for srcurl, shasum in cachefiles:
 347         filename = os.path.basename(srcurl)
 348         local_filename = os.path.join(cachedir, filename)
 349
 350         if os.path.exists(local_filename):
 351             local_length = os.path.getsize(local_filename)
 352         else:
 353             local_length = -1
 354
 355         resume_header = {}
 356         download = True
 357
 358         try:
 359             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 360             if r.status_code == 200:
 361                 content_length = int(r.headers.get('content-length'))
 362             else:
 363                 content_length = local_length  # skip the download
 364         except requests.exceptions.RequestException as e:
 365             content_length = local_length  # skip the download
 366             logger.warn('%s', e)
 367
 368         if local_length == content_length:
 369             download = False
 370         elif local_length > content_length:
 371             logger.info('deleting corrupt file from cache: %s', local_filename)
 372             os.remove(local_filename)
 373             logger.info("Downloading %s to cache", filename)
 374         elif local_length > -1 and local_length < content_length:
 375             logger.info("Resuming download of %s", local_filename)
 376             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 377         else:
 378             logger.info("Downloading %s to cache", filename)
 379
 380         if download:
 381             r = requests.get(srcurl, headers=resume_header,
 382                              stream=True, verify=False, allow_redirects=True)
 383             content_length = int(r.headers.get('content-length'))
 384             with open(local_filename, 'ab') as f:
 385                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 386                                           expected_size=(content_length / 65536) + 1):
 387                     if chunk:  # filter out keep-alive new chunks
 388                         f.write(chunk)
 389
 390         v = sha256_for_file(local_filename)
 391         if v == shasum:
 392             logger.info("\t...shasum verified for %s", local_filename)
 393         else:
 394             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 395             os.remove(local_filename)
 396             sys.exit(1)
 397
 398
 399 def debug_log_vagrant_vm(vm_dir, config):
 400     if options.verbosity >= 3:
 401         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 402         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 403         logger.debug('> vagrant status')
 404         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 405         logger.debug('> vagrant box list')
 406         subprocess.call(['vagrant', 'box', 'list'])
 407         if config['vm_provider'] == 'libvirt':
 408             logger.debug('> virsh -c qmeu:///system list --all')
 409             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 410             domain = 'buildserver_default'
 411             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 412             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 413
 414
 415 def main():
 416     global cachedir, cachefiles, config, tail
 417
 418     if options.skip_cache_update:
 419         logger.info('skipping cache update and verification...')
 420     else:
 421         update_cache(cachedir, cachefiles)
 422
 423     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 424     logger.info("Setting executable bit for %s", local_qt_filename)
 425     os.chmod(local_qt_filename, 0o755)
 426
 427     # use VirtualBox software virtualization if hardware is not available,
 428     # like if this is being run in kvm or some other VM platform, like
 429     # http://jenkins.debian.net, the values are 'on' or 'off'
 430     if sys.platform.startswith('darwin'):
 431         # all < 10 year old Macs work, and OSX servers as VM host are very
 432         # rare, but this could also be auto-detected if someone codes it
 433         config['hwvirtex'] = 'on'
 434         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 435     elif os.path.exists('/proc/cpuinfo'):
 436         with open('/proc/cpuinfo') as f:
 437             contents = f.read()
 438         if 'vmx' in contents or 'svm' in contents:
 439             config['hwvirtex'] = 'on'
 440             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 441
 442     serverdir = os.path.join(os.getcwd(), 'buildserver')
 443     logfilename = os.path.join(serverdir, 'up.log')
 444     if not os.path.exists(logfilename):
 445         open(logfilename, 'a').close()  # create blank file
 446     log_cm = vagrant.make_file_cm(logfilename)
 447     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 448
 449     if options.verbosity >= 2:
 450         tail = fdroidserver.tail.Tail(logfilename)
 451         tail.start()
 452
 453     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 454     if options.clean:
 455         vm.destroy()
 456
 457     # Check against the existing Vagrantfile.yaml, and if they differ, we
 458     # need to create a new box:
 459     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 460     writevf = True
 461     if os.path.exists(vf):
 462         logger.info('Halting %s', serverdir)
 463         v.halt()
 464         with open(vf, 'r', encoding='utf-8') as f:
 465             oldconfig = yaml.load(f)
 466         if config != oldconfig:
 467             logger.info("Server configuration has changed, rebuild from scratch is required")
 468             vm.destroy()
 469         else:
 470             logger.info("Re-provisioning existing server")
 471             writevf = False
 472     else:
 473         logger.info("No existing server - building from scratch")
 474     if writevf:
 475         with open(vf, 'w', encoding='utf-8') as f:
 476             yaml.dump(config, f)
 477
 478     if config['vm_provider'] == 'libvirt':
 479         found_basebox = False
 480         needs_mutate = False
 481         for box in v.box_list():
 482             if box.name == config['basebox']:
 483                 found_basebox = True
 484                 if box.provider != 'libvirt':
 485                     needs_mutate = True
 486                 continue
 487         if not found_basebox:
 488             if isinstance(config['baseboxurl'], str):
 489                 baseboxurl = config['baseboxurl']
 490             else:
 491                 baseboxurl = config['baseboxurl'][0]
 492             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 493             v.box_add(config['basebox'], baseboxurl)
 494             needs_mutate = True
 495         if needs_mutate:
 496             logger.info('Converting %s to libvirt format', config['basebox'])
 497             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 498             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 499             v.box_remove(config['basebox'], 'virtualbox')
 500
 501     logger.info("Configuring build server VM")
 502     debug_log_vagrant_vm(serverdir, config)
 503     try:
 504         v.up(provision=True)
 505     except fdroidserver.vmtools.FDroidBuildVmException as e:
 506         debug_log_vagrant_vm(serverdir, config)
 507         logger.exception('could not bring buildserver vm up. %s', e)
 508         sys.exit(1)
 509
 510     if config['copy_caches_from_host']:
 511         ssh_config = v.ssh_config()
 512         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 513         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 514         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 515         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 516
 517         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 518             fullpath = os.path.join(os.getenv('HOME'), d)
 519             if os.path.isdir(fullpath):
 520                 # TODO newer versions of vagrant provide `vagrant rsync`
 521                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 522                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 523                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 524                                  fullpath + '/',
 525                                  user + '@' + hostname + ':~/' + d + '/'])
 526
 527         # this file changes every time but should not be cached
 528         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 529         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 530
 531     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 532                          universal_newlines=True)
 533     buildserverid = p.communicate()[0].strip()
 534     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 535     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 536
 537     logger.info("Stopping build server VM")
 538     v.halt()
 539
 540     logger.info("Packaging")
 541     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 542     if os.path.exists(boxfile):
 543         os.remove(boxfile)
 544
 545     vm.package(output=boxfile)
 546
 547     logger.info("Adding box")
 548     vm.box_add('buildserver', boxfile, force=True)
 549
 550     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 551         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 552         sys.exit(1)
 553
 554     if not options.keep_box_file:
 555         logger.debug('box added to vagrant, ' +
 556                      'removing generated box file \'%s\'',
 557                      boxfile)
 558         os.remove(boxfile)
 559
 560
 561 if __name__ == '__main__':
 562     try:
 563         main()
 564     finally:
 565         if tail is not None:
 566             tail.stop()