makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import subprocess
  10 import vagrant
  11 import hashlib
  12 import yaml
  13 from clint.textui import progress
  14 from optparse import OptionParser
  15 import fdroidserver.tail
  16
  17
  18 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  19     print('This must be run as ./makebuildserver in fdroidserver.git!')
  20     sys.exit(1)
  21
  22
  23 tail = None
  24
  25 parser = OptionParser()
  26 parser.add_option("-v", "--verbose", action="store_true", default=False,
  27                   help="Spew out even more information than normal")
  28 parser.add_option("-c", "--clean", action="store_true", default=False,
  29                   help="Build from scratch, rather than attempting to update the existing server")
  30 options, args = parser.parse_args()
  31
  32 # set up default config
  33 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  34 config = {
  35     'basebox': 'jessie64',
  36     'baseboxurl': [
  37         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  38         'https://f-droid.org/jessie64.box',
  39     ],
  40     'debian_mirror': 'http://http.debian.net/debian/',
  41     'apt_package_cache': False,
  42     'copy_caches_from_host': False,
  43     'boot_timeout': 600,
  44     'cachedir': cachedir,
  45     'cpus': 1,
  46     'memory': 1024,
  47     'hwvirtex': 'off',
  48     'vm_provider': 'virtualbox',
  49 }
  50
  51 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  52     try:
  53         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  54     except subprocess.CalledProcessError as e:
  55         virt = 'none'
  56     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  57         print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  58         config['vm_provider'] = 'libvirt'
  59     elif virt != 'none':
  60         print('Running in an unsupported VM guest (' + virt + ')!')
  61
  62 # load config file, if present
  63 if os.path.exists('makebuildserver.config.py'):
  64     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  65 elif os.path.exists('makebs.config.py'):
  66     # this is the old name for the config file
  67     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  68 if '__builtins__' in config:
  69     del(config['__builtins__'])  # added by compile/exec
  70
  71 # Update cached files.
  72 cachedir = config['cachedir']
  73 if not os.path.exists(cachedir):
  74     os.makedirs(cachedir, 0o755)
  75
  76 if config['vm_provider'] == 'libvirt':
  77     tmp = cachedir
  78     while tmp != '/':
  79         mode = os.stat(tmp).st_mode
  80         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
  81             print('ERROR:', tmp, 'will not be accessible to the VM!  To fix, run:')
  82             print('  chmod a+X', tmp)
  83             sys.exit(1)
  84         tmp = os.path.dirname(tmp)
  85
  86 if config['apt_package_cache']:
  87     config['aptcachedir'] = cachedir + '/apt/archives'
  88
  89 cachefiles = [
  90     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
  91      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
  92     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
  93      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
  94     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
  95      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
  96     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
  97      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
  98     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
  99      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 100     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 101      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 102     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 103      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 104     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 105      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 106     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 107      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 108     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 109      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 110     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 111      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 112     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 113      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 114     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 115      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 116     ('https://dl.google.com/android/repository/android-14_r04.zip',
 117      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 118     ('https://dl.google.com/android/repository/android-15_r05.zip',
 119      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 120     ('https://dl.google.com/android/repository/android-16_r05.zip',
 121      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 122     ('https://dl.google.com/android/repository/android-17_r03.zip',
 123      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 124     ('https://dl.google.com/android/repository/android-18_r03.zip',
 125      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 126     ('https://dl.google.com/android/repository/android-19_r04.zip',
 127      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 128     ('https://dl.google.com/android/repository/android-20_r02.zip',
 129      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 130     ('https://dl.google.com/android/repository/android-21_r02.zip',
 131      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 132     ('https://dl.google.com/android/repository/android-22_r02.zip',
 133      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 134     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 135      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 136     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 137      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 138     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 139      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 140     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 141      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 142     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 143      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 144     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 145      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 146     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 147      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 148     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 149      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 150     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 151      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 152     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 153      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 154     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 155      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 156     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 157      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 158     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 159      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 160     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 161      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 162     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 163      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 164     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 165      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 166     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 167      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 168     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 169      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 170     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 171      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 172     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 173      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 174     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 175      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 176     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 177      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 178     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 179      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 180     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 181      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 182     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 183      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 184     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 185      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 186     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 187      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 188     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 189      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 190     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 191      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 192     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 193      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 194     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 195      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 196     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 197      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 198     # the binaries that Google uses are here:
 199     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 200     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 201      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 202     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 203      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 204     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 205      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 206     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 207      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 208     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 209      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 210     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 211      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 212     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 213      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 214     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 215      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 216     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 217      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 218     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 219      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 220     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 221      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 222     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 223      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 224     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 225      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 226     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 227      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 228     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 229      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 230     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 231      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 232     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 233      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 234     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 235      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 236     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 237      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 238     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 239      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 240     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 241      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 242     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 243      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 244     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 245      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 246     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 247      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 248     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 249      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 250     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 251      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 252     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 253      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 254     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 255      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 256     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 257      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 258     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 259      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 260     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 261      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 262     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 263      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 264     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 265      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 266     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 267      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 268     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 269      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 270     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 271      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 272     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 273      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 274     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 275      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 276     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 277      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 278     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 279      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 280 ]
 281
 282
 283 def sha256_for_file(path):
 284     with open(path, 'rb') as f:
 285         s = hashlib.sha256()
 286         while True:
 287             data = f.read(4096)
 288             if not data:
 289                 break
 290             s.update(data)
 291         return s.hexdigest()
 292
 293
 294 def run_via_vagrant_ssh(v, cmdlist):
 295     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 296         cmd = cmdlist
 297     else:
 298         cmd = ' '.join(cmdlist)
 299     v._run_vagrant_command(['ssh', '-c', cmd])
 300
 301
 302 def main():
 303     global cachedir, cachefiles, config, tail
 304
 305     for srcurl, shasum in cachefiles:
 306         filename = os.path.basename(srcurl)
 307         local_filename = os.path.join(cachedir, filename)
 308
 309         if os.path.exists(local_filename):
 310             local_length = os.path.getsize(local_filename)
 311         else:
 312             local_length = -1
 313
 314         resume_header = {}
 315         download = True
 316
 317         try:
 318             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 319             if r.status_code == 200:
 320                 content_length = int(r.headers.get('content-length'))
 321             else:
 322                 content_length = local_length  # skip the download
 323         except requests.exceptions.RequestException as e:
 324             content_length = local_length  # skip the download
 325             print(e)
 326
 327         if local_length == content_length:
 328             download = False
 329         elif local_length > content_length:
 330             print('deleting corrupt file from cache: ' + local_filename)
 331             os.remove(local_filename)
 332             print("Downloading " + filename + " to cache")
 333         elif local_length > -1 and local_length < content_length:
 334             print("Resuming download of " + local_filename)
 335             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 336         else:
 337             print("Downloading " + filename + " to cache")
 338
 339         if download:
 340             r = requests.get(srcurl, headers=resume_header,
 341                              stream=True, verify=False, allow_redirects=True)
 342             content_length = int(r.headers.get('content-length'))
 343             with open(local_filename, 'ab') as f:
 344                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 345                                           expected_size=(content_length / 65536) + 1):
 346                     if chunk:  # filter out keep-alive new chunks
 347                         f.write(chunk)
 348
 349         v = sha256_for_file(local_filename)
 350         if v == shasum:
 351             print("\t...shasum verified for " + local_filename)
 352         else:
 353             print("Invalid shasum of '" + v + "' detected for " + local_filename)
 354             os.remove(local_filename)
 355             sys.exit(1)
 356
 357     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 358     print("Setting executable bit for " + local_qt_filename)
 359     os.chmod(local_qt_filename, 0o755)
 360
 361     # use VirtualBox software virtualization if hardware is not available,
 362     # like if this is being run in kvm or some other VM platform, like
 363     # http://jenkins.debian.net, the values are 'on' or 'off'
 364     if sys.platform.startswith('darwin'):
 365         # all < 10 year old Macs work, and OSX servers as VM host are very
 366         # rare, but this could also be auto-detected if someone codes it
 367         config['hwvirtex'] = 'on'
 368     elif os.path.exists('/proc/cpuinfo'):
 369         with open('/proc/cpuinfo') as f:
 370             contents = f.read()
 371         if 'vmx' in contents or 'svm' in contents:
 372             config['hwvirtex'] = 'on'
 373
 374     serverdir = os.path.join(os.getcwd(), 'buildserver')
 375     logfilename = os.path.join(serverdir, 'up.log')
 376     if not os.path.exists(logfilename):
 377         open(logfilename, 'a').close()  # create blank file
 378     log_cm = vagrant.make_file_cm(logfilename)
 379     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 380
 381     if options.verbose:
 382         tail = fdroidserver.tail.Tail(logfilename)
 383         tail.start()
 384
 385     if options.clean:
 386         v.destroy()
 387         if config['vm_provider'] == 'libvirt':
 388             subprocess.call(['virsh', 'undefine', 'buildserver_default'])
 389             subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img'])
 390
 391     # Check against the existing Vagrantfile.yaml, and if they differ, we
 392     # need to create a new box:
 393     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 394     writevf = True
 395     if os.path.exists(vf):
 396         print('Halting', serverdir)
 397         v.halt()
 398         with open(vf, 'r', encoding='utf-8') as f:
 399             oldconfig = yaml.load(f)
 400         if config != oldconfig:
 401             print("Server configuration has changed, rebuild from scratch is required")
 402             v.destroy()
 403         else:
 404             print("Re-provisioning existing server")
 405             writevf = False
 406     else:
 407         print("No existing server - building from scratch")
 408     if writevf:
 409         with open(vf, 'w', encoding='utf-8') as f:
 410             yaml.dump(config, f)
 411
 412     if config['vm_provider'] == 'libvirt':
 413         found_basebox = False
 414         needs_mutate = False
 415         for box in v.box_list():
 416             if box.name == config['basebox']:
 417                 found_basebox = True
 418                 if box.provider != 'libvirt':
 419                     needs_mutate = True
 420                 continue
 421         if not found_basebox:
 422             if isinstance(config['baseboxurl'], str):
 423                 baseboxurl = config['baseboxurl']
 424             else:
 425                 baseboxurl = config['baseboxurl'][0]
 426             print('Adding', config['basebox'], 'from', baseboxurl)
 427             v.box_add(config['basebox'], baseboxurl)
 428             needs_mutate = True
 429         if needs_mutate:
 430             print('Converting', config['basebox'], 'to libvirt format')
 431             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 432             print('Removing virtualbox format copy of', config['basebox'])
 433             v.box_remove(config['basebox'], 'virtualbox')
 434
 435     print("Configuring build server VM")
 436     v.up(provision=True)
 437
 438     if config['copy_caches_from_host']:
 439         ssh_config = v.ssh_config()
 440         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 441         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 442         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 443         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 444
 445         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 446             fullpath = os.path.join(os.getenv('HOME'), d)
 447             if os.path.isdir(fullpath):
 448                 # TODO newer versions of vagrant provide `vagrant rsync`
 449                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 450                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 451                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 452                                  fullpath + '/',
 453                                  user + '@' + hostname + ':~/' + d + '/'])
 454
 455         # this file changes every time but should not be cached
 456         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 457         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 458
 459     print("Writing buildserver ID")
 460     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 461                          universal_newlines=True)
 462     buildserverid = p.communicate()[0].strip()
 463     print("...ID is " + buildserverid)
 464     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 465
 466     print("Stopping build server VM")
 467     v.halt()
 468
 469     print("Packaging")
 470     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 471     if os.path.exists(boxfile):
 472         os.remove(boxfile)
 473     v.package(output=boxfile)
 474
 475     print("Adding box")
 476     v.box_add('buildserver', boxfile, force=True)
 477
 478     os.remove(boxfile)
 479
 480
 481 if __name__ == '__main__':
 482     try:
 483         main()
 484     finally:
 485         if tail is not None:
 486             tail.stop()