makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 import math
  16 import json
  17 import logging
  18 from clint.textui import progress
  19 from optparse import OptionParser
  20 import fdroidserver.tail
  21
  22
  23 parser = OptionParser()
  24 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  25                   help="Spew out even more information than normal")
  26 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  27 parser.add_option("-c", "--clean", action="store_true", default=False,
  28                   help="Build from scratch, rather than attempting to update the existing server")
  29 parser.add_option('--skip-cache-update', action="store_true", default=False,
  30                   help="""Skip downloading and checking cache."""
  31                        """This assumes that the cache is already downloaded completely.""")
  32 options, args = parser.parse_args()
  33
  34 logger = logging.getLogger('fdroid-makebuildserver')
  35 if options.verbosity >= 2:
  36     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  37     logger.setLevel(logging.DEBUG)
  38 elif options.verbosity == 1:
  39     logging.basicConfig(format='%(message)s', level=logging.INFO)
  40     logger.setLevel(logging.INFO)
  41 elif options.verbosity <= 0:
  42     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  43     logger.setLevel(logging.WARNING)
  44
  45
  46 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  47     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  48     sys.exit(1)
  49
  50 tail = None
  51
  52 # set up default config
  53 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  54 logger.debug('cachedir set to: %s', cachedir)
  55
  56 config = {
  57     'basebox': 'jessie64',
  58     'baseboxurl': [
  59         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  60         'https://f-droid.org/jessie64.box',
  61     ],
  62     'debian_mirror': 'http://http.debian.net/debian/',
  63     'apt_package_cache': False,
  64     'copy_caches_from_host': False,
  65     'boot_timeout': 600,
  66     'cachedir': cachedir,
  67     'cpus': 1,
  68     'memory': 1024,
  69     'hwvirtex': 'off',
  70     'vm_provider': 'virtualbox',
  71 }
  72
  73 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  74     try:
  75         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  76     except subprocess.CalledProcessError as e:
  77         virt = 'none'
  78     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  79         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  80         config['vm_provider'] = 'libvirt'
  81         config['domain'] = 'buildserver_default'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84 logger.debug('deceted virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116
 117 cachefiles = [
 118     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 119      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 120     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 121      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 122     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 123      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 124     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 125      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 126     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 127      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 128     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 129      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 130     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 131      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 132     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 133      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 134     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 135      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 136     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 137      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 138     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 139      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 140     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 141      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 142     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 143      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 144     ('https://dl.google.com/android/repository/android-14_r04.zip',
 145      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 146     ('https://dl.google.com/android/repository/android-15_r05.zip',
 147      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 148     ('https://dl.google.com/android/repository/android-16_r05.zip',
 149      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 150     ('https://dl.google.com/android/repository/android-17_r03.zip',
 151      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 152     ('https://dl.google.com/android/repository/android-18_r03.zip',
 153      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 154     ('https://dl.google.com/android/repository/android-19_r04.zip',
 155      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 156     ('https://dl.google.com/android/repository/android-20_r02.zip',
 157      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 158     ('https://dl.google.com/android/repository/android-21_r02.zip',
 159      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 160     ('https://dl.google.com/android/repository/android-22_r02.zip',
 161      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 162     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 163      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 164     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 165      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 166     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 167      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 168     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 169      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 170     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 171      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 172     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 173      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 174     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 175      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 176     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 177      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 178     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 179      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 180     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 181      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 182     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 183      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 184     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 185      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 186     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 187      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 188     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 189      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 190     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 191      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 192     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 193      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 194     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 195      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 196     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 197      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 198     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 199      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 200     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 201      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 202     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 203      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 204     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 205      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 206     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 207      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 208     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 209      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 210     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 211      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 212     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 213      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 214     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 215      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 216     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 217      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 218     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 219      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 220     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 221      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 222     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 223      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 224     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 225      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 226     # the binaries that Google uses are here:
 227     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 228     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 229      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 230     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 231      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 232     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 233      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 234     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 235      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 236     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 237      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 238     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 239      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 240     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 241      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 242     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 243      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 244     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 245      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 246     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 247      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 248     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 249      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 250     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 251      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 252     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 253      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 254     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 255      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 256     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 257      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 258     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 259      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 260     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 261      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 262     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 263      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 264     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 265      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 266     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 267      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 268     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 269      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 270     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 271      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 272     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 273      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 274     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 275      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 276     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 277      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 278     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 279      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 280     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 281      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 282     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 283      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 284     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 285      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 286     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 287      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 288     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 289      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 290     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 291      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 292     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 293      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 294     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 295      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 296     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 297      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 298     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 299      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 300     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 301      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 302     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 303      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 304     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 305      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 306     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 307      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 308 ]
 309
 310
 311 def sha256_for_file(path):
 312     with open(path, 'rb') as f:
 313         s = hashlib.sha256()
 314         while True:
 315             data = f.read(4096)
 316             if not data:
 317                 break
 318             s.update(data)
 319         return s.hexdigest()
 320
 321
 322 def destroy_current_image(v, serverdir):
 323     global config
 324
 325     logger.info('destroying buildserver vm, removing images and vagrant-configs...')
 326
 327     try:
 328         v.destroy()
 329         logger.debug('vagrant destroy completed')
 330     except subprocess.CalledProcessError as e:
 331         logger.debug('vagrant destroy failed: %s', e)
 332     try:
 333         subprocess.check_call(['vagrant', 'global-status', '--prune'])
 334     except subprocess.CalledProcessError as e:
 335         logger.debug('pruning global vagrant status failed: %s', e)
 336
 337     try:
 338         shutil.rmtree(os.path.join(serverdir, '.vagrant'))
 339     except Exception as e:
 340         logger.debug("could not delete vagrant dir: %s, %s", os.path.join(serverdir, '.vagrant'), e)
 341
 342     if config['vm_provider'] == 'libvirt':
 343         import libvirt
 344         try:
 345             conn = libvirt.open('qemu:///system')
 346             try:
 347                 dom = conn.lookupByName(config['domain'])
 348                 try:
 349                     logger.debug('virsh -c qemu:///system destroy %s', config['domain'])
 350                     subprocess.check_call(['virsh', '-c', 'qemu:///system', 'destroy', config['domain']])
 351                 except subprocess.CalledProcessError as e:
 352                     logging.info("could not force libvirt domain '%s' off: %s", config['domain'], e)
 353                 try:
 354                     # libvirt python bindings do not support all flags required
 355                     # for undefining domains correctly.
 356                     logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', config['domain'])
 357                     subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', config['domain'], '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
 358                 except subprocess.CalledProcessError as e:
 359                     logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e)
 360             except libvirt.libvirtError as e:
 361                 logging.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e)
 362         except libvirt.libvirtError as e:
 363             logging.critical('could not connect to libvirtd: %s', e)
 364             sys.exit(1)
 365
 366
 367 def kvm_package(boxfile):
 368     '''
 369     Hack to replace missing `vagrant package` for kvm, based  on the script
 370     `tools/create_box.sh from vagrant-libvirt
 371     '''
 372     import libvirt
 373     virConnect = libvirt.open('qemu:///system')
 374     storagePool = virConnect.storagePoolLookupByName('default')
 375     if storagePool:
 376         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 377         imagepath = vol.path()
 378         # TODO use a libvirt storage pool to ensure the img file is readable
 379         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 380         shutil.copy2(imagepath, 'box.img')
 381         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 382         img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True)
 383         img_info = json.loads(img_info_raw.decode('utf-8'))
 384         metadata = {"provider": "libvirt",
 385                     "format": img_info['format'],
 386                     "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3),
 387                     }
 388
 389         vagrantfile = """Vagrant.configure("2") do |config|
 390   config.ssh.username = "vagrant"
 391   config.ssh.password = "vagrant"
 392
 393   config.vm.provider :libvirt do |libvirt|
 394
 395     libvirt.driver = "kvm"
 396     libvirt.host = ""
 397     libvirt.connect_via_ssh = false
 398     libvirt.storage_pool_name = "default"
 399
 400   end
 401 end
 402 """
 403         with open('metadata.json', 'w') as fp:
 404             fp.write(json.dumps(metadata))
 405         with open('Vagrantfile', 'w') as fp:
 406             fp.write(vagrantfile)
 407         with tarfile.open(boxfile, 'w:gz') as tar:
 408             tar.add('metadata.json')
 409             tar.add('Vagrantfile')
 410             tar.add('box.img')
 411         os.remove('metadata.json')
 412         os.remove('Vagrantfile')
 413         os.remove('box.img')
 414
 415
 416 def run_via_vagrant_ssh(v, cmdlist):
 417     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 418         cmd = cmdlist
 419     else:
 420         cmd = ' '.join(cmdlist)
 421     v._run_vagrant_command(['ssh', '-c', cmd])
 422
 423
 424 def update_cache(cachedir, cachefiles):
 425     for srcurl, shasum in cachefiles:
 426         filename = os.path.basename(srcurl)
 427         local_filename = os.path.join(cachedir, filename)
 428
 429         if os.path.exists(local_filename):
 430             local_length = os.path.getsize(local_filename)
 431         else:
 432             local_length = -1
 433
 434         resume_header = {}
 435         download = True
 436
 437         try:
 438             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 439             if r.status_code == 200:
 440                 content_length = int(r.headers.get('content-length'))
 441             else:
 442                 content_length = local_length  # skip the download
 443         except requests.exceptions.RequestException as e:
 444             content_length = local_length  # skip the download
 445             logger.warn('%s', e)
 446
 447         if local_length == content_length:
 448             download = False
 449         elif local_length > content_length:
 450             logger.info('deleting corrupt file from cache: %s', local_filename)
 451             os.remove(local_filename)
 452             logger.info("Downloading %s to cache", filename)
 453         elif local_length > -1 and local_length < content_length:
 454             logger.info("Resuming download of %s", local_filename)
 455             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 456         else:
 457             logger.info("Downloading %s to cache", filename)
 458
 459         if download:
 460             r = requests.get(srcurl, headers=resume_header,
 461                              stream=True, verify=False, allow_redirects=True)
 462             content_length = int(r.headers.get('content-length'))
 463             with open(local_filename, 'ab') as f:
 464                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 465                                           expected_size=(content_length / 65536) + 1):
 466                     if chunk:  # filter out keep-alive new chunks
 467                         f.write(chunk)
 468
 469         v = sha256_for_file(local_filename)
 470         if v == shasum:
 471             logger.info("\t...shasum verified for %s", local_filename)
 472         else:
 473             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 474             os.remove(local_filename)
 475             sys.exit(1)
 476
 477
 478 def debug_log_vagrant_vm(vm_dir, vm_name):
 479     if options.verbosity >= 3:
 480         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 481         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 482         logger.debug('> vagrant status')
 483         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 484         logger.debug('> vagrant box list')
 485         subprocess.call(['vagrant', 'box', 'list'])
 486         logger.debug('> virsh -c qmeu:///system list --all')
 487         subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 488         logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name)
 489         subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name])
 490
 491
 492 def main():
 493     global cachedir, cachefiles, config, tail
 494
 495     if options.skip_cache_update:
 496         logger.info('skipping cache update and verification...')
 497     else:
 498         update_cache(cachedir, cachefiles)
 499
 500     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 501     logger.info("Setting executable bit for %s", local_qt_filename)
 502     os.chmod(local_qt_filename, 0o755)
 503
 504     # use VirtualBox software virtualization if hardware is not available,
 505     # like if this is being run in kvm or some other VM platform, like
 506     # http://jenkins.debian.net, the values are 'on' or 'off'
 507     if sys.platform.startswith('darwin'):
 508         # all < 10 year old Macs work, and OSX servers as VM host are very
 509         # rare, but this could also be auto-detected if someone codes it
 510         config['hwvirtex'] = 'on'
 511         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 512     elif os.path.exists('/proc/cpuinfo'):
 513         with open('/proc/cpuinfo') as f:
 514             contents = f.read()
 515         if 'vmx' in contents or 'svm' in contents:
 516             config['hwvirtex'] = 'on'
 517             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 518
 519     serverdir = os.path.join(os.getcwd(), 'buildserver')
 520     logfilename = os.path.join(serverdir, 'up.log')
 521     if not os.path.exists(logfilename):
 522         open(logfilename, 'a').close()  # create blank file
 523     log_cm = vagrant.make_file_cm(logfilename)
 524     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 525
 526     if options.verbosity >= 2:
 527         tail = fdroidserver.tail.Tail(logfilename)
 528         tail.start()
 529
 530     if options.clean:
 531         destroy_current_image(v, serverdir)
 532
 533     # Check against the existing Vagrantfile.yaml, and if they differ, we
 534     # need to create a new box:
 535     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 536     writevf = True
 537     if os.path.exists(vf):
 538         logger.info('Halting %s', serverdir)
 539         v.halt()
 540         with open(vf, 'r', encoding='utf-8') as f:
 541             oldconfig = yaml.load(f)
 542         if config != oldconfig:
 543             logger.info("Server configuration has changed, rebuild from scratch is required")
 544             destroy_current_image(v, serverdir)
 545         else:
 546             logger.info("Re-provisioning existing server")
 547             writevf = False
 548     else:
 549         logger.info("No existing server - building from scratch")
 550     if writevf:
 551         with open(vf, 'w', encoding='utf-8') as f:
 552             yaml.dump(config, f)
 553
 554     if config['vm_provider'] == 'libvirt':
 555         found_basebox = False
 556         needs_mutate = False
 557         for box in v.box_list():
 558             if box.name == config['basebox']:
 559                 found_basebox = True
 560                 if box.provider != 'libvirt':
 561                     needs_mutate = True
 562                 continue
 563         if not found_basebox:
 564             if isinstance(config['baseboxurl'], str):
 565                 baseboxurl = config['baseboxurl']
 566             else:
 567                 baseboxurl = config['baseboxurl'][0]
 568             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 569             v.box_add(config['basebox'], baseboxurl)
 570             needs_mutate = True
 571         if needs_mutate:
 572             logger.info('Converting %s to libvirt format', config['basebox'])
 573             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 574             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 575             v.box_remove(config['basebox'], 'virtualbox')
 576
 577     logger.info("Configuring build server VM")
 578     debug_log_vagrant_vm(serverdir, config['domain'])
 579     try:
 580         try:
 581             v.up(provision=True)
 582         except subprocess.CalledProcessError as e:
 583             v.up(provision=True)
 584     except subprocess.CalledProcessError as e:
 585         debug_log_vagrant_vm(serverdir, config['domain'])
 586         logging.critical('could not bring buildserver vm up. %s', e)
 587         sys.exit(1)
 588
 589     if config['copy_caches_from_host']:
 590         ssh_config = v.ssh_config()
 591         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 592         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 593         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 594         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 595
 596         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 597             fullpath = os.path.join(os.getenv('HOME'), d)
 598             if os.path.isdir(fullpath):
 599                 # TODO newer versions of vagrant provide `vagrant rsync`
 600                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 601                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 602                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 603                                  fullpath + '/',
 604                                  user + '@' + hostname + ':~/' + d + '/'])
 605
 606         # this file changes every time but should not be cached
 607         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 608         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 609
 610     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 611                          universal_newlines=True)
 612     buildserverid = p.communicate()[0].strip()
 613     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 614     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 615
 616     logger.info("Stopping build server VM")
 617     v.halt()
 618
 619     logger.info("Packaging")
 620     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 621     if os.path.exists(boxfile):
 622         os.remove(boxfile)
 623
 624     if config['vm_provider'] == 'libvirt':
 625         kvm_package(boxfile)
 626     else:
 627         v.package(output=boxfile)
 628
 629     logger.info("Adding box")
 630     v.box_add('buildserver', boxfile, force=True)
 631
 632     os.remove(boxfile)
 633
 634
 635 if __name__ == '__main__':
 636     try:
 637         main()
 638     finally:
 639         if tail is not None:
 640             tail.stop()