makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 import math
  16 import json
  17 import logging
  18 from clint.textui import progress
  19 from optparse import OptionParser
  20 import fdroidserver.tail
  21
  22
  23
  24 parser = OptionParser()
  25 #parser.add_option("-v", "--verbose", action="store_true", default=False,
  26 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  27                   help="Spew out even more information than normal")
  28 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  29 parser.add_option("-c", "--clean", action="store_true", default=False,
  30                   help="Build from scratch, rather than attempting to update the existing server")
  31 parser.add_option('--skip-cache-update', action="store_true", default=False,
  32                   help="""Skip downloading and checking cache."""
  33                        """This assumes that the cache is already downloaded completely.""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroid-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83         config['domain'] = 'buildserver_default'
  84     elif virt != 'none':
  85         logger.info('Running in an unsupported VM guest (%s)!', virt)
  86 logger.debug('deceted virt: %s', virt)
  87
  88 # load config file, if present
  89 if os.path.exists('makebuildserver.config.py'):
  90     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  91 elif os.path.exists('makebs.config.py'):
  92     # this is the old name for the config file
  93     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  94 if '__builtins__' in config:
  95     del(config['__builtins__'])  # added by compile/exec
  96 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  97
  98 # Update cached files.
  99 cachedir = config['cachedir']
 100 if not os.path.exists(cachedir):
 101     os.makedirs(cachedir, 0o755)
 102     logger.debug('created cachedir %s because it did not exists.', cachedir)
 103
 104 if config['vm_provider'] == 'libvirt':
 105     tmp = cachedir
 106     while tmp != '/':
 107         mode = os.stat(tmp).st_mode
 108         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 109             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 110             logger.critical('  chmod a+X %s', tmp)
 111             sys.exit(1)
 112         tmp = os.path.dirname(tmp)
 113     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 114
 115 if config['apt_package_cache']:
 116     config['aptcachedir'] = cachedir + '/apt/archives'
 117     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 118
 119 cachefiles = [
 120     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 121      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 122     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 123      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 124     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 125      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 126     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 127      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 128     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 129      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 130     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 131      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 132     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 133      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 134     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 135      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 136     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 137      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 138     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 139      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 140     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 141      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 142     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 143      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 144     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 145      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 146     ('https://dl.google.com/android/repository/android-14_r04.zip',
 147      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 148     ('https://dl.google.com/android/repository/android-15_r05.zip',
 149      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 150     ('https://dl.google.com/android/repository/android-16_r05.zip',
 151      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 152     ('https://dl.google.com/android/repository/android-17_r03.zip',
 153      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 154     ('https://dl.google.com/android/repository/android-18_r03.zip',
 155      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 156     ('https://dl.google.com/android/repository/android-19_r04.zip',
 157      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 158     ('https://dl.google.com/android/repository/android-20_r02.zip',
 159      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 160     ('https://dl.google.com/android/repository/android-21_r02.zip',
 161      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 162     ('https://dl.google.com/android/repository/android-22_r02.zip',
 163      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 164     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 165      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 166     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 167      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 168     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 169      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 170     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 171      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 172     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 173      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 174     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 175      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 176     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 177      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 178     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 179      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 180     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 181      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 182     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 183      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 184     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 185      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 186     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 187      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 188     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 189      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 190     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 191      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 192     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 193      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 194     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 195      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 196     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 197      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 198     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 199      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 200     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 201      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 202     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 203      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 204     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 205      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 206     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 207      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 208     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 209      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 210     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 211      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 212     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 213      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 214     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 215      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 216     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 217      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 218     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 219      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 220     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 221      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 222     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 223      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 224     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 225      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 226     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 227      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 228     # the binaries that Google uses are here:
 229     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 230     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 231      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 232     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 233      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 234     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 235      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 236     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 237      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 238     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 239      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 240     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 241      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 242     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 243      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 244     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 245      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 246     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 247      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 248     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 249      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 250     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 251      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 252     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 253      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 254     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 255      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 256     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 257      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 258     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 259      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 260     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 261      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 262     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 263      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 264     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 265      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 266     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 267      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 268     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 269      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 270     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 271      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 272     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 273      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 274     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 275      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 276     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 277      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 278     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 279      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 280     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 281      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 282     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 283      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 284     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 285      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 286     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 287      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 288     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 289      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 290     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 291      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 292     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 293      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 294     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 295      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 296     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 297      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 298     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 299      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 300     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 301      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 302     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 303      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 304     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 305      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 306     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 307      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 308     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 309      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 310 ]
 311
 312
 313 def sha256_for_file(path):
 314     with open(path, 'rb') as f:
 315         s = hashlib.sha256()
 316         while True:
 317             data = f.read(4096)
 318             if not data:
 319                 break
 320             s.update(data)
 321         return s.hexdigest()
 322
 323
 324 def destroy_current_image(v, serverdir):
 325     global config
 326
 327     logger.info('destroying buildserver vm, removing images and vagrand-configs...')
 328
 329     # cannot run vagrant without the config in the YAML file
 330     if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')):
 331         v.destroy()
 332         logger.debug('vagrant destroy completed')
 333     if logger.level <= logging.DEBUG:
 334         logger.debug('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!')
 335         subprocess.check_call(['vagrant', 'global-status', '--prune'])
 336
 337     try:
 338         shutil.rmtree(os.path.join(serverdir, '.vagrant'))
 339     except Exception as e:
 340         logger.debug("could not delete vagrant dir: %s, %s", os.path.join(serverdir, '.vagrant'), e)
 341
 342     if config['vm_provider'] == 'libvirt':
 343         import libvirt
 344         try:
 345             virConnect = libvirt.open('qemu:///system')
 346             virDomain = virConnect.lookupByName(config['domain'])
 347             if virDomain:
 348                 virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE
 349                                         | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA
 350                                         | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM)
 351             storagePool = virConnect.storagePoolLookupByName('default')
 352             if storagePool:
 353                 for vol in storagePool.listAllVolumes():
 354                     vol.delete()
 355         except libvirt.libvirtError as e:
 356             logger.warn('%s', e)
 357
 358
 359 def kvm_package(boxfile):
 360     '''
 361     Hack to replace missing `vagrant package` for kvm, based  on the script
 362     `tools/create_box.sh from vagrant-libvirt
 363     '''
 364     import libvirt
 365     virConnect = libvirt.open('qemu:///system')
 366     storagePool = virConnect.storagePoolLookupByName('default')
 367     if storagePool:
 368         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 369         imagepath = vol.path()
 370         # TODO use a libvirt storage pool to ensure the img file is readable
 371         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 372         shutil.copy2(imagepath, 'box.img')
 373         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 374         img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True)
 375         img_info = json.loads(img_info_raw.decode('utf-8'))
 376         metadata = {"provider": "libvirt",
 377                     "format": img_info['format'],
 378                     "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3),
 379                     }
 380
 381         vagrantfile = """Vagrant.configure("2") do |config|
 382   config.ssh.username = "vagrant"
 383   config.ssh.password = "vagrant"
 384
 385   config.vm.provider :libvirt do |libvirt|
 386
 387     libvirt.driver = "kvm"
 388     libvirt.host = ""
 389     libvirt.connect_via_ssh = false
 390     libvirt.storage_pool_name = "default"
 391
 392   end
 393 end
 394 """
 395         with open('metadata.json', 'w') as fp:
 396             fp.write(json.dumps(metadata))
 397         with open('Vagrantfile', 'w') as fp:
 398             fp.write(vagrantfile)
 399         with tarfile.open(boxfile, 'w:gz') as tar:
 400             tar.add('metadata.json')
 401             tar.add('Vagrantfile')
 402             tar.add('box.img')
 403         os.remove('metadata.json')
 404         os.remove('Vagrantfile')
 405         os.remove('box.img')
 406
 407
 408 def run_via_vagrant_ssh(v, cmdlist):
 409     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 410         cmd = cmdlist
 411     else:
 412         cmd = ' '.join(cmdlist)
 413     v._run_vagrant_command(['ssh', '-c', cmd])
 414
 415
 416 def update_cache(cachedir, cachefiles):
 417     for srcurl, shasum in cachefiles:
 418         filename = os.path.basename(srcurl)
 419         local_filename = os.path.join(cachedir, filename)
 420
 421         if os.path.exists(local_filename):
 422             local_length = os.path.getsize(local_filename)
 423         else:
 424             local_length = -1
 425
 426         resume_header = {}
 427         download = True
 428
 429         try:
 430             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 431             if r.status_code == 200:
 432                 content_length = int(r.headers.get('content-length'))
 433             else:
 434                 content_length = local_length  # skip the download
 435         except requests.exceptions.RequestException as e:
 436             content_length = local_length  # skip the download
 437             logger.warn('%s', e)
 438
 439         if local_length == content_length:
 440             download = False
 441         elif local_length > content_length:
 442             logger.info('deleting corrupt file from cache: %s', local_filename)
 443             os.remove(local_filename)
 444             logger.info("Downloading %s to cache", filename)
 445         elif local_length > -1 and local_length < content_length:
 446             logger.info("Resuming download of %s", local_filename)
 447             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 448         else:
 449             logger.info("Downloading %s to cache", filename)
 450
 451         if download:
 452             r = requests.get(srcurl, headers=resume_header,
 453                              stream=True, verify=False, allow_redirects=True)
 454             content_length = int(r.headers.get('content-length'))
 455             with open(local_filename, 'ab') as f:
 456                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 457                                           expected_size=(content_length / 65536) + 1):
 458                     if chunk:  # filter out keep-alive new chunks
 459                         f.write(chunk)
 460
 461         v = sha256_for_file(local_filename)
 462         if v == shasum:
 463             logger.info("\t...shasum verified for %s", local_filename)
 464         else:
 465             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 466             os.remove(local_filename)
 467             sys.exit(1)
 468
 469
 470 def debug_log_vagrant_vm(vm_dir, vm_name):
 471     if options.verbosity >= 3:
 472         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 473         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 474         logger.debug('> vagrant status')
 475         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 476         logger.debug('> vagrant box list')
 477         subprocess.call(['vagrant', 'box', 'list'])
 478         logger.debug('> virsh -c qmeu:///system list --all')
 479         subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 480         logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name)
 481         subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name])
 482
 483
 484 def main():
 485     global cachedir, cachefiles, config, tail
 486
 487     if options.skip_cache_update:
 488         logger.info('skipping cache update and verification...')
 489     else:
 490         update_cache(cachedir, cachefiles)
 491
 492     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 493     logger.info("Setting executable bit for %s", local_qt_filename)
 494     os.chmod(local_qt_filename, 0o755)
 495
 496     # use VirtualBox software virtualization if hardware is not available,
 497     # like if this is being run in kvm or some other VM platform, like
 498     # http://jenkins.debian.net, the values are 'on' or 'off'
 499     if sys.platform.startswith('darwin'):
 500         # all < 10 year old Macs work, and OSX servers as VM host are very
 501         # rare, but this could also be auto-detected if someone codes it
 502         config['hwvirtex'] = 'on'
 503         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 504     elif os.path.exists('/proc/cpuinfo'):
 505         with open('/proc/cpuinfo') as f:
 506             contents = f.read()
 507         if 'vmx' in contents or 'svm' in contents:
 508             config['hwvirtex'] = 'on'
 509             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 510
 511     serverdir = os.path.join(os.getcwd(), 'buildserver')
 512     logfilename = os.path.join(serverdir, 'up.log')
 513     if not os.path.exists(logfilename):
 514         open(logfilename, 'a').close()  # create blank file
 515     log_cm = vagrant.make_file_cm(logfilename)
 516     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 517
 518     if options.verbosity >= 2:
 519         tail = fdroidserver.tail.Tail(logfilename)
 520         tail.start()
 521
 522     if options.clean:
 523         destroy_current_image(v, serverdir)
 524
 525     # Check against the existing Vagrantfile.yaml, and if they differ, we
 526     # need to create a new box:
 527     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 528     writevf = True
 529     if os.path.exists(vf):
 530         logger.info('Halting %s', serverdir)
 531         v.halt()
 532         with open(vf, 'r', encoding='utf-8') as f:
 533             oldconfig = yaml.load(f)
 534         if config != oldconfig:
 535             logger.info("Server configuration has changed, rebuild from scratch is required")
 536             destroy_current_image(v, serverdir)
 537         else:
 538             logger.info("Re-provisioning existing server")
 539             writevf = False
 540     else:
 541         logger.info("No existing server - building from scratch")
 542     if writevf:
 543         with open(vf, 'w', encoding='utf-8') as f:
 544             yaml.dump(config, f)
 545
 546     if config['vm_provider'] == 'libvirt':
 547         found_basebox = False
 548         needs_mutate = False
 549         for box in v.box_list():
 550             if box.name == config['basebox']:
 551                 found_basebox = True
 552                 if box.provider != 'libvirt':
 553                     needs_mutate = True
 554                 continue
 555         if not found_basebox:
 556             if isinstance(config['baseboxurl'], str):
 557                 baseboxurl = config['baseboxurl']
 558             else:
 559                 baseboxurl = config['baseboxurl'][0]
 560             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 561             v.box_add(config['basebox'], baseboxurl)
 562             needs_mutate = True
 563         if needs_mutate:
 564             logger.info('Converting %s to libvirt format', config['basebox'])
 565             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 566             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 567             v.box_remove(config['basebox'], 'virtualbox')
 568
 569     logger.info("Configuring build server VM")
 570     debug_log_vagrant_vm(serverdir, 'buildserver_default')
 571     v.up(provision=True)
 572     debug_log_vagrant_vm(serverdir, 'buildserver_default')
 573
 574     if config['copy_caches_from_host']:
 575         ssh_config = v.ssh_config()
 576         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 577         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 578         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 579         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 580
 581         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 582             fullpath = os.path.join(os.getenv('HOME'), d)
 583             if os.path.isdir(fullpath):
 584                 # TODO newer versions of vagrant provide `vagrant rsync`
 585                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 586                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 587                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 588                                  fullpath + '/',
 589                                  user + '@' + hostname + ':~/' + d + '/'])
 590
 591         # this file changes every time but should not be cached
 592         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 593         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 594
 595     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 596                          universal_newlines=True)
 597     buildserverid = p.communicate()[0].strip()
 598     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 599     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 600
 601     logger.info("Stopping build server VM")
 602     v.halt()
 603
 604     logger.info("Packaging")
 605     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 606     if os.path.exists(boxfile):
 607         os.remove(boxfile)
 608
 609     if config['vm_provider'] == 'libvirt':
 610         kvm_package(boxfile)
 611     else:
 612         v.package(output=boxfile)
 613
 614     logger.info("Adding box")
 615     v.box_add('buildserver', boxfile, force=True)
 616
 617     os.remove(boxfile)
 618
 619
 620 if __name__ == '__main__':
 621     try:
 622         main()
 623     finally:
 624         if tail is not None:
 625             tail.stop()