makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 import math
  16 import json
  17 import logging
  18 from clint.textui import progress
  19 from optparse import OptionParser
  20 import fdroidserver.tail
  21
  22
  23 parser = OptionParser()
  24 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  25                   help="Spew out even more information than normal")
  26 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  27 parser.add_option("-c", "--clean", action="store_true", default=False,
  28                   help="Build from scratch, rather than attempting to update the existing server")
  29 parser.add_option('--skip-cache-update', action="store_true", default=False,
  30                   help="""Skip downloading and checking cache."""
  31                        """This assumes that the cache is already downloaded completely.""")
  32 options, args = parser.parse_args()
  33
  34 logger = logging.getLogger('fdroid-makebuildserver')
  35 if options.verbosity >= 2:
  36     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  37     logger.setLevel(logging.DEBUG)
  38 elif options.verbosity == 1:
  39     logging.basicConfig(format='%(message)s', level=logging.INFO)
  40     logger.setLevel(logging.INFO)
  41 elif options.verbosity <= 0:
  42     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  43     logger.setLevel(logging.WARNING)
  44
  45
  46 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  47     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  48     sys.exit(1)
  49
  50 tail = None
  51
  52 # set up default config
  53 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  54 logger.debug('cachedir set to: %s', cachedir)
  55
  56 config = {
  57     'basebox': 'jessie64',
  58     'baseboxurl': [
  59         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  60         'https://f-droid.org/jessie64.box',
  61     ],
  62     'debian_mirror': 'http://http.debian.net/debian/',
  63     'apt_package_cache': False,
  64     'copy_caches_from_host': False,
  65     'boot_timeout': 600,
  66     'cachedir': cachedir,
  67     'cpus': 1,
  68     'memory': 1024,
  69     'hwvirtex': 'off',
  70     'vm_provider': 'virtualbox',
  71 }
  72
  73 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  74     try:
  75         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  76     except subprocess.CalledProcessError as e:
  77         virt = 'none'
  78     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  79         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  80         config['vm_provider'] = 'libvirt'
  81         config['domain'] = 'buildserver_default'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84 logger.debug('deceted virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116
 117 cachefiles = [
 118     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 119      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 120     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 121      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 122     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 123      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 124     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 125      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 126     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 127      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 128     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 129      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 130     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 131      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 132     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 133      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 134     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 135      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 136     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 137      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 138     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 139      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 140     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 141      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 142     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 143      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 144     ('https://dl.google.com/android/repository/android-14_r04.zip',
 145      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 146     ('https://dl.google.com/android/repository/android-15_r05.zip',
 147      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 148     ('https://dl.google.com/android/repository/android-16_r05.zip',
 149      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 150     ('https://dl.google.com/android/repository/android-17_r03.zip',
 151      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 152     ('https://dl.google.com/android/repository/android-18_r03.zip',
 153      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 154     ('https://dl.google.com/android/repository/android-19_r04.zip',
 155      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 156     ('https://dl.google.com/android/repository/android-20_r02.zip',
 157      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 158     ('https://dl.google.com/android/repository/android-21_r02.zip',
 159      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 160     ('https://dl.google.com/android/repository/android-22_r02.zip',
 161      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 162     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 163      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 164     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 165      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 166     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 167      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 168     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 169      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 170     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 171      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 172     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 173      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 174     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 175      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 176     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 177      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 178     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 179      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 180     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 181      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 182     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 183      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 184     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 185      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 186     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 187      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 188     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 189      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 190     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 191      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 192     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 193      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 194     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 195      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 196     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 197      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 198     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 199      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 200     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 201      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 202     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 203      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 204     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 205      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 206     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 207      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 208     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 209      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 210     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 211      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 212     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 213      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 214     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 215      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 216     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 217      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 218     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 219      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 220     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 221      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 222     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 223      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 224     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 225      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 226     # the binaries that Google uses are here:
 227     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 228     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 229      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 230     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 231      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 232     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 233      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 234     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 235      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 236     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 237      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 238     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 239      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 240     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 241      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 242     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 243      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 244     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 245      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 246     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 247      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 248     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 249      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 250     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 251      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 252     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 253      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 254     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 255      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 256     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 257      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 258     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 259      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 260     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 261      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 262     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 263      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 264     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 265      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 266     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 267      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 268     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 269      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 270     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 271      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 272     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 273      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 274     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 275      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 276     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 277      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 278     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 279      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 280     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 281      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 282     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 283      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 284     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 285      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 286     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 287      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 288     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 289      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 290     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 291      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 292     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 293      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 294     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 295      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 296     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 297      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 298     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 299      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 300     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 301      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 302     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 303      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 304     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 305      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 306     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 307      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 308 ]
 309
 310
 311 def sha256_for_file(path):
 312     with open(path, 'rb') as f:
 313         s = hashlib.sha256()
 314         while True:
 315             data = f.read(4096)
 316             if not data:
 317                 break
 318             s.update(data)
 319         return s.hexdigest()
 320
 321
 322 def destroy_current_image(v, serverdir):
 323     global config
 324
 325     logger.info('destroying buildserver vm, removing images and vagrand-configs...')
 326
 327     # cannot run vagrant without the config in the YAML file
 328     if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')):
 329         v.destroy()
 330         logger.debug('vagrant destroy completed')
 331     if logger.level <= logging.DEBUG:
 332         logger.debug('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!')
 333         subprocess.check_call(['vagrant', 'global-status', '--prune'])
 334
 335     try:
 336         shutil.rmtree(os.path.join(serverdir, '.vagrant'))
 337     except Exception as e:
 338         logger.debug("could not delete vagrant dir: %s, %s", os.path.join(serverdir, '.vagrant'), e)
 339
 340     if config['vm_provider'] == 'libvirt':
 341         import libvirt
 342         try:
 343             conn = libvirt.open('qemu:///system')
 344             try:
 345                 dom = conn.lookupByName(config['domain'])
 346                 try:
 347                     logger.debug('virsh -c qemu:///system destroy %s', config['domain'])
 348                     subprocess.check_call(['virsh', '-c', 'qemu:///system', 'destroy', config['domain']])
 349                 except subprocess.CalledProcessError as e:
 350                     logging.info("could not force libvirt domain '%s' off: %s", config['domain'], e)
 351                 try:
 352                     # libvirt python bindings do not support all flags required
 353                     # for undefining domains correctly.
 354                     logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', config['domain'])
 355                     subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', config['domain'], '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
 356                 except subprocess.CalledProcessError as e:
 357                     logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e)
 358             except libvirt.libvirtError as e:
 359                 logging.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e)
 360         except libvirt.libvirtError as e:
 361             logging.critical('could not connect to libvirtd: %s', e)
 362             sys.exit(1)
 363
 364
 365 def kvm_package(boxfile):
 366     '''
 367     Hack to replace missing `vagrant package` for kvm, based  on the script
 368     `tools/create_box.sh from vagrant-libvirt
 369     '''
 370     import libvirt
 371     virConnect = libvirt.open('qemu:///system')
 372     storagePool = virConnect.storagePoolLookupByName('default')
 373     if storagePool:
 374         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 375         imagepath = vol.path()
 376         # TODO use a libvirt storage pool to ensure the img file is readable
 377         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 378         shutil.copy2(imagepath, 'box.img')
 379         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 380         img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True)
 381         img_info = json.loads(img_info_raw.decode('utf-8'))
 382         metadata = {"provider": "libvirt",
 383                     "format": img_info['format'],
 384                     "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3),
 385                     }
 386
 387         vagrantfile = """Vagrant.configure("2") do |config|
 388   config.ssh.username = "vagrant"
 389   config.ssh.password = "vagrant"
 390
 391   config.vm.provider :libvirt do |libvirt|
 392
 393     libvirt.driver = "kvm"
 394     libvirt.host = ""
 395     libvirt.connect_via_ssh = false
 396     libvirt.storage_pool_name = "default"
 397
 398   end
 399 end
 400 """
 401         with open('metadata.json', 'w') as fp:
 402             fp.write(json.dumps(metadata))
 403         with open('Vagrantfile', 'w') as fp:
 404             fp.write(vagrantfile)
 405         with tarfile.open(boxfile, 'w:gz') as tar:
 406             tar.add('metadata.json')
 407             tar.add('Vagrantfile')
 408             tar.add('box.img')
 409         os.remove('metadata.json')
 410         os.remove('Vagrantfile')
 411         os.remove('box.img')
 412
 413
 414 def run_via_vagrant_ssh(v, cmdlist):
 415     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 416         cmd = cmdlist
 417     else:
 418         cmd = ' '.join(cmdlist)
 419     v._run_vagrant_command(['ssh', '-c', cmd])
 420
 421
 422 def update_cache(cachedir, cachefiles):
 423     for srcurl, shasum in cachefiles:
 424         filename = os.path.basename(srcurl)
 425         local_filename = os.path.join(cachedir, filename)
 426
 427         if os.path.exists(local_filename):
 428             local_length = os.path.getsize(local_filename)
 429         else:
 430             local_length = -1
 431
 432         resume_header = {}
 433         download = True
 434
 435         try:
 436             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 437             if r.status_code == 200:
 438                 content_length = int(r.headers.get('content-length'))
 439             else:
 440                 content_length = local_length  # skip the download
 441         except requests.exceptions.RequestException as e:
 442             content_length = local_length  # skip the download
 443             logger.warn('%s', e)
 444
 445         if local_length == content_length:
 446             download = False
 447         elif local_length > content_length:
 448             logger.info('deleting corrupt file from cache: %s', local_filename)
 449             os.remove(local_filename)
 450             logger.info("Downloading %s to cache", filename)
 451         elif local_length > -1 and local_length < content_length:
 452             logger.info("Resuming download of %s", local_filename)
 453             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 454         else:
 455             logger.info("Downloading %s to cache", filename)
 456
 457         if download:
 458             r = requests.get(srcurl, headers=resume_header,
 459                              stream=True, verify=False, allow_redirects=True)
 460             content_length = int(r.headers.get('content-length'))
 461             with open(local_filename, 'ab') as f:
 462                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 463                                           expected_size=(content_length / 65536) + 1):
 464                     if chunk:  # filter out keep-alive new chunks
 465                         f.write(chunk)
 466
 467         v = sha256_for_file(local_filename)
 468         if v == shasum:
 469             logger.info("\t...shasum verified for %s", local_filename)
 470         else:
 471             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 472             os.remove(local_filename)
 473             sys.exit(1)
 474
 475
 476 def debug_log_vagrant_vm(vm_dir, vm_name):
 477     if options.verbosity >= 3:
 478         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 479         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 480         logger.debug('> vagrant status')
 481         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 482         logger.debug('> vagrant box list')
 483         subprocess.call(['vagrant', 'box', 'list'])
 484         logger.debug('> virsh -c qmeu:///system list --all')
 485         subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 486         logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name)
 487         subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name])
 488
 489
 490 def main():
 491     global cachedir, cachefiles, config, tail
 492
 493     if options.skip_cache_update:
 494         logger.info('skipping cache update and verification...')
 495     else:
 496         update_cache(cachedir, cachefiles)
 497
 498     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 499     logger.info("Setting executable bit for %s", local_qt_filename)
 500     os.chmod(local_qt_filename, 0o755)
 501
 502     # use VirtualBox software virtualization if hardware is not available,
 503     # like if this is being run in kvm or some other VM platform, like
 504     # http://jenkins.debian.net, the values are 'on' or 'off'
 505     if sys.platform.startswith('darwin'):
 506         # all < 10 year old Macs work, and OSX servers as VM host are very
 507         # rare, but this could also be auto-detected if someone codes it
 508         config['hwvirtex'] = 'on'
 509         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 510     elif os.path.exists('/proc/cpuinfo'):
 511         with open('/proc/cpuinfo') as f:
 512             contents = f.read()
 513         if 'vmx' in contents or 'svm' in contents:
 514             config['hwvirtex'] = 'on'
 515             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 516
 517     serverdir = os.path.join(os.getcwd(), 'buildserver')
 518     logfilename = os.path.join(serverdir, 'up.log')
 519     if not os.path.exists(logfilename):
 520         open(logfilename, 'a').close()  # create blank file
 521     log_cm = vagrant.make_file_cm(logfilename)
 522     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 523
 524     if options.verbosity >= 2:
 525         tail = fdroidserver.tail.Tail(logfilename)
 526         tail.start()
 527
 528     if options.clean:
 529         destroy_current_image(v, serverdir)
 530
 531     # Check against the existing Vagrantfile.yaml, and if they differ, we
 532     # need to create a new box:
 533     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 534     writevf = True
 535     if os.path.exists(vf):
 536         logger.info('Halting %s', serverdir)
 537         v.halt()
 538         with open(vf, 'r', encoding='utf-8') as f:
 539             oldconfig = yaml.load(f)
 540         if config != oldconfig:
 541             logger.info("Server configuration has changed, rebuild from scratch is required")
 542             destroy_current_image(v, serverdir)
 543         else:
 544             logger.info("Re-provisioning existing server")
 545             writevf = False
 546     else:
 547         logger.info("No existing server - building from scratch")
 548     if writevf:
 549         with open(vf, 'w', encoding='utf-8') as f:
 550             yaml.dump(config, f)
 551
 552     if config['vm_provider'] == 'libvirt':
 553         found_basebox = False
 554         needs_mutate = False
 555         for box in v.box_list():
 556             if box.name == config['basebox']:
 557                 found_basebox = True
 558                 if box.provider != 'libvirt':
 559                     needs_mutate = True
 560                 continue
 561         if not found_basebox:
 562             if isinstance(config['baseboxurl'], str):
 563                 baseboxurl = config['baseboxurl']
 564             else:
 565                 baseboxurl = config['baseboxurl'][0]
 566             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 567             v.box_add(config['basebox'], baseboxurl)
 568             needs_mutate = True
 569         if needs_mutate:
 570             logger.info('Converting %s to libvirt format', config['basebox'])
 571             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 572             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 573             v.box_remove(config['basebox'], 'virtualbox')
 574
 575     logger.info("Configuring build server VM")
 576     debug_log_vagrant_vm(serverdir, 'buildserver_default')
 577     try:
 578         v.up(provision=True)
 579     except subprocess.CalledProcessError as e:
 580         debug_log_vagrant_vm(serverdir, 'buildserver_default')
 581         logging.critical('could not bring buildserver vm up. %s', e)
 582         sys.exit(1)
 583
 584     if config['copy_caches_from_host']:
 585         ssh_config = v.ssh_config()
 586         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 587         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 588         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 589         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 590
 591         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 592             fullpath = os.path.join(os.getenv('HOME'), d)
 593             if os.path.isdir(fullpath):
 594                 # TODO newer versions of vagrant provide `vagrant rsync`
 595                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 596                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 597                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 598                                  fullpath + '/',
 599                                  user + '@' + hostname + ':~/' + d + '/'])
 600
 601         # this file changes every time but should not be cached
 602         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 603         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 604
 605     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 606                          universal_newlines=True)
 607     buildserverid = p.communicate()[0].strip()
 608     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 609     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 610
 611     logger.info("Stopping build server VM")
 612     v.halt()
 613
 614     logger.info("Packaging")
 615     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 616     if os.path.exists(boxfile):
 617         os.remove(boxfile)
 618
 619     if config['vm_provider'] == 'libvirt':
 620         kvm_package(boxfile)
 621     else:
 622         v.package(output=boxfile)
 623
 624     logger.info("Adding box")
 625     v.box_add('buildserver', boxfile, force=True)
 626
 627     os.remove(boxfile)
 628
 629
 630 if __name__ == '__main__':
 631     try:
 632         main()
 633     finally:
 634         if tail is not None:
 635             tail.stop()