makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 import math
  16 import json
  17 import logging
  18 from clint.textui import progress
  19 from optparse import OptionParser
  20 import fdroidserver.tail
  21
  22
  23 parser = OptionParser()
  24 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  25                   help="Spew out even more information than normal")
  26 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  27 parser.add_option("-c", "--clean", action="store_true", default=False,
  28                   help="Build from scratch, rather than attempting to update the existing server")
  29 parser.add_option('--skip-cache-update', action="store_true", default=False,
  30                   help="""Skip downloading and checking cache."""
  31                        """This assumes that the cache is already downloaded completely.""")
  32 options, args = parser.parse_args()
  33
  34 logger = logging.getLogger('fdroid-makebuildserver')
  35 if options.verbosity >= 2:
  36     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  37     logger.setLevel(logging.DEBUG)
  38 elif options.verbosity == 1:
  39     logging.basicConfig(format='%(message)s', level=logging.INFO)
  40     logger.setLevel(logging.INFO)
  41 elif options.verbosity <= 0:
  42     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  43     logger.setLevel(logging.WARNING)
  44
  45
  46 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  47     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  48     sys.exit(1)
  49
  50 tail = None
  51
  52 # set up default config
  53 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  54 logger.debug('cachedir set to: %s', cachedir)
  55
  56 config = {
  57     'basebox': 'jessie64',
  58     'baseboxurl': [
  59         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  60         'https://f-droid.org/jessie64.box',
  61     ],
  62     'debian_mirror': 'http://http.debian.net/debian/',
  63     'apt_package_cache': False,
  64     'copy_caches_from_host': False,
  65     'boot_timeout': 600,
  66     'cachedir': cachedir,
  67     'cpus': 1,
  68     'memory': 1024,
  69     'hwvirtex': 'off',
  70     'vm_provider': 'virtualbox',
  71 }
  72
  73 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  74     try:
  75         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  76     except subprocess.CalledProcessError as e:
  77         virt = 'none'
  78     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  79         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  80         config['vm_provider'] = 'libvirt'
  81         config['domain'] = 'buildserver_default'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84 logger.debug('deceted virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116
 117 cachefiles = [
 118     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 119      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 120     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 121      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 122     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 123      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 124     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 125      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 126     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 127      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 128     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 129      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 130     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 131      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 132     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 133      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 134     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 135      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 136     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 137      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 138     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 139      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 140     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 141      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 142     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 143      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 144     ('https://dl.google.com/android/repository/android-14_r04.zip',
 145      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 146     ('https://dl.google.com/android/repository/android-15_r05.zip',
 147      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 148     ('https://dl.google.com/android/repository/android-16_r05.zip',
 149      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 150     ('https://dl.google.com/android/repository/android-17_r03.zip',
 151      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 152     ('https://dl.google.com/android/repository/android-18_r03.zip',
 153      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 154     ('https://dl.google.com/android/repository/android-19_r04.zip',
 155      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 156     ('https://dl.google.com/android/repository/android-20_r02.zip',
 157      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 158     ('https://dl.google.com/android/repository/android-21_r02.zip',
 159      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 160     ('https://dl.google.com/android/repository/android-22_r02.zip',
 161      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 162     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 163      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 164     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 165      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 166     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 167      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 168     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 169      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 170     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 171      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 172     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 173      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 174     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 175      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 176     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 177      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 178     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 179      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 180     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 181      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 182     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 183      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 184     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 185      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 186     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 187      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 188     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 189      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 190     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 191      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 192     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 193      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 194     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 195      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 196     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 197      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 198     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 199      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 200     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 201      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 202     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 203      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 204     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 205      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 206     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 207      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 208     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 209      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 210     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 211      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 212     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 213      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 214     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 215      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 216     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 217      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 218     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 219      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 220     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 221      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 222     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 223      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 224     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 225      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 226     # the binaries that Google uses are here:
 227     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 228     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 229      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 230     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 231      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 232     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 233      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 234     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 235      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 236     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 237      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 238     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 239      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 240     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 241      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 242     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 243      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 244     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 245      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 246     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 247      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 248     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 249      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 250     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 251      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 252     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 253      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 254     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 255      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 256     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 257      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 258     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 259      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 260     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 261      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 262     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 263      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 264     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 265      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 266     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 267      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 268     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 269      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 270     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 271      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 272     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 273      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 274     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 275      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 276     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 277      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 278     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 279      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 280     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 281      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 282     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 283      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 284     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 285      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 286     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 287      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 288     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 289      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 290     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 291      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 292     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 293      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 294     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 295      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 296     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 297      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 298     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 299      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 300     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 301      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 302     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 303      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 304     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 305      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 306     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 307      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 308 ]
 309
 310
 311 def sha256_for_file(path):
 312     with open(path, 'rb') as f:
 313         s = hashlib.sha256()
 314         while True:
 315             data = f.read(4096)
 316             if not data:
 317                 break
 318             s.update(data)
 319         return s.hexdigest()
 320
 321
 322 def destroy_current_image(v, serverdir):
 323     global config
 324
 325     logger.info('destroying buildserver vm, removing images and vagrand-configs...')
 326
 327     # cannot run vagrant without the config in the YAML file
 328     if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')):
 329         v.destroy()
 330         logger.debug('vagrant destroy completed')
 331     if logger.level <= logging.DEBUG:
 332         logger.debug('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!')
 333         subprocess.check_call(['vagrant', 'global-status', '--prune'])
 334
 335     try:
 336         shutil.rmtree(os.path.join(serverdir, '.vagrant'))
 337     except Exception as e:
 338         logger.debug("could not delete vagrant dir: %s, %s", os.path.join(serverdir, '.vagrant'), e)
 339
 340     if config['vm_provider'] == 'libvirt':
 341         import libvirt
 342         try:
 343             conn = libvirt.open('qemu:///system')
 344             try:
 345                 dom = conn.lookupByName(config['domain'])
 346                 try:
 347                     dom.destroy()
 348                 except libvirt.libvirtError as e:
 349                     logging.info("could not force libvirt domain '%s' off: %s", dom.name(), e)
 350                 # libvirt python bindings do not support all flags required
 351                 # for undefining domains correctly.
 352                 try:
 353                     logger.debug('virsh -c qemu:///system undefine builder_defaul --nvram --managed-save --remove-all-storage --snapshots-metadata')
 354                     subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', 'builder_default', '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
 355                 except subprocess.CalledProcessError as e:
 356                     logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e)
 357             except libvirt.libvirtError as e:
 358                 logging.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e)
 359         except libvirt.libvirtError as e:
 360             logging.critical('could not connect to libvirtd: %s', e)
 361             sys.exit(1)
 362
 363
 364 def kvm_package(boxfile):
 365     '''
 366     Hack to replace missing `vagrant package` for kvm, based  on the script
 367     `tools/create_box.sh from vagrant-libvirt
 368     '''
 369     import libvirt
 370     virConnect = libvirt.open('qemu:///system')
 371     storagePool = virConnect.storagePoolLookupByName('default')
 372     if storagePool:
 373         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 374         imagepath = vol.path()
 375         # TODO use a libvirt storage pool to ensure the img file is readable
 376         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 377         shutil.copy2(imagepath, 'box.img')
 378         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 379         img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True)
 380         img_info = json.loads(img_info_raw.decode('utf-8'))
 381         metadata = {"provider": "libvirt",
 382                     "format": img_info['format'],
 383                     "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3),
 384                     }
 385
 386         vagrantfile = """Vagrant.configure("2") do |config|
 387   config.ssh.username = "vagrant"
 388   config.ssh.password = "vagrant"
 389
 390   config.vm.provider :libvirt do |libvirt|
 391
 392     libvirt.driver = "kvm"
 393     libvirt.host = ""
 394     libvirt.connect_via_ssh = false
 395     libvirt.storage_pool_name = "default"
 396
 397   end
 398 end
 399 """
 400         with open('metadata.json', 'w') as fp:
 401             fp.write(json.dumps(metadata))
 402         with open('Vagrantfile', 'w') as fp:
 403             fp.write(vagrantfile)
 404         with tarfile.open(boxfile, 'w:gz') as tar:
 405             tar.add('metadata.json')
 406             tar.add('Vagrantfile')
 407             tar.add('box.img')
 408         os.remove('metadata.json')
 409         os.remove('Vagrantfile')
 410         os.remove('box.img')
 411
 412
 413 def run_via_vagrant_ssh(v, cmdlist):
 414     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 415         cmd = cmdlist
 416     else:
 417         cmd = ' '.join(cmdlist)
 418     v._run_vagrant_command(['ssh', '-c', cmd])
 419
 420
 421 def update_cache(cachedir, cachefiles):
 422     for srcurl, shasum in cachefiles:
 423         filename = os.path.basename(srcurl)
 424         local_filename = os.path.join(cachedir, filename)
 425
 426         if os.path.exists(local_filename):
 427             local_length = os.path.getsize(local_filename)
 428         else:
 429             local_length = -1
 430
 431         resume_header = {}
 432         download = True
 433
 434         try:
 435             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 436             if r.status_code == 200:
 437                 content_length = int(r.headers.get('content-length'))
 438             else:
 439                 content_length = local_length  # skip the download
 440         except requests.exceptions.RequestException as e:
 441             content_length = local_length  # skip the download
 442             logger.warn('%s', e)
 443
 444         if local_length == content_length:
 445             download = False
 446         elif local_length > content_length:
 447             logger.info('deleting corrupt file from cache: %s', local_filename)
 448             os.remove(local_filename)
 449             logger.info("Downloading %s to cache", filename)
 450         elif local_length > -1 and local_length < content_length:
 451             logger.info("Resuming download of %s", local_filename)
 452             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 453         else:
 454             logger.info("Downloading %s to cache", filename)
 455
 456         if download:
 457             r = requests.get(srcurl, headers=resume_header,
 458                              stream=True, verify=False, allow_redirects=True)
 459             content_length = int(r.headers.get('content-length'))
 460             with open(local_filename, 'ab') as f:
 461                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 462                                           expected_size=(content_length / 65536) + 1):
 463                     if chunk:  # filter out keep-alive new chunks
 464                         f.write(chunk)
 465
 466         v = sha256_for_file(local_filename)
 467         if v == shasum:
 468             logger.info("\t...shasum verified for %s", local_filename)
 469         else:
 470             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 471             os.remove(local_filename)
 472             sys.exit(1)
 473
 474
 475 def debug_log_vagrant_vm(vm_dir, vm_name):
 476     if options.verbosity >= 3:
 477         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 478         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 479         logger.debug('> vagrant status')
 480         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 481         logger.debug('> vagrant box list')
 482         subprocess.call(['vagrant', 'box', 'list'])
 483         logger.debug('> virsh -c qmeu:///system list --all')
 484         subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 485         logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name)
 486         subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name])
 487
 488
 489 def main():
 490     global cachedir, cachefiles, config, tail
 491
 492     if options.skip_cache_update:
 493         logger.info('skipping cache update and verification...')
 494     else:
 495         update_cache(cachedir, cachefiles)
 496
 497     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 498     logger.info("Setting executable bit for %s", local_qt_filename)
 499     os.chmod(local_qt_filename, 0o755)
 500
 501     # use VirtualBox software virtualization if hardware is not available,
 502     # like if this is being run in kvm or some other VM platform, like
 503     # http://jenkins.debian.net, the values are 'on' or 'off'
 504     if sys.platform.startswith('darwin'):
 505         # all < 10 year old Macs work, and OSX servers as VM host are very
 506         # rare, but this could also be auto-detected if someone codes it
 507         config['hwvirtex'] = 'on'
 508         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 509     elif os.path.exists('/proc/cpuinfo'):
 510         with open('/proc/cpuinfo') as f:
 511             contents = f.read()
 512         if 'vmx' in contents or 'svm' in contents:
 513             config['hwvirtex'] = 'on'
 514             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 515
 516     serverdir = os.path.join(os.getcwd(), 'buildserver')
 517     logfilename = os.path.join(serverdir, 'up.log')
 518     if not os.path.exists(logfilename):
 519         open(logfilename, 'a').close()  # create blank file
 520     log_cm = vagrant.make_file_cm(logfilename)
 521     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 522
 523     if options.verbosity >= 2:
 524         tail = fdroidserver.tail.Tail(logfilename)
 525         tail.start()
 526
 527     if options.clean:
 528         destroy_current_image(v, serverdir)
 529
 530     # Check against the existing Vagrantfile.yaml, and if they differ, we
 531     # need to create a new box:
 532     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 533     writevf = True
 534     if os.path.exists(vf):
 535         logger.info('Halting %s', serverdir)
 536         v.halt()
 537         with open(vf, 'r', encoding='utf-8') as f:
 538             oldconfig = yaml.load(f)
 539         if config != oldconfig:
 540             logger.info("Server configuration has changed, rebuild from scratch is required")
 541             destroy_current_image(v, serverdir)
 542         else:
 543             logger.info("Re-provisioning existing server")
 544             writevf = False
 545     else:
 546         logger.info("No existing server - building from scratch")
 547     if writevf:
 548         with open(vf, 'w', encoding='utf-8') as f:
 549             yaml.dump(config, f)
 550
 551     if config['vm_provider'] == 'libvirt':
 552         found_basebox = False
 553         needs_mutate = False
 554         for box in v.box_list():
 555             if box.name == config['basebox']:
 556                 found_basebox = True
 557                 if box.provider != 'libvirt':
 558                     needs_mutate = True
 559                 continue
 560         if not found_basebox:
 561             if isinstance(config['baseboxurl'], str):
 562                 baseboxurl = config['baseboxurl']
 563             else:
 564                 baseboxurl = config['baseboxurl'][0]
 565             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 566             v.box_add(config['basebox'], baseboxurl)
 567             needs_mutate = True
 568         if needs_mutate:
 569             logger.info('Converting %s to libvirt format', config['basebox'])
 570             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 571             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 572             v.box_remove(config['basebox'], 'virtualbox')
 573
 574     logger.info("Configuring build server VM")
 575     debug_log_vagrant_vm(serverdir, 'buildserver_default')
 576     try:
 577         #subprocess.check_call(['vagrant', 'up', '--provision'], pwd=serverdir)
 578         v.up(provision=True)
 579     except subprocess.CalledProcessError as e:
 580         debug_log_vagrant_vm(serverdir, 'buildserver_default')
 581         logging.critical('could not bring buildserver vm up. %s', e)
 582         sys.exit(1)
 583
 584     if config['copy_caches_from_host']:
 585         ssh_config = v.ssh_config()
 586         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 587         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 588         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 589         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 590
 591         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 592             fullpath = os.path.join(os.getenv('HOME'), d)
 593             if os.path.isdir(fullpath):
 594                 # TODO newer versions of vagrant provide `vagrant rsync`
 595                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 596                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 597                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 598                                  fullpath + '/',
 599                                  user + '@' + hostname + ':~/' + d + '/'])
 600
 601         # this file changes every time but should not be cached
 602         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 603         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 604
 605     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 606                          universal_newlines=True)
 607     buildserverid = p.communicate()[0].strip()
 608     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 609     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 610
 611     logger.info("Stopping build server VM")
 612     v.halt()
 613
 614     logger.info("Packaging")
 615     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 616     if os.path.exists(boxfile):
 617         os.remove(boxfile)
 618
 619     if config['vm_provider'] == 'libvirt':
 620         kvm_package(boxfile)
 621     else:
 622         v.package(output=boxfile)
 623
 624     logger.info("Adding box")
 625     v.box_add('buildserver', boxfile, force=True)
 626
 627     os.remove(boxfile)
 628
 629
 630 if __name__ == '__main__':
 631     try:
 632         main()
 633     finally:
 634         if tail is not None:
 635             tail.stop()