makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 import math
  16 import json
  17 import time
  18 import logging
  19 from clint.textui import progress
  20 from optparse import OptionParser
  21 import fdroidserver.tail
  22 import fdroidserver.vmtools
  23
  24
  25 parser = OptionParser()
  26 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  27                   help="Spew out even more information than normal")
  28 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  29 parser.add_option("-c", "--clean", action="store_true", default=False,
  30                   help="Build from scratch, rather than attempting to update the existing server")
  31 parser.add_option('--skip-cache-update', action="store_true", default=False,
  32                   help="""Skip downloading and checking cache."""
  33                        """This assumes that the cache is already downloaded completely.""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83         config['domain'] = 'buildserver_default'
  84     elif virt != 'none':
  85         logger.info('Running in an unsupported VM guest (%s)!', virt)
  86 logger.debug('deceted virt: %s', virt)
  87
  88 # load config file, if present
  89 if os.path.exists('makebuildserver.config.py'):
  90     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  91 elif os.path.exists('makebs.config.py'):
  92     # this is the old name for the config file
  93     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  94 if '__builtins__' in config:
  95     del(config['__builtins__'])  # added by compile/exec
  96 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  97
  98 # Update cached files.
  99 cachedir = config['cachedir']
 100 if not os.path.exists(cachedir):
 101     os.makedirs(cachedir, 0o755)
 102     logger.debug('created cachedir %s because it did not exists.', cachedir)
 103
 104 if config['vm_provider'] == 'libvirt':
 105     tmp = cachedir
 106     while tmp != '/':
 107         mode = os.stat(tmp).st_mode
 108         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 109             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 110             logger.critical('  chmod a+X %s', tmp)
 111             sys.exit(1)
 112         tmp = os.path.dirname(tmp)
 113     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 114
 115 if config['apt_package_cache']:
 116     config['aptcachedir'] = cachedir + '/apt/archives'
 117     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 118
 119 cachefiles = [
 120     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 121      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 122     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 123      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 124     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 125      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 126     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 127      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 128     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 129      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 130     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 131      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 132     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 133      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 134     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 135      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 136     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 137      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 138     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 139      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 140     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 141      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 142     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 143      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 144     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 145      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 146     ('https://dl.google.com/android/repository/android-14_r04.zip',
 147      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 148     ('https://dl.google.com/android/repository/android-15_r05.zip',
 149      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 150     ('https://dl.google.com/android/repository/android-16_r05.zip',
 151      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 152     ('https://dl.google.com/android/repository/android-17_r03.zip',
 153      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 154     ('https://dl.google.com/android/repository/android-18_r03.zip',
 155      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 156     ('https://dl.google.com/android/repository/android-19_r04.zip',
 157      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 158     ('https://dl.google.com/android/repository/android-20_r02.zip',
 159      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 160     ('https://dl.google.com/android/repository/android-21_r02.zip',
 161      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 162     ('https://dl.google.com/android/repository/android-22_r02.zip',
 163      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 164     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 165      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 166     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 167      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 168     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 169      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 170     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 171      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 172     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 173      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 174     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 175      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 176     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 177      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 178     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 179      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 180     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 181      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 182     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 183      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 184     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 185      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 186     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 187      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 188     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 189      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 190     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 191      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 192     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 193      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 194     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 195      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 196     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 197      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 198     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 199      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 200     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 201      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 202     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 203      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 204     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 205      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 206     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 207      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 208     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 209      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 210     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 211      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 212     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 213      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 214     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 215      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 216     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 217      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 218     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 219      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 220     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 221      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 222     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 223      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 224     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 225      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 226     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 227      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 228     # the binaries that Google uses are here:
 229     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 230     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 231      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 232     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 233      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 234     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 235      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 236     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 237      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 238     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 239      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 240     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 241      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 242     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 243      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 244     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 245      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 246     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 247      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 248     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 249      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 250     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 251      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 252     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 253      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 254     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 255      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 256     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 257      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 258     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 259      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 260     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 261      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 262     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 263      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 264     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 265      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 266     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 267      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 268     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 269      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 270     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 271      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 272     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 273      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 274     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 275      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 276     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 277      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 278     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 279      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 280     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 281      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 282     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 283      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 284     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 285      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 286     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 287      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 288     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 289      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 290     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 291      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 292     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 293      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 294     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 295      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 296     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 297      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 298     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 299      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 300     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 301      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 302     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 303      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 304     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 305      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 306     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 307      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 308     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 309      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 310 ]
 311
 312
 313 def sha256_for_file(path):
 314     with open(path, 'rb') as f:
 315         s = hashlib.sha256()
 316         while True:
 317             data = f.read(4096)
 318             if not data:
 319                 break
 320             s.update(data)
 321         return s.hexdigest()
 322
 323
 324 def kvm_package(boxfile):
 325     '''
 326     Hack to replace missing `vagrant package` for kvm, based  on the script
 327     `tools/create_box.sh from vagrant-libvirt
 328     '''
 329     import libvirt
 330     virConnect = libvirt.open('qemu:///system')
 331     storagePool = virConnect.storagePoolLookupByName('default')
 332     if storagePool:
 333         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 334         imagepath = vol.path()
 335         # TODO use a libvirt storage pool to ensure the img file is readable
 336         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 337         shutil.copy2(imagepath, 'box.img')
 338         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 339         img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True)
 340         img_info = json.loads(img_info_raw.decode('utf-8'))
 341         metadata = {"provider": "libvirt",
 342                     "format": img_info['format'],
 343                     "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3),
 344                     }
 345
 346         vagrantfile = """Vagrant.configure("2") do |config|
 347   config.ssh.username = "vagrant"
 348   config.ssh.password = "vagrant"
 349
 350   config.vm.provider :libvirt do |libvirt|
 351
 352     libvirt.driver = "kvm"
 353     libvirt.host = ""
 354     libvirt.connect_via_ssh = false
 355     libvirt.storage_pool_name = "default"
 356
 357   end
 358 end
 359 """
 360         with open('metadata.json', 'w') as fp:
 361             fp.write(json.dumps(metadata))
 362         with open('Vagrantfile', 'w') as fp:
 363             fp.write(vagrantfile)
 364         with tarfile.open(boxfile, 'w:gz') as tar:
 365             tar.add('metadata.json')
 366             tar.add('Vagrantfile')
 367             tar.add('box.img')
 368         os.remove('metadata.json')
 369         os.remove('Vagrantfile')
 370         os.remove('box.img')
 371
 372
 373 def run_via_vagrant_ssh(v, cmdlist):
 374     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 375         cmd = cmdlist
 376     else:
 377         cmd = ' '.join(cmdlist)
 378     v._run_vagrant_command(['ssh', '-c', cmd])
 379
 380
 381 def update_cache(cachedir, cachefiles):
 382     for srcurl, shasum in cachefiles:
 383         filename = os.path.basename(srcurl)
 384         local_filename = os.path.join(cachedir, filename)
 385
 386         if os.path.exists(local_filename):
 387             local_length = os.path.getsize(local_filename)
 388         else:
 389             local_length = -1
 390
 391         resume_header = {}
 392         download = True
 393
 394         try:
 395             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 396             if r.status_code == 200:
 397                 content_length = int(r.headers.get('content-length'))
 398             else:
 399                 content_length = local_length  # skip the download
 400         except requests.exceptions.RequestException as e:
 401             content_length = local_length  # skip the download
 402             logger.warn('%s', e)
 403
 404         if local_length == content_length:
 405             download = False
 406         elif local_length > content_length:
 407             logger.info('deleting corrupt file from cache: %s', local_filename)
 408             os.remove(local_filename)
 409             logger.info("Downloading %s to cache", filename)
 410         elif local_length > -1 and local_length < content_length:
 411             logger.info("Resuming download of %s", local_filename)
 412             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 413         else:
 414             logger.info("Downloading %s to cache", filename)
 415
 416         if download:
 417             r = requests.get(srcurl, headers=resume_header,
 418                              stream=True, verify=False, allow_redirects=True)
 419             content_length = int(r.headers.get('content-length'))
 420             with open(local_filename, 'ab') as f:
 421                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 422                                           expected_size=(content_length / 65536) + 1):
 423                     if chunk:  # filter out keep-alive new chunks
 424                         f.write(chunk)
 425
 426         v = sha256_for_file(local_filename)
 427         if v == shasum:
 428             logger.info("\t...shasum verified for %s", local_filename)
 429         else:
 430             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 431             os.remove(local_filename)
 432             sys.exit(1)
 433
 434
 435 def debug_log_vagrant_vm(vm_dir, vm_name):
 436     if options.verbosity >= 3:
 437         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 438         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 439         logger.debug('> vagrant status')
 440         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 441         logger.debug('> vagrant box list')
 442         subprocess.call(['vagrant', 'box', 'list'])
 443         logger.debug('> virsh -c qmeu:///system list --all')
 444         subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 445         logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name)
 446         subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name])
 447
 448
 449 def main():
 450     global cachedir, cachefiles, config, tail
 451
 452     if options.skip_cache_update:
 453         logger.info('skipping cache update and verification...')
 454     else:
 455         update_cache(cachedir, cachefiles)
 456
 457     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 458     logger.info("Setting executable bit for %s", local_qt_filename)
 459     os.chmod(local_qt_filename, 0o755)
 460
 461     # use VirtualBox software virtualization if hardware is not available,
 462     # like if this is being run in kvm or some other VM platform, like
 463     # http://jenkins.debian.net, the values are 'on' or 'off'
 464     if sys.platform.startswith('darwin'):
 465         # all < 10 year old Macs work, and OSX servers as VM host are very
 466         # rare, but this could also be auto-detected if someone codes it
 467         config['hwvirtex'] = 'on'
 468         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 469     elif os.path.exists('/proc/cpuinfo'):
 470         with open('/proc/cpuinfo') as f:
 471             contents = f.read()
 472         if 'vmx' in contents or 'svm' in contents:
 473             config['hwvirtex'] = 'on'
 474             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 475
 476     serverdir = os.path.join(os.getcwd(), 'buildserver')
 477     logfilename = os.path.join(serverdir, 'up.log')
 478     if not os.path.exists(logfilename):
 479         open(logfilename, 'a').close()  # create blank file
 480     log_cm = vagrant.make_file_cm(logfilename)
 481     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 482
 483     if options.verbosity >= 2:
 484         tail = fdroidserver.tail.Tail(logfilename)
 485         tail.start()
 486
 487     vm = fdroidserver.vmtools.get_build_vm(serverdir)
 488     if options.clean:
 489         vm.destroy()
 490
 491     # Check against the existing Vagrantfile.yaml, and if they differ, we
 492     # need to create a new box:
 493     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 494     writevf = True
 495     if os.path.exists(vf):
 496         logger.info('Halting %s', serverdir)
 497         v.halt()
 498         with open(vf, 'r', encoding='utf-8') as f:
 499             oldconfig = yaml.load(f)
 500         if config != oldconfig:
 501             logger.info("Server configuration has changed, rebuild from scratch is required")
 502             vm.destroy()
 503         else:
 504             logger.info("Re-provisioning existing server")
 505             writevf = False
 506     else:
 507         logger.info("No existing server - building from scratch")
 508     if writevf:
 509         with open(vf, 'w', encoding='utf-8') as f:
 510             yaml.dump(config, f)
 511
 512     if config['vm_provider'] == 'libvirt':
 513         found_basebox = False
 514         needs_mutate = False
 515         for box in v.box_list():
 516             if box.name == config['basebox']:
 517                 found_basebox = True
 518                 if box.provider != 'libvirt':
 519                     needs_mutate = True
 520                 continue
 521         if not found_basebox:
 522             if isinstance(config['baseboxurl'], str):
 523                 baseboxurl = config['baseboxurl']
 524             else:
 525                 baseboxurl = config['baseboxurl'][0]
 526             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 527             v.box_add(config['basebox'], baseboxurl)
 528             needs_mutate = True
 529         if needs_mutate:
 530             logger.info('Converting %s to libvirt format', config['basebox'])
 531             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 532             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 533             v.box_remove(config['basebox'], 'virtualbox')
 534
 535     logger.info("Configuring build server VM")
 536     debug_log_vagrant_vm(serverdir, config['domain'])
 537     try:
 538         try:
 539             v.up(provision=True)
 540         except subprocess.CalledProcessError as e:
 541             v.up(provision=True)
 542     except subprocess.CalledProcessError as e:
 543         debug_log_vagrant_vm(serverdir, config['domain'])
 544         logger.critical('could not bring buildserver vm up. %s', e)
 545         sys.exit(1)
 546
 547     if config['copy_caches_from_host']:
 548         ssh_config = v.ssh_config()
 549         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 550         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 551         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 552         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 553
 554         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 555             fullpath = os.path.join(os.getenv('HOME'), d)
 556             if os.path.isdir(fullpath):
 557                 # TODO newer versions of vagrant provide `vagrant rsync`
 558                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 559                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 560                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 561                                  fullpath + '/',
 562                                  user + '@' + hostname + ':~/' + d + '/'])
 563
 564         # this file changes every time but should not be cached
 565         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 566         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 567
 568     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 569                          universal_newlines=True)
 570     buildserverid = p.communicate()[0].strip()
 571     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 572     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 573
 574     logger.info("Stopping build server VM")
 575     v.halt()
 576
 577     logger.info("Packaging")
 578     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 579     if os.path.exists(boxfile):
 580         os.remove(boxfile)
 581
 582     if config['vm_provider'] == 'libvirt':
 583         kvm_package(boxfile)
 584     else:
 585         v.package(output=boxfile)
 586
 587     logger.info("Adding box")
 588     v.box_add('buildserver', boxfile, force=True)
 589
 590     os.remove(boxfile)
 591
 592
 593 if __name__ == '__main__':
 594     try:
 595         main()
 596     finally:
 597         if tail is not None:
 598             tail.stop()