makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import subprocess
  10 import vagrant
  11 import hashlib
  12 import yaml
  13 import json
  14 import logging
  15 from clint.textui import progress
  16 from optparse import OptionParser
  17 import fdroidserver.tail
  18 import fdroidserver.vmtools
  19
  20
  21 parser = OptionParser()
  22 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  23                   help="Spew out even more information than normal")
  24 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  25 parser.add_option("-c", "--clean", action="store_true", default=False,
  26                   help="Build from scratch, rather than attempting to update the existing server")
  27 parser.add_option('--skip-cache-update', action="store_true", default=False,
  28                   help="""Skip downloading and checking cache."""
  29                        """This assumes that the cache is already downloaded completely.""")
  30 parser.add_option('--keep-box-file', action="store_true", default=False,
  31                   help="""Box file will not be deleted after adding it to box storage.""")
  32 options, args = parser.parse_args()
  33
  34 logger = logging.getLogger('fdroidserver-makebuildserver')
  35 if options.verbosity >= 2:
  36     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  37     logger.setLevel(logging.DEBUG)
  38 elif options.verbosity == 1:
  39     logging.basicConfig(format='%(message)s', level=logging.INFO)
  40     logger.setLevel(logging.INFO)
  41 elif options.verbosity <= 0:
  42     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  43     logger.setLevel(logging.WARNING)
  44
  45
  46 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  47     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  48     sys.exit(1)
  49
  50 tail = None
  51
  52 # set up default config
  53 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  54 logger.debug('cachedir set to: %s', cachedir)
  55
  56 config = {
  57     'basebox': 'jessie64',
  58     'baseboxurl': [
  59         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  60         'https://f-droid.org/jessie64.box',
  61     ],
  62     'debian_mirror': 'http://http.debian.net/debian/',
  63     'apt_package_cache': False,
  64     'copy_caches_from_host': False,
  65     'boot_timeout': 600,
  66     'cachedir': cachedir,
  67     'cpus': 1,
  68     'memory': 1024,
  69     'hwvirtex': 'off',
  70     'vm_provider': 'virtualbox',
  71 }
  72
  73 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  74     try:
  75         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  76     except subprocess.CalledProcessError as e:
  77         virt = 'none'
  78     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  79         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  80         config['vm_provider'] = 'libvirt'
  81         config['domain'] = 'buildserver_default'
  82     elif virt != 'none':
  83         logger.info('Running in an unsupported VM guest (%s)!', virt)
  84 logger.debug('deceted virt: %s', virt)
  85
  86 # load config file, if present
  87 if os.path.exists('makebuildserver.config.py'):
  88     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  89 elif os.path.exists('makebs.config.py'):
  90     # this is the old name for the config file
  91     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  92 if '__builtins__' in config:
  93     del(config['__builtins__'])  # added by compile/exec
  94 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  95
  96 # Update cached files.
  97 cachedir = config['cachedir']
  98 if not os.path.exists(cachedir):
  99     os.makedirs(cachedir, 0o755)
 100     logger.debug('created cachedir %s because it did not exists.', cachedir)
 101
 102 if config['vm_provider'] == 'libvirt':
 103     tmp = cachedir
 104     while tmp != '/':
 105         mode = os.stat(tmp).st_mode
 106         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 107             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 108             logger.critical('  chmod a+X %s', tmp)
 109             sys.exit(1)
 110         tmp = os.path.dirname(tmp)
 111     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 112
 113 if config['apt_package_cache']:
 114     config['aptcachedir'] = cachedir + '/apt/archives'
 115     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 116
 117 cachefiles = [
 118     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 119      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 120     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 121      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 122     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 123      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 124     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 125      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 126     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 127      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 128     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 129      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 130     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 131      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 132     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 133      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 134     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 135      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 136     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 137      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 138     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 139      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 140     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 141      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 142     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 143      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 144     ('https://dl.google.com/android/repository/android-14_r04.zip',
 145      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 146     ('https://dl.google.com/android/repository/android-15_r05.zip',
 147      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 148     ('https://dl.google.com/android/repository/android-16_r05.zip',
 149      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 150     ('https://dl.google.com/android/repository/android-17_r03.zip',
 151      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 152     ('https://dl.google.com/android/repository/android-18_r03.zip',
 153      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 154     ('https://dl.google.com/android/repository/android-19_r04.zip',
 155      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 156     ('https://dl.google.com/android/repository/android-20_r02.zip',
 157      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 158     ('https://dl.google.com/android/repository/android-21_r02.zip',
 159      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 160     ('https://dl.google.com/android/repository/android-22_r02.zip',
 161      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 162     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 163      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 164     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 165      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 166     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 167      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 168     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 169      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 170     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 171      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 172     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 173      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 174     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 175      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 176     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 177      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 178     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 179      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 180     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 181      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 182     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 183      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 184     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 185      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 186     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 187      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 188     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 189      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 190     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 191      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 192     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 193      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 194     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 195      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 196     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 197      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 198     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 199      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 200     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 201      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 202     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 203      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 204     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 205      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 206     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 207      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 208     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 209      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 210     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 211      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 212     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 213      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 214     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 215      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 216     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 217      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 218     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 219      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 220     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 221      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 222     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 223      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 224     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 225      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 226     # the binaries that Google uses are here:
 227     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 228     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 229      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 230     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 231      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 232     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 233      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 234     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 235      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 236     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 237      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 238     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 239      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 240     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 241      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 242     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 243      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 244     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 245      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 246     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 247      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 248     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 249      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 250     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 251      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 252     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 253      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 254     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 255      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 256     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 257      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 258     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 259      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 260     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 261      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 262     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 263      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 264     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 265      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 266     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 267      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 268     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 269      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 270     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 271      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 272     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 273      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 274     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 275      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 276     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 277      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 278     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 279      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 280     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 281      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 282     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 283      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 284     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 285      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 286     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 287      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 288     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 289      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 290     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 291      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 292     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 293      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 294     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 295      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 296     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 297      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 298     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 299      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 300     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 301      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 302     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 303      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 304     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 305      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 306     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 307      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 308 ]
 309
 310
 311 def sha256_for_file(path):
 312     with open(path, 'rb') as f:
 313         s = hashlib.sha256()
 314         while True:
 315             data = f.read(4096)
 316             if not data:
 317                 break
 318             s.update(data)
 319         return s.hexdigest()
 320
 321
 322 def run_via_vagrant_ssh(v, cmdlist):
 323     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 324         cmd = cmdlist
 325     else:
 326         cmd = ' '.join(cmdlist)
 327     v._run_vagrant_command(['ssh', '-c', cmd])
 328
 329
 330 def update_cache(cachedir, cachefiles):
 331     for srcurl, shasum in cachefiles:
 332         filename = os.path.basename(srcurl)
 333         local_filename = os.path.join(cachedir, filename)
 334
 335         if os.path.exists(local_filename):
 336             local_length = os.path.getsize(local_filename)
 337         else:
 338             local_length = -1
 339
 340         resume_header = {}
 341         download = True
 342
 343         try:
 344             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 345             if r.status_code == 200:
 346                 content_length = int(r.headers.get('content-length'))
 347             else:
 348                 content_length = local_length  # skip the download
 349         except requests.exceptions.RequestException as e:
 350             content_length = local_length  # skip the download
 351             logger.warn('%s', e)
 352
 353         if local_length == content_length:
 354             download = False
 355         elif local_length > content_length:
 356             logger.info('deleting corrupt file from cache: %s', local_filename)
 357             os.remove(local_filename)
 358             logger.info("Downloading %s to cache", filename)
 359         elif local_length > -1 and local_length < content_length:
 360             logger.info("Resuming download of %s", local_filename)
 361             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 362         else:
 363             logger.info("Downloading %s to cache", filename)
 364
 365         if download:
 366             r = requests.get(srcurl, headers=resume_header,
 367                              stream=True, verify=False, allow_redirects=True)
 368             content_length = int(r.headers.get('content-length'))
 369             with open(local_filename, 'ab') as f:
 370                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 371                                           expected_size=(content_length / 65536) + 1):
 372                     if chunk:  # filter out keep-alive new chunks
 373                         f.write(chunk)
 374
 375         v = sha256_for_file(local_filename)
 376         if v == shasum:
 377             logger.info("\t...shasum verified for %s", local_filename)
 378         else:
 379             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 380             os.remove(local_filename)
 381             sys.exit(1)
 382
 383
 384 def debug_log_vagrant_vm(vm_dir, vm_name):
 385     if options.verbosity >= 3:
 386         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 387         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 388         logger.debug('> vagrant status')
 389         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 390         logger.debug('> vagrant box list')
 391         subprocess.call(['vagrant', 'box', 'list'])
 392         logger.debug('> virsh -c qmeu:///system list --all')
 393         subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 394         logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name)
 395         subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name])
 396
 397
 398 def main():
 399     global cachedir, cachefiles, config, tail
 400
 401     if options.skip_cache_update:
 402         logger.info('skipping cache update and verification...')
 403     else:
 404         update_cache(cachedir, cachefiles)
 405
 406     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 407     logger.info("Setting executable bit for %s", local_qt_filename)
 408     os.chmod(local_qt_filename, 0o755)
 409
 410     # use VirtualBox software virtualization if hardware is not available,
 411     # like if this is being run in kvm or some other VM platform, like
 412     # http://jenkins.debian.net, the values are 'on' or 'off'
 413     if sys.platform.startswith('darwin'):
 414         # all < 10 year old Macs work, and OSX servers as VM host are very
 415         # rare, but this could also be auto-detected if someone codes it
 416         config['hwvirtex'] = 'on'
 417         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 418     elif os.path.exists('/proc/cpuinfo'):
 419         with open('/proc/cpuinfo') as f:
 420             contents = f.read()
 421         if 'vmx' in contents or 'svm' in contents:
 422             config['hwvirtex'] = 'on'
 423             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 424
 425     serverdir = os.path.join(os.getcwd(), 'buildserver')
 426     logfilename = os.path.join(serverdir, 'up.log')
 427     if not os.path.exists(logfilename):
 428         open(logfilename, 'a').close()  # create blank file
 429     log_cm = vagrant.make_file_cm(logfilename)
 430     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 431
 432     if options.verbosity >= 2:
 433         tail = fdroidserver.tail.Tail(logfilename)
 434         tail.start()
 435
 436     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 437     if options.clean:
 438         vm.destroy()
 439
 440     # Check against the existing Vagrantfile.yaml, and if they differ, we
 441     # need to create a new box:
 442     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 443     writevf = True
 444     if os.path.exists(vf):
 445         logger.info('Halting %s', serverdir)
 446         v.halt()
 447         with open(vf, 'r', encoding='utf-8') as f:
 448             oldconfig = yaml.load(f)
 449         if config != oldconfig:
 450             logger.info("Server configuration has changed, rebuild from scratch is required")
 451             vm.destroy()
 452         else:
 453             logger.info("Re-provisioning existing server")
 454             writevf = False
 455     else:
 456         logger.info("No existing server - building from scratch")
 457     if writevf:
 458         with open(vf, 'w', encoding='utf-8') as f:
 459             yaml.dump(config, f)
 460
 461     if config['vm_provider'] == 'libvirt':
 462         found_basebox = False
 463         needs_mutate = False
 464         for box in v.box_list():
 465             if box.name == config['basebox']:
 466                 found_basebox = True
 467                 if box.provider != 'libvirt':
 468                     needs_mutate = True
 469                 continue
 470         if not found_basebox:
 471             if isinstance(config['baseboxurl'], str):
 472                 baseboxurl = config['baseboxurl']
 473             else:
 474                 baseboxurl = config['baseboxurl'][0]
 475             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 476             v.box_add(config['basebox'], baseboxurl)
 477             needs_mutate = True
 478         if needs_mutate:
 479             logger.info('Converting %s to libvirt format', config['basebox'])
 480             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 481             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 482             v.box_remove(config['basebox'], 'virtualbox')
 483
 484     logger.info("Configuring build server VM")
 485     debug_log_vagrant_vm(serverdir, config['domain'])
 486     try:
 487         v.up(provision=True)
 488     except subprocess.CalledProcessError as e:
 489         debug_log_vagrant_vm(serverdir, config['domain'])
 490         logger.critical('could not bring buildserver vm up. %s', e)
 491         sys.exit(1)
 492
 493     if config['copy_caches_from_host']:
 494         ssh_config = v.ssh_config()
 495         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 496         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 497         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 498         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 499
 500         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 501             fullpath = os.path.join(os.getenv('HOME'), d)
 502             if os.path.isdir(fullpath):
 503                 # TODO newer versions of vagrant provide `vagrant rsync`
 504                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 505                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 506                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 507                                  fullpath + '/',
 508                                  user + '@' + hostname + ':~/' + d + '/'])
 509
 510         # this file changes every time but should not be cached
 511         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 512         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 513
 514     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 515                          universal_newlines=True)
 516     buildserverid = p.communicate()[0].strip()
 517     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 518     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 519
 520     logger.info("Stopping build server VM")
 521     v.halt()
 522
 523     logger.info("Packaging")
 524     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 525     if os.path.exists(boxfile):
 526         os.remove(boxfile)
 527
 528     vm.package(output=boxfile, debug_keep_box_file=options.debug_keep_box_file)
 529
 530     logger.info("Adding box")
 531     v.box_add('buildserver', boxfile, force=True)
 532
 533     if not 'buildserver' in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 534         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 535         sys.exit(1)
 536
 537     if not options.keep_box_file:
 538         logger.debug('box added to vagrant, ' +
 539                      'removing generated box file \'%s\'',
 540                      boxfile)
 541         os.remove(boxfile)
 542
 543
 544 if __name__ == '__main__':
 545     try:
 546         main()
 547     finally:
 548         if tail is not None:
 549             tail.stop()