makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import subprocess
  10 import vagrant
  11 import hashlib
  12 import yaml
  13 import json
  14 import logging
  15 import textwrap
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage.""")
  33 options, args = parser.parse_args()
  34
  35 logger = logging.getLogger('fdroidserver-makebuildserver')
  36 if options.verbosity >= 2:
  37     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  38     logger.setLevel(logging.DEBUG)
  39 elif options.verbosity == 1:
  40     logging.basicConfig(format='%(message)s', level=logging.INFO)
  41     logger.setLevel(logging.INFO)
  42 elif options.verbosity <= 0:
  43     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  44     logger.setLevel(logging.WARNING)
  45
  46
  47 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  48     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  49     sys.exit(1)
  50
  51 tail = None
  52
  53 # set up default config
  54 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  55 logger.debug('cachedir set to: %s', cachedir)
  56
  57 config = {
  58     'basebox': 'jessie64',
  59     'baseboxurl': [
  60         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  61         'https://f-droid.org/jessie64.box',
  62     ],
  63     'debian_mirror': 'http://http.debian.net/debian/',
  64     'apt_package_cache': False,
  65     'copy_caches_from_host': False,
  66     'boot_timeout': 600,
  67     'cachedir': cachedir,
  68     'cpus': 1,
  69     'memory': 1024,
  70     'hwvirtex': 'off',
  71     'vm_provider': 'virtualbox',
  72 }
  73
  74 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  75     try:
  76         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  77     except subprocess.CalledProcessError as e:
  78         virt = 'none'
  79     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  80         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  81         config['vm_provider'] = 'libvirt'
  82         config['domain'] = 'buildserver_default'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85 logger.debug('deceted virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117
 118 cachefiles = [
 119     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 120      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 121     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 122      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 123     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 124      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 125     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 126      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 127     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 128      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 129     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 130      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 131     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 132      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 133     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 134      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 135     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 136      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 137     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 138      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 139     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 140      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 141     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 142      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 143     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 144      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 145     ('https://dl.google.com/android/repository/android-14_r04.zip',
 146      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 147     ('https://dl.google.com/android/repository/android-15_r05.zip',
 148      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 149     ('https://dl.google.com/android/repository/android-16_r05.zip',
 150      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 151     ('https://dl.google.com/android/repository/android-17_r03.zip',
 152      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 153     ('https://dl.google.com/android/repository/android-18_r03.zip',
 154      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 155     ('https://dl.google.com/android/repository/android-19_r04.zip',
 156      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 157     ('https://dl.google.com/android/repository/android-20_r02.zip',
 158      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 159     ('https://dl.google.com/android/repository/android-21_r02.zip',
 160      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 161     ('https://dl.google.com/android/repository/android-22_r02.zip',
 162      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 163     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 164      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 165     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 166      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 167     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 168      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 169     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 170      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 171     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 172      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 173     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 174      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 175     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 176      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 177     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 178      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 179     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 180      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 181     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 182      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 183     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 184      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 185     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 186      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 187     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 188      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 189     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 190      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 191     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 192      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 193     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 194      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 195     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 196      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 197     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 198      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 199     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 200      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 201     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 202      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 203     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 204      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 205     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 206      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 207     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 208      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 209     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 210      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 211     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 212      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 213     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 214      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 215     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 216      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 217     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 218      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 219     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 220      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 221     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 222      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 223     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 224      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 225     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 226      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 227     # the binaries that Google uses are here:
 228     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 229     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 230      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 231     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 232      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 233     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 234      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 235     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 236      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 237     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 238      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 239     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 240      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 241     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 242      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 243     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 244      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 245     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 246      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 247     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 248      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 249     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 250      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 251     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 252      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 253     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 254      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 255     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 256      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 257     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 258      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 259     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 260      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 261     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 262      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 263     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 264      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 265     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 266      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 267     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 268      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 269     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 270      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 271     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 272      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 273     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 274      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 275     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 276      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 277     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 278      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 279     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 280      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 281     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 282      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 283     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 284      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 285     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 286      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 287     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 288      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 289     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 290      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 291     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 292      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 293     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 294      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 295     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 296      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 297     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 298      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 299     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 300      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 301     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 302      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 303     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 304      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 305     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 306      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 307     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 308      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 309 ]
 310
 311
 312 def sha256_for_file(path):
 313     with open(path, 'rb') as f:
 314         s = hashlib.sha256()
 315         while True:
 316             data = f.read(4096)
 317             if not data:
 318                 break
 319             s.update(data)
 320         return s.hexdigest()
 321
 322
 323 def run_via_vagrant_ssh(v, cmdlist):
 324     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 325         cmd = cmdlist
 326     else:
 327         cmd = ' '.join(cmdlist)
 328     v._run_vagrant_command(['ssh', '-c', cmd])
 329
 330
 331 def update_cache(cachedir, cachefiles):
 332     for srcurl, shasum in cachefiles:
 333         filename = os.path.basename(srcurl)
 334         local_filename = os.path.join(cachedir, filename)
 335
 336         if os.path.exists(local_filename):
 337             local_length = os.path.getsize(local_filename)
 338         else:
 339             local_length = -1
 340
 341         resume_header = {}
 342         download = True
 343
 344         try:
 345             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 346             if r.status_code == 200:
 347                 content_length = int(r.headers.get('content-length'))
 348             else:
 349                 content_length = local_length  # skip the download
 350         except requests.exceptions.RequestException as e:
 351             content_length = local_length  # skip the download
 352             logger.warn('%s', e)
 353
 354         if local_length == content_length:
 355             download = False
 356         elif local_length > content_length:
 357             logger.info('deleting corrupt file from cache: %s', local_filename)
 358             os.remove(local_filename)
 359             logger.info("Downloading %s to cache", filename)
 360         elif local_length > -1 and local_length < content_length:
 361             logger.info("Resuming download of %s", local_filename)
 362             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 363         else:
 364             logger.info("Downloading %s to cache", filename)
 365
 366         if download:
 367             r = requests.get(srcurl, headers=resume_header,
 368                              stream=True, verify=False, allow_redirects=True)
 369             content_length = int(r.headers.get('content-length'))
 370             with open(local_filename, 'ab') as f:
 371                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 372                                           expected_size=(content_length / 65536) + 1):
 373                     if chunk:  # filter out keep-alive new chunks
 374                         f.write(chunk)
 375
 376         v = sha256_for_file(local_filename)
 377         if v == shasum:
 378             logger.info("\t...shasum verified for %s", local_filename)
 379         else:
 380             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 381             os.remove(local_filename)
 382             sys.exit(1)
 383
 384
 385 def debug_log_vagrant_vm(vm_dir, vm_name):
 386     if options.verbosity >= 3:
 387         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 388         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 389         logger.debug('> vagrant status')
 390         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 391         logger.debug('> vagrant box list')
 392         subprocess.call(['vagrant', 'box', 'list'])
 393         logger.debug('> virsh -c qmeu:///system list --all')
 394         subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 395         logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name)
 396         subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name])
 397
 398
 399 def main():
 400     global cachedir, cachefiles, config, tail
 401
 402     if options.skip_cache_update:
 403         logger.info('skipping cache update and verification...')
 404     else:
 405         update_cache(cachedir, cachefiles)
 406
 407     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 408     logger.info("Setting executable bit for %s", local_qt_filename)
 409     os.chmod(local_qt_filename, 0o755)
 410
 411     # use VirtualBox software virtualization if hardware is not available,
 412     # like if this is being run in kvm or some other VM platform, like
 413     # http://jenkins.debian.net, the values are 'on' or 'off'
 414     if sys.platform.startswith('darwin'):
 415         # all < 10 year old Macs work, and OSX servers as VM host are very
 416         # rare, but this could also be auto-detected if someone codes it
 417         config['hwvirtex'] = 'on'
 418         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 419     elif os.path.exists('/proc/cpuinfo'):
 420         with open('/proc/cpuinfo') as f:
 421             contents = f.read()
 422         if 'vmx' in contents or 'svm' in contents:
 423             config['hwvirtex'] = 'on'
 424             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 425
 426     serverdir = os.path.join(os.getcwd(), 'buildserver')
 427     logfilename = os.path.join(serverdir, 'up.log')
 428     if not os.path.exists(logfilename):
 429         open(logfilename, 'a').close()  # create blank file
 430     log_cm = vagrant.make_file_cm(logfilename)
 431     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 432
 433     if options.verbosity >= 2:
 434         tail = fdroidserver.tail.Tail(logfilename)
 435         tail.start()
 436
 437     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 438     if options.clean:
 439         vm.destroy()
 440
 441     # Check against the existing Vagrantfile.yaml, and if they differ, we
 442     # need to create a new box:
 443     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 444     writevf = True
 445     if os.path.exists(vf):
 446         logger.info('Halting %s', serverdir)
 447         v.halt()
 448         with open(vf, 'r', encoding='utf-8') as f:
 449             oldconfig = yaml.load(f)
 450         if config != oldconfig:
 451             logger.info("Server configuration has changed, rebuild from scratch is required")
 452             vm.destroy()
 453         else:
 454             logger.info("Re-provisioning existing server")
 455             writevf = False
 456     else:
 457         logger.info("No existing server - building from scratch")
 458     if writevf:
 459         with open(vf, 'w', encoding='utf-8') as f:
 460             yaml.dump(config, f)
 461
 462     if config['vm_provider'] == 'libvirt':
 463         found_basebox = False
 464         needs_mutate = False
 465         for box in v.box_list():
 466             if box.name == config['basebox']:
 467                 found_basebox = True
 468                 if box.provider != 'libvirt':
 469                     needs_mutate = True
 470                 continue
 471         if not found_basebox:
 472             if isinstance(config['baseboxurl'], str):
 473                 baseboxurl = config['baseboxurl']
 474             else:
 475                 baseboxurl = config['baseboxurl'][0]
 476             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 477             v.box_add(config['basebox'], baseboxurl)
 478             needs_mutate = True
 479         if needs_mutate:
 480             logger.info('Converting %s to libvirt format', config['basebox'])
 481             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 482             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 483             v.box_remove(config['basebox'], 'virtualbox')
 484
 485     logger.info("Configuring build server VM")
 486     debug_log_vagrant_vm(serverdir, config['domain'])
 487     try:
 488         v.up(provision=True)
 489     except subprocess.CalledProcessError as e:
 490         debug_log_vagrant_vm(serverdir, config['domain'])
 491         logger.critical('could not bring buildserver vm up. %s', e)
 492         sys.exit(1)
 493
 494     if config['copy_caches_from_host']:
 495         ssh_config = v.ssh_config()
 496         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 497         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 498         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 499         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 500
 501         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 502             fullpath = os.path.join(os.getenv('HOME'), d)
 503             if os.path.isdir(fullpath):
 504                 # TODO newer versions of vagrant provide `vagrant rsync`
 505                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 506                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 507                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 508                                  fullpath + '/',
 509                                  user + '@' + hostname + ':~/' + d + '/'])
 510
 511         # this file changes every time but should not be cached
 512         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 513         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 514
 515     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 516                          universal_newlines=True)
 517     buildserverid = p.communicate()[0].strip()
 518     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 519     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 520
 521     logger.info("Stopping build server VM")
 522     v.halt()
 523
 524     logger.info("Packaging")
 525     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 526     if os.path.exists(boxfile):
 527         os.remove(boxfile)
 528
 529     vagrantfile = textwrap.dedent("""\
 530                   Vagrant.configure("2") do |config|
 531                     config.ssh.username = "vagrant"
 532                     config.ssh.password = "vagrant"
 533
 534                     config.vm.provider :libvirt do |libvirt|
 535
 536                       libvirt.driver = "kvm"
 537                       libvirt.host = ""
 538                       libvirt.connect_via_ssh = false
 539                       libvirt.storage_pool_name = "default"
 540
 541                     end
 542                   end""")
 543
 544     vm.package(output=boxfile, vagrantfile=vagrantfile, keep_box_file=options.keep_box_file)
 545
 546     logger.info("Adding box")
 547     vm.box_add('buildserver', boxfile, force=True)
 548
 549     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 550         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 551         sys.exit(1)
 552
 553     if not options.keep_box_file:
 554         logger.debug('box added to vagrant, ' +
 555                      'removing generated box file \'%s\'',
 556                      boxfile)
 557         os.remove(boxfile)
 558
 559
 560 if __name__ == '__main__':
 561     try:
 562         main()
 563     finally:
 564         if tail is not None:
 565             tail.stop()