makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 128      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 129     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 130      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 131     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 132      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 133     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 134      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 135     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 136      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 137     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 138      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 139     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 140      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 141     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 142      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 143     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 144      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 145     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 146      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 147     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 148      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 149     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 150      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 151     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 152      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 153     ('https://dl.google.com/android/repository/android-14_r04.zip',
 154      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 155     ('https://dl.google.com/android/repository/android-15_r05.zip',
 156      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 157     ('https://dl.google.com/android/repository/android-16_r05.zip',
 158      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 159     ('https://dl.google.com/android/repository/android-17_r03.zip',
 160      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 161     ('https://dl.google.com/android/repository/android-18_r03.zip',
 162      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 163     ('https://dl.google.com/android/repository/android-19_r04.zip',
 164      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 165     ('https://dl.google.com/android/repository/android-20_r02.zip',
 166      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 167     ('https://dl.google.com/android/repository/android-21_r02.zip',
 168      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 169     ('https://dl.google.com/android/repository/android-22_r02.zip',
 170      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 171     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 172      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 173     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 174      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 175     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 176      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 177     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 178      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 179     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 180      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 181     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 182      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 184      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 185     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 186      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 187     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 188      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 189     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 190      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 192      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 194      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 195     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 196      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 197     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 198      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 199     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 200      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 201     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 202      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 204      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 206      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 208      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 209     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 210      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 211     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 212      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 213     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 214      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 215     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 216      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 217     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 218      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 220      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 221     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 222      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 223     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 224      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 225     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 226      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 228      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 229     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 230      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 231     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 232      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 233     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 234      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 236      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 237     # the binaries that Google uses are here:
 238     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 239     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 240      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 241     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 242      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 243     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 244      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 245     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 246      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 247     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 248      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 249     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 250      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 251     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 252      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 253     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 254      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 255     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 256      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 257     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 258      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 259     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 260      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 261     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 262      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 263     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 264      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 265     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 266      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 267     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 268      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 269     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 270      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 271     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 272      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 273     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 274      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 275     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 276      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 277     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 278      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 279     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 280      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 281     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 282      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 283     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 284      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 285     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 286      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 287     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 288      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 289     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 290      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 291     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 292      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 293     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 294      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 295     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 296      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 297     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 298      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 299     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 300      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 301     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 302      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 303     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 304      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 305     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 306      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 307     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 308      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 309     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 310      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 311     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 312      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 313     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 314      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 315     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 316      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 317     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 318      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 319 ]
 320
 321
 322 def sha256_for_file(path):
 323     with open(path, 'rb') as f:
 324         s = hashlib.sha256()
 325         while True:
 326             data = f.read(4096)
 327             if not data:
 328                 break
 329             s.update(data)
 330         return s.hexdigest()
 331
 332
 333 def run_via_vagrant_ssh(v, cmdlist):
 334     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 335         cmd = cmdlist
 336     else:
 337         cmd = ' '.join(cmdlist)
 338     v._run_vagrant_command(['ssh', '-c', cmd])
 339
 340
 341 def update_cache(cachedir, cachefiles):
 342     for srcurl, shasum in cachefiles:
 343         filename = os.path.basename(srcurl)
 344         local_filename = os.path.join(cachedir, filename)
 345
 346         if os.path.exists(local_filename):
 347             local_length = os.path.getsize(local_filename)
 348         else:
 349             local_length = -1
 350
 351         resume_header = {}
 352         download = True
 353
 354         try:
 355             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 356             if r.status_code == 200:
 357                 content_length = int(r.headers.get('content-length'))
 358             else:
 359                 content_length = local_length  # skip the download
 360         except requests.exceptions.RequestException as e:
 361             content_length = local_length  # skip the download
 362             logger.warn('%s', e)
 363
 364         if local_length == content_length:
 365             download = False
 366         elif local_length > content_length:
 367             logger.info('deleting corrupt file from cache: %s', local_filename)
 368             os.remove(local_filename)
 369             logger.info("Downloading %s to cache", filename)
 370         elif local_length > -1 and local_length < content_length:
 371             logger.info("Resuming download of %s", local_filename)
 372             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 373         else:
 374             logger.info("Downloading %s to cache", filename)
 375
 376         if download:
 377             r = requests.get(srcurl, headers=resume_header,
 378                              stream=True, verify=False, allow_redirects=True)
 379             content_length = int(r.headers.get('content-length'))
 380             with open(local_filename, 'ab') as f:
 381                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 382                                           expected_size=(content_length / 65536) + 1):
 383                     if chunk:  # filter out keep-alive new chunks
 384                         f.write(chunk)
 385
 386         v = sha256_for_file(local_filename)
 387         if v == shasum:
 388             logger.info("\t...shasum verified for %s", local_filename)
 389         else:
 390             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 391             os.remove(local_filename)
 392             sys.exit(1)
 393
 394
 395 def debug_log_vagrant_vm(vm_dir, config):
 396     if options.verbosity >= 3:
 397         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 398         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 399         logger.debug('> vagrant status')
 400         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 401         logger.debug('> vagrant box list')
 402         subprocess.call(['vagrant', 'box', 'list'])
 403         if config['vm_provider'] == 'libvirt':
 404             logger.debug('> virsh -c qmeu:///system list --all')
 405             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 406             domain = 'buildserver_default'
 407             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 408             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 409
 410
 411 def main():
 412     global cachedir, cachefiles, config, tail
 413
 414     if options.skip_cache_update:
 415         logger.info('skipping cache update and verification...')
 416     else:
 417         update_cache(cachedir, cachefiles)
 418
 419     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 420     logger.info("Setting executable bit for %s", local_qt_filename)
 421     os.chmod(local_qt_filename, 0o755)
 422
 423     # use VirtualBox software virtualization if hardware is not available,
 424     # like if this is being run in kvm or some other VM platform, like
 425     # http://jenkins.debian.net, the values are 'on' or 'off'
 426     if sys.platform.startswith('darwin'):
 427         # all < 10 year old Macs work, and OSX servers as VM host are very
 428         # rare, but this could also be auto-detected if someone codes it
 429         config['hwvirtex'] = 'on'
 430         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 431     elif os.path.exists('/proc/cpuinfo'):
 432         with open('/proc/cpuinfo') as f:
 433             contents = f.read()
 434         if 'vmx' in contents or 'svm' in contents:
 435             config['hwvirtex'] = 'on'
 436             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 437
 438     serverdir = os.path.join(os.getcwd(), 'buildserver')
 439     logfilename = os.path.join(serverdir, 'up.log')
 440     if not os.path.exists(logfilename):
 441         open(logfilename, 'a').close()  # create blank file
 442     log_cm = vagrant.make_file_cm(logfilename)
 443     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 444
 445     if options.verbosity >= 2:
 446         tail = fdroidserver.tail.Tail(logfilename)
 447         tail.start()
 448
 449     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 450     if options.clean:
 451         vm.destroy()
 452
 453     # Check against the existing Vagrantfile.yaml, and if they differ, we
 454     # need to create a new box:
 455     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 456     writevf = True
 457     if os.path.exists(vf):
 458         logger.info('Halting %s', serverdir)
 459         v.halt()
 460         with open(vf, 'r', encoding='utf-8') as f:
 461             oldconfig = yaml.load(f)
 462         if config != oldconfig:
 463             logger.info("Server configuration has changed, rebuild from scratch is required")
 464             vm.destroy()
 465         else:
 466             logger.info("Re-provisioning existing server")
 467             writevf = False
 468     else:
 469         logger.info("No existing server - building from scratch")
 470     if writevf:
 471         with open(vf, 'w', encoding='utf-8') as f:
 472             yaml.dump(config, f)
 473
 474     if config['vm_provider'] == 'libvirt':
 475         found_basebox = False
 476         needs_mutate = False
 477         for box in v.box_list():
 478             if box.name == config['basebox']:
 479                 found_basebox = True
 480                 if box.provider != 'libvirt':
 481                     needs_mutate = True
 482                 continue
 483         if not found_basebox:
 484             if isinstance(config['baseboxurl'], str):
 485                 baseboxurl = config['baseboxurl']
 486             else:
 487                 baseboxurl = config['baseboxurl'][0]
 488             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 489             v.box_add(config['basebox'], baseboxurl)
 490             needs_mutate = True
 491         if needs_mutate:
 492             logger.info('Converting %s to libvirt format', config['basebox'])
 493             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 494             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 495             v.box_remove(config['basebox'], 'virtualbox')
 496
 497     logger.info("Configuring build server VM")
 498     debug_log_vagrant_vm(serverdir, config)
 499     try:
 500         v.up(provision=True)
 501     except fdroidserver.vmtools.FDroidBuildVmException as e:
 502         debug_log_vagrant_vm(serverdir, config)
 503         logger.exception('could not bring buildserver vm up. %s', e)
 504         sys.exit(1)
 505
 506     if config['copy_caches_from_host']:
 507         ssh_config = v.ssh_config()
 508         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 509         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 510         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 511         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 512
 513         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 514             fullpath = os.path.join(os.getenv('HOME'), d)
 515             if os.path.isdir(fullpath):
 516                 # TODO newer versions of vagrant provide `vagrant rsync`
 517                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 518                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 519                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 520                                  fullpath + '/',
 521                                  user + '@' + hostname + ':~/' + d + '/'])
 522
 523         # this file changes every time but should not be cached
 524         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 525         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 526
 527     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 528                          universal_newlines=True)
 529     buildserverid = p.communicate()[0].strip()
 530     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 531     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 532
 533     logger.info("Stopping build server VM")
 534     v.halt()
 535
 536     logger.info("Packaging")
 537     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 538     if os.path.exists(boxfile):
 539         os.remove(boxfile)
 540
 541     vm.package(output=boxfile)
 542
 543     logger.info("Adding box")
 544     vm.box_add('buildserver', boxfile, force=True)
 545
 546     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 547         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 548         sys.exit(1)
 549
 550     if not options.keep_box_file:
 551         logger.debug('box added to vagrant, ' +
 552                      'removing generated box file \'%s\'',
 553                      boxfile)
 554         os.remove(boxfile)
 555
 556
 557 if __name__ == '__main__':
 558     try:
 559         main()
 560     finally:
 561         if tail is not None:
 562             tail.stop()