makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 128      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 129     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 130      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 131     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 132      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 133     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 134      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 135     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 136      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 137     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 138      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 139     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 140      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 141     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 142      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 143     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 144      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 145     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 146      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 147     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 148      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 149     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 150      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 151     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 152      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 153     ('https://dl.google.com/android/repository/android-14_r04.zip',
 154      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 155     ('https://dl.google.com/android/repository/android-15_r05.zip',
 156      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 157     ('https://dl.google.com/android/repository/android-16_r05.zip',
 158      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 159     ('https://dl.google.com/android/repository/android-17_r03.zip',
 160      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 161     ('https://dl.google.com/android/repository/android-18_r03.zip',
 162      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 163     ('https://dl.google.com/android/repository/android-19_r04.zip',
 164      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 165     ('https://dl.google.com/android/repository/android-20_r02.zip',
 166      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 167     ('https://dl.google.com/android/repository/android-21_r02.zip',
 168      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 169     ('https://dl.google.com/android/repository/android-22_r02.zip',
 170      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 171     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 172      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 173     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 174      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 175     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 176      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 177     ('https://dl.google.com/android/repository/platform-26_r01.zip',
 178      '44e7eca5923320db1abb422aa7a3d0ac9ee199a379af6f5c7603e714d7717561'),
 179     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 180      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 181     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 182      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 184      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 186      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 187     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 188      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 189     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 190      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 192      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 194      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 196      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 197     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 198      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 199     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 200      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 201     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 202      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 204      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 206      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 208      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 210      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 211     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 212      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 213     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 214      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 215     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 216      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 217     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 218      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 220      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 222      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 223     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 224      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 225     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 226      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 228      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 230      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 231     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 232      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 233     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 234      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 236      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 238      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 239     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 240      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 241     # the binaries that Google uses are here:
 242     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 243     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 244      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 245     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 246      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 247     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 248      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 249     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 250      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 251     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 252      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 253     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 254      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 255     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 256      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 257     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 258      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 259     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 260      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 261     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 262      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 263     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 264      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 265     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 266      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 267     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 268      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 269     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 270      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 271     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 272      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 273     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 274      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 275     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 276      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 277     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 278      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 279     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 280      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 281     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 282      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 283     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 284      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 285     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 286      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 287     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 288      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 289     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 290      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 291     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 292      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 293     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 294      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 295     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 296      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 297     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 298      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 299     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 300      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 301     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 302      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 303     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 304      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 305     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 306      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 307     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 308      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 309     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 310      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 311     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 312      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 313     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 314      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 315     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 316      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 317     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 318      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 319     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 320      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 321     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 322      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 323     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
 324      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
 325     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 326      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 327 ]
 328
 329
 330 def sha256_for_file(path):
 331     with open(path, 'rb') as f:
 332         s = hashlib.sha256()
 333         while True:
 334             data = f.read(4096)
 335             if not data:
 336                 break
 337             s.update(data)
 338         return s.hexdigest()
 339
 340
 341 def run_via_vagrant_ssh(v, cmdlist):
 342     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 343         cmd = cmdlist
 344     else:
 345         cmd = ' '.join(cmdlist)
 346     v._run_vagrant_command(['ssh', '-c', cmd])
 347
 348
 349 def update_cache(cachedir, cachefiles):
 350     for srcurl, shasum in cachefiles:
 351         filename = os.path.basename(srcurl)
 352         local_filename = os.path.join(cachedir, filename)
 353
 354         if os.path.exists(local_filename):
 355             local_length = os.path.getsize(local_filename)
 356         else:
 357             local_length = -1
 358
 359         resume_header = {}
 360         download = True
 361
 362         try:
 363             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 364             if r.status_code == 200:
 365                 content_length = int(r.headers.get('content-length'))
 366             else:
 367                 content_length = local_length  # skip the download
 368         except requests.exceptions.RequestException as e:
 369             content_length = local_length  # skip the download
 370             logger.warn('%s', e)
 371
 372         if local_length == content_length:
 373             download = False
 374         elif local_length > content_length:
 375             logger.info('deleting corrupt file from cache: %s', local_filename)
 376             os.remove(local_filename)
 377             logger.info("Downloading %s to cache", filename)
 378         elif local_length > -1 and local_length < content_length:
 379             logger.info("Resuming download of %s", local_filename)
 380             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 381         else:
 382             logger.info("Downloading %s to cache", filename)
 383
 384         if download:
 385             r = requests.get(srcurl, headers=resume_header,
 386                              stream=True, verify=False, allow_redirects=True)
 387             content_length = int(r.headers.get('content-length'))
 388             with open(local_filename, 'ab') as f:
 389                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 390                                           expected_size=(content_length / 65536) + 1):
 391                     if chunk:  # filter out keep-alive new chunks
 392                         f.write(chunk)
 393
 394         v = sha256_for_file(local_filename)
 395         if v == shasum:
 396             logger.info("\t...shasum verified for %s", local_filename)
 397         else:
 398             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 399             os.remove(local_filename)
 400             sys.exit(1)
 401
 402
 403 def debug_log_vagrant_vm(vm_dir, config):
 404     if options.verbosity >= 3:
 405         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 406         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 407         logger.debug('> vagrant status')
 408         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 409         logger.debug('> vagrant box list')
 410         subprocess.call(['vagrant', 'box', 'list'])
 411         if config['vm_provider'] == 'libvirt':
 412             logger.debug('> virsh -c qmeu:///system list --all')
 413             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 414             domain = 'buildserver_default'
 415             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 416             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 417
 418
 419 def main():
 420     global cachedir, cachefiles, config, tail
 421
 422     if options.skip_cache_update:
 423         logger.info('skipping cache update and verification...')
 424     else:
 425         update_cache(cachedir, cachefiles)
 426
 427     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 428     logger.info("Setting executable bit for %s", local_qt_filename)
 429     os.chmod(local_qt_filename, 0o755)
 430
 431     # use VirtualBox software virtualization if hardware is not available,
 432     # like if this is being run in kvm or some other VM platform, like
 433     # http://jenkins.debian.net, the values are 'on' or 'off'
 434     if sys.platform.startswith('darwin'):
 435         # all < 10 year old Macs work, and OSX servers as VM host are very
 436         # rare, but this could also be auto-detected if someone codes it
 437         config['hwvirtex'] = 'on'
 438         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 439     elif os.path.exists('/proc/cpuinfo'):
 440         with open('/proc/cpuinfo') as f:
 441             contents = f.read()
 442         if 'vmx' in contents or 'svm' in contents:
 443             config['hwvirtex'] = 'on'
 444             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 445
 446     serverdir = os.path.join(os.getcwd(), 'buildserver')
 447     logfilename = os.path.join(serverdir, 'up.log')
 448     if not os.path.exists(logfilename):
 449         open(logfilename, 'a').close()  # create blank file
 450     log_cm = vagrant.make_file_cm(logfilename)
 451     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 452
 453     if options.verbosity >= 2:
 454         tail = fdroidserver.tail.Tail(logfilename)
 455         tail.start()
 456
 457     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 458     if options.clean:
 459         vm.destroy()
 460
 461     # Check against the existing Vagrantfile.yaml, and if they differ, we
 462     # need to create a new box:
 463     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 464     writevf = True
 465     if os.path.exists(vf):
 466         logger.info('Halting %s', serverdir)
 467         v.halt()
 468         with open(vf, 'r', encoding='utf-8') as f:
 469             oldconfig = yaml.load(f)
 470         if config != oldconfig:
 471             logger.info("Server configuration has changed, rebuild from scratch is required")
 472             vm.destroy()
 473         else:
 474             logger.info("Re-provisioning existing server")
 475             writevf = False
 476     else:
 477         logger.info("No existing server - building from scratch")
 478     if writevf:
 479         with open(vf, 'w', encoding='utf-8') as f:
 480             yaml.dump(config, f)
 481
 482     if config['vm_provider'] == 'libvirt':
 483         found_basebox = False
 484         needs_mutate = False
 485         for box in v.box_list():
 486             if box.name == config['basebox']:
 487                 found_basebox = True
 488                 if box.provider != 'libvirt':
 489                     needs_mutate = True
 490                 continue
 491         if not found_basebox:
 492             if isinstance(config['baseboxurl'], str):
 493                 baseboxurl = config['baseboxurl']
 494             else:
 495                 baseboxurl = config['baseboxurl'][0]
 496             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 497             v.box_add(config['basebox'], baseboxurl)
 498             needs_mutate = True
 499         if needs_mutate:
 500             logger.info('Converting %s to libvirt format', config['basebox'])
 501             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 502             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 503             v.box_remove(config['basebox'], 'virtualbox')
 504
 505     logger.info("Configuring build server VM")
 506     debug_log_vagrant_vm(serverdir, config)
 507     try:
 508         v.up(provision=True)
 509     except fdroidserver.vmtools.FDroidBuildVmException as e:
 510         debug_log_vagrant_vm(serverdir, config)
 511         logger.exception('could not bring buildserver vm up. %s', e)
 512         sys.exit(1)
 513
 514     if config['copy_caches_from_host']:
 515         ssh_config = v.ssh_config()
 516         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 517         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 518         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 519         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 520
 521         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 522             fullpath = os.path.join(os.getenv('HOME'), d)
 523             if os.path.isdir(fullpath):
 524                 # TODO newer versions of vagrant provide `vagrant rsync`
 525                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 526                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 527                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 528                                  fullpath + '/',
 529                                  user + '@' + hostname + ':~/' + d + '/'])
 530
 531         # this file changes every time but should not be cached
 532         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 533         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 534
 535     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 536                          universal_newlines=True)
 537     buildserverid = p.communicate()[0].strip()
 538     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 539     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 540
 541     logger.info("Stopping build server VM")
 542     v.halt()
 543
 544     logger.info("Packaging")
 545     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 546     if os.path.exists(boxfile):
 547         os.remove(boxfile)
 548
 549     vm.package(output=boxfile)
 550
 551     logger.info("Adding box")
 552     vm.box_add('buildserver', boxfile, force=True)
 553
 554     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 555         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 556         sys.exit(1)
 557
 558     if not options.keep_box_file:
 559         logger.debug('box added to vagrant, ' +
 560                      'removing generated box file \'%s\'',
 561                      boxfile)
 562         os.remove(boxfile)
 563
 564
 565 if __name__ == '__main__':
 566     try:
 567         main()
 568     finally:
 569         if tail is not None:
 570             tail.stop()