makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
 128      '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
 129     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 130      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 131     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 132      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 133     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 134      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 135     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 136      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 137     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 138      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 139     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 140      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 141     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 142      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 143     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 144      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 145     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 146      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 147     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 148      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 149     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 150      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 151     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 152      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 153     ('https://dl.google.com/android/repository/android-14_r04.zip',
 154      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 155     ('https://dl.google.com/android/repository/android-15_r05.zip',
 156      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 157     ('https://dl.google.com/android/repository/android-16_r05.zip',
 158      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 159     ('https://dl.google.com/android/repository/android-17_r03.zip',
 160      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 161     ('https://dl.google.com/android/repository/android-18_r03.zip',
 162      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 163     ('https://dl.google.com/android/repository/android-19_r04.zip',
 164      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 165     ('https://dl.google.com/android/repository/android-20_r02.zip',
 166      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 167     ('https://dl.google.com/android/repository/android-21_r02.zip',
 168      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 169     ('https://dl.google.com/android/repository/android-22_r02.zip',
 170      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 171     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 172      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 173     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 174      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 175     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 176      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 177     ('https://dl.google.com/android/repository/platform-26_r01.zip',
 178      '44e7eca5923320db1abb422aa7a3d0ac9ee199a379af6f5c7603e714d7717561'),
 179     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 180      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 181     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 182      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 184      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 186      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 187     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 188      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 189     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 190      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 192      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 194      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 196      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 197     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 198      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 199     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 200      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 201     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 202      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 204      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 206      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 208      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 210      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 211     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 212      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 213     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 214      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 215     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 216      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 217     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 218      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 220      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 222      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 223     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 224      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 225     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 226      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 228      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 230      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 231     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 232      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 233     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 234      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 236      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 237     ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
 238      '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
 239     ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
 240      '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
 241     # the binaries that Google uses are here:
 242     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 243     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 244      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 245     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 246      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 247     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 248      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 249     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 250      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 251     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 252      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 253     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 254      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 255     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 256      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 257     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 258      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 259     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 260      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 261     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 262      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 263     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 264      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 265     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 266      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 267     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 268      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 269     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 270      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 271     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 272      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 273     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 274      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 275     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 276      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 277     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 278      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 279     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 280      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 281     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 282      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 283     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 284      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 285     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 286      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 287     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 288      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 289     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 290      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 291     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 292      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 293     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 294      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 295     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 296      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 297     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 298      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 299     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 300      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 301     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 302      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 303     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 304      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 305     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 306      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 307     ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
 308      '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
 309     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 310      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 311     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 312      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 313     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 314      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 315     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 316      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 317     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 318      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 319     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 320      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 321     ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
 322      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
 323     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 324      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 325 ]
 326
 327
 328 def sha256_for_file(path):
 329     with open(path, 'rb') as f:
 330         s = hashlib.sha256()
 331         while True:
 332             data = f.read(4096)
 333             if not data:
 334                 break
 335             s.update(data)
 336         return s.hexdigest()
 337
 338
 339 def run_via_vagrant_ssh(v, cmdlist):
 340     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 341         cmd = cmdlist
 342     else:
 343         cmd = ' '.join(cmdlist)
 344     v._run_vagrant_command(['ssh', '-c', cmd])
 345
 346
 347 def update_cache(cachedir, cachefiles):
 348     for srcurl, shasum in cachefiles:
 349         filename = os.path.basename(srcurl)
 350         local_filename = os.path.join(cachedir, filename)
 351
 352         if os.path.exists(local_filename):
 353             local_length = os.path.getsize(local_filename)
 354         else:
 355             local_length = -1
 356
 357         resume_header = {}
 358         download = True
 359
 360         try:
 361             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 362             if r.status_code == 200:
 363                 content_length = int(r.headers.get('content-length'))
 364             else:
 365                 content_length = local_length  # skip the download
 366         except requests.exceptions.RequestException as e:
 367             content_length = local_length  # skip the download
 368             logger.warn('%s', e)
 369
 370         if local_length == content_length:
 371             download = False
 372         elif local_length > content_length:
 373             logger.info('deleting corrupt file from cache: %s', local_filename)
 374             os.remove(local_filename)
 375             logger.info("Downloading %s to cache", filename)
 376         elif local_length > -1 and local_length < content_length:
 377             logger.info("Resuming download of %s", local_filename)
 378             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 379         else:
 380             logger.info("Downloading %s to cache", filename)
 381
 382         if download:
 383             r = requests.get(srcurl, headers=resume_header,
 384                              stream=True, verify=False, allow_redirects=True)
 385             content_length = int(r.headers.get('content-length'))
 386             with open(local_filename, 'ab') as f:
 387                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 388                                           expected_size=(content_length / 65536) + 1):
 389                     if chunk:  # filter out keep-alive new chunks
 390                         f.write(chunk)
 391
 392         v = sha256_for_file(local_filename)
 393         if v == shasum:
 394             logger.info("\t...shasum verified for %s", local_filename)
 395         else:
 396             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 397             os.remove(local_filename)
 398             sys.exit(1)
 399
 400
 401 def debug_log_vagrant_vm(vm_dir, config):
 402     if options.verbosity >= 3:
 403         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 404         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 405         logger.debug('> vagrant status')
 406         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 407         logger.debug('> vagrant box list')
 408         subprocess.call(['vagrant', 'box', 'list'])
 409         if config['vm_provider'] == 'libvirt':
 410             logger.debug('> virsh -c qmeu:///system list --all')
 411             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 412             domain = 'buildserver_default'
 413             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 414             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 415
 416
 417 def main():
 418     global cachedir, cachefiles, config, tail
 419
 420     if options.skip_cache_update:
 421         logger.info('skipping cache update and verification...')
 422     else:
 423         update_cache(cachedir, cachefiles)
 424
 425     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 426     logger.info("Setting executable bit for %s", local_qt_filename)
 427     os.chmod(local_qt_filename, 0o755)
 428
 429     # use VirtualBox software virtualization if hardware is not available,
 430     # like if this is being run in kvm or some other VM platform, like
 431     # http://jenkins.debian.net, the values are 'on' or 'off'
 432     if sys.platform.startswith('darwin'):
 433         # all < 10 year old Macs work, and OSX servers as VM host are very
 434         # rare, but this could also be auto-detected if someone codes it
 435         config['hwvirtex'] = 'on'
 436         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 437     elif os.path.exists('/proc/cpuinfo'):
 438         with open('/proc/cpuinfo') as f:
 439             contents = f.read()
 440         if 'vmx' in contents or 'svm' in contents:
 441             config['hwvirtex'] = 'on'
 442             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 443
 444     serverdir = os.path.join(os.getcwd(), 'buildserver')
 445     logfilename = os.path.join(serverdir, 'up.log')
 446     if not os.path.exists(logfilename):
 447         open(logfilename, 'a').close()  # create blank file
 448     log_cm = vagrant.make_file_cm(logfilename)
 449     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 450
 451     if options.verbosity >= 2:
 452         tail = fdroidserver.tail.Tail(logfilename)
 453         tail.start()
 454
 455     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 456     if options.clean:
 457         vm.destroy()
 458
 459     # Check against the existing Vagrantfile.yaml, and if they differ, we
 460     # need to create a new box:
 461     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 462     writevf = True
 463     if os.path.exists(vf):
 464         logger.info('Halting %s', serverdir)
 465         v.halt()
 466         with open(vf, 'r', encoding='utf-8') as f:
 467             oldconfig = yaml.load(f)
 468         if config != oldconfig:
 469             logger.info("Server configuration has changed, rebuild from scratch is required")
 470             vm.destroy()
 471         else:
 472             logger.info("Re-provisioning existing server")
 473             writevf = False
 474     else:
 475         logger.info("No existing server - building from scratch")
 476     if writevf:
 477         with open(vf, 'w', encoding='utf-8') as f:
 478             yaml.dump(config, f)
 479
 480     if config['vm_provider'] == 'libvirt':
 481         found_basebox = False
 482         needs_mutate = False
 483         for box in v.box_list():
 484             if box.name == config['basebox']:
 485                 found_basebox = True
 486                 if box.provider != 'libvirt':
 487                     needs_mutate = True
 488                 continue
 489         if not found_basebox:
 490             if isinstance(config['baseboxurl'], str):
 491                 baseboxurl = config['baseboxurl']
 492             else:
 493                 baseboxurl = config['baseboxurl'][0]
 494             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 495             v.box_add(config['basebox'], baseboxurl)
 496             needs_mutate = True
 497         if needs_mutate:
 498             logger.info('Converting %s to libvirt format', config['basebox'])
 499             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 500             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 501             v.box_remove(config['basebox'], 'virtualbox')
 502
 503     logger.info("Configuring build server VM")
 504     debug_log_vagrant_vm(serverdir, config)
 505     try:
 506         v.up(provision=True)
 507     except fdroidserver.vmtools.FDroidBuildVmException as e:
 508         debug_log_vagrant_vm(serverdir, config)
 509         logger.exception('could not bring buildserver vm up. %s', e)
 510         sys.exit(1)
 511
 512     if config['copy_caches_from_host']:
 513         ssh_config = v.ssh_config()
 514         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 515         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 516         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 517         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 518
 519         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 520             fullpath = os.path.join(os.getenv('HOME'), d)
 521             if os.path.isdir(fullpath):
 522                 # TODO newer versions of vagrant provide `vagrant rsync`
 523                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 524                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 525                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 526                                  fullpath + '/',
 527                                  user + '@' + hostname + ':~/' + d + '/'])
 528
 529         # this file changes every time but should not be cached
 530         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 531         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 532
 533     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 534                          universal_newlines=True)
 535     buildserverid = p.communicate()[0].strip()
 536     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 537     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 538
 539     logger.info("Stopping build server VM")
 540     v.halt()
 541
 542     logger.info("Packaging")
 543     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 544     if os.path.exists(boxfile):
 545         os.remove(boxfile)
 546
 547     vm.package(output=boxfile)
 548
 549     logger.info("Adding box")
 550     vm.box_add('buildserver', boxfile, force=True)
 551
 552     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 553         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 554         sys.exit(1)
 555
 556     if not options.keep_box_file:
 557         logger.debug('box added to vagrant, ' +
 558                      'removing generated box file \'%s\'',
 559                      boxfile)
 560         os.remove(boxfile)
 561
 562
 563 if __name__ == '__main__':
 564     try:
 565         main()
 566     finally:
 567         if tail is not None:
 568             tail.stop()