makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83         config['domain'] = 'buildserver_default'
  84     elif virt != 'none':
  85         logger.info('Running in an unsupported VM guest (%s)!', virt)
  86     logger.debug('detected virt: %s', virt)
  87
  88 # load config file, if present
  89 if os.path.exists('makebuildserver.config.py'):
  90     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  91 elif os.path.exists('makebs.config.py'):
  92     # this is the old name for the config file
  93     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  94 if '__builtins__' in config:
  95     del(config['__builtins__'])  # added by compile/exec
  96 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  97
  98 # Update cached files.
  99 cachedir = config['cachedir']
 100 if not os.path.exists(cachedir):
 101     os.makedirs(cachedir, 0o755)
 102     logger.debug('created cachedir %s because it did not exists.', cachedir)
 103
 104 if config['vm_provider'] == 'libvirt':
 105     tmp = cachedir
 106     while tmp != '/':
 107         mode = os.stat(tmp).st_mode
 108         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 109             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 110             logger.critical('  chmod a+X %s', tmp)
 111             sys.exit(1)
 112         tmp = os.path.dirname(tmp)
 113     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 114
 115 if config['apt_package_cache']:
 116     config['aptcachedir'] = cachedir + '/apt/archives'
 117     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 118     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 119     if os.path.isfile(aptcachelock):
 120         logger.info('apt cache dir is locked, removing lock')
 121         os.remove(aptcachelock)
 122     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 123     if os.path.isdir(aptcachepartial):
 124         logger.info('removing partial downloads from apt cache dir')
 125         shutil.rmtree(aptcachepartial)
 126
 127 cachefiles = [
 128     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 129      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 130     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 131      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 132     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 133      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 134     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 135      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 136     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 137      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 138     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 139      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 140     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 141      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 142     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 143      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 144     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 145      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 146     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 147      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 148     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 149      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 150     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 151      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 152     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 153      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 154     ('https://dl.google.com/android/repository/android-14_r04.zip',
 155      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 156     ('https://dl.google.com/android/repository/android-15_r05.zip',
 157      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 158     ('https://dl.google.com/android/repository/android-16_r05.zip',
 159      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 160     ('https://dl.google.com/android/repository/android-17_r03.zip',
 161      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 162     ('https://dl.google.com/android/repository/android-18_r03.zip',
 163      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 164     ('https://dl.google.com/android/repository/android-19_r04.zip',
 165      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 166     ('https://dl.google.com/android/repository/android-20_r02.zip',
 167      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 168     ('https://dl.google.com/android/repository/android-21_r02.zip',
 169      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 170     ('https://dl.google.com/android/repository/android-22_r02.zip',
 171      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 172     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 173      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 174     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 175      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 176     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 177      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 178     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 179      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 180     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 181      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 182     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 183      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 184     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 185      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 186     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 187      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 188     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 189      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 190     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 191      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 192     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 193      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 194     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 195      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 196     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 197      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 198     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 199      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 200     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 201      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 202     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 203      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 204     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 205      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 206     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 207      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 208     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 209      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 210     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 211      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 212     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 213      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 214     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 215      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 216     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 217      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 218     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 219      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 220     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 221      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 222     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 223      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 224     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 225      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 226     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 227      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 228     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 229      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 230     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 231      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 232     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 233      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 234     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 235      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 236     # the binaries that Google uses are here:
 237     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 238     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 239      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 240     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 241      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 242     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 243      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 244     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 245      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 246     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 247      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 248     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 249      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 250     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 251      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 252     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 253      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 254     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 255      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 256     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 257      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 258     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 259      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 260     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 261      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 262     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 263      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 264     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 265      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 266     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 267      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 268     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 269      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 270     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 271      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 272     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 273      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 274     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 275      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 276     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 277      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 278     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 279      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 280     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 281      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 282     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 283      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 284     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 285      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 286     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 287      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 288     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 289      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 290     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 291      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 292     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 293      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 294     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 295      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 296     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 297      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 298     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 299      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 300     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 301      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 302     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 303      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 304     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 305      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 306     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 307      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 308     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 309      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 310     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 311      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 312     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 313      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 314     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 315      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 316     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 317      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 318 ]
 319
 320
 321 def sha256_for_file(path):
 322     with open(path, 'rb') as f:
 323         s = hashlib.sha256()
 324         while True:
 325             data = f.read(4096)
 326             if not data:
 327                 break
 328             s.update(data)
 329         return s.hexdigest()
 330
 331
 332 def run_via_vagrant_ssh(v, cmdlist):
 333     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 334         cmd = cmdlist
 335     else:
 336         cmd = ' '.join(cmdlist)
 337     v._run_vagrant_command(['ssh', '-c', cmd])
 338
 339
 340 def update_cache(cachedir, cachefiles):
 341     for srcurl, shasum in cachefiles:
 342         filename = os.path.basename(srcurl)
 343         local_filename = os.path.join(cachedir, filename)
 344
 345         if os.path.exists(local_filename):
 346             local_length = os.path.getsize(local_filename)
 347         else:
 348             local_length = -1
 349
 350         resume_header = {}
 351         download = True
 352
 353         try:
 354             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 355             if r.status_code == 200:
 356                 content_length = int(r.headers.get('content-length'))
 357             else:
 358                 content_length = local_length  # skip the download
 359         except requests.exceptions.RequestException as e:
 360             content_length = local_length  # skip the download
 361             logger.warn('%s', e)
 362
 363         if local_length == content_length:
 364             download = False
 365         elif local_length > content_length:
 366             logger.info('deleting corrupt file from cache: %s', local_filename)
 367             os.remove(local_filename)
 368             logger.info("Downloading %s to cache", filename)
 369         elif local_length > -1 and local_length < content_length:
 370             logger.info("Resuming download of %s", local_filename)
 371             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 372         else:
 373             logger.info("Downloading %s to cache", filename)
 374
 375         if download:
 376             r = requests.get(srcurl, headers=resume_header,
 377                              stream=True, verify=False, allow_redirects=True)
 378             content_length = int(r.headers.get('content-length'))
 379             with open(local_filename, 'ab') as f:
 380                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 381                                           expected_size=(content_length / 65536) + 1):
 382                     if chunk:  # filter out keep-alive new chunks
 383                         f.write(chunk)
 384
 385         v = sha256_for_file(local_filename)
 386         if v == shasum:
 387             logger.info("\t...shasum verified for %s", local_filename)
 388         else:
 389             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 390             os.remove(local_filename)
 391             sys.exit(1)
 392
 393
 394 def debug_log_vagrant_vm(vm_dir, config):
 395     if options.verbosity >= 3:
 396         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 397         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 398         logger.debug('> vagrant status')
 399         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 400         logger.debug('> vagrant box list')
 401         subprocess.call(['vagrant', 'box', 'list'])
 402         if config['vm_provider'] == 'libvirt':
 403             logger.debug('> virsh -c qmeu:///system list --all')
 404             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 405             logger.debug('> virsh -c qemu:///system snapshot-list %s', config['domain'])
 406             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', config['domain']])
 407
 408
 409 def main():
 410     global cachedir, cachefiles, config, tail
 411
 412     if options.skip_cache_update:
 413         logger.info('skipping cache update and verification...')
 414     else:
 415         update_cache(cachedir, cachefiles)
 416
 417     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 418     logger.info("Setting executable bit for %s", local_qt_filename)
 419     os.chmod(local_qt_filename, 0o755)
 420
 421     # use VirtualBox software virtualization if hardware is not available,
 422     # like if this is being run in kvm or some other VM platform, like
 423     # http://jenkins.debian.net, the values are 'on' or 'off'
 424     if sys.platform.startswith('darwin'):
 425         # all < 10 year old Macs work, and OSX servers as VM host are very
 426         # rare, but this could also be auto-detected if someone codes it
 427         config['hwvirtex'] = 'on'
 428         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 429     elif os.path.exists('/proc/cpuinfo'):
 430         with open('/proc/cpuinfo') as f:
 431             contents = f.read()
 432         if 'vmx' in contents or 'svm' in contents:
 433             config['hwvirtex'] = 'on'
 434             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 435
 436     serverdir = os.path.join(os.getcwd(), 'buildserver')
 437     logfilename = os.path.join(serverdir, 'up.log')
 438     if not os.path.exists(logfilename):
 439         open(logfilename, 'a').close()  # create blank file
 440     log_cm = vagrant.make_file_cm(logfilename)
 441     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 442
 443     if options.verbosity >= 2:
 444         tail = fdroidserver.tail.Tail(logfilename)
 445         tail.start()
 446
 447     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 448     if options.clean:
 449         vm.destroy()
 450
 451     # Check against the existing Vagrantfile.yaml, and if they differ, we
 452     # need to create a new box:
 453     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 454     writevf = True
 455     if os.path.exists(vf):
 456         logger.info('Halting %s', serverdir)
 457         v.halt()
 458         with open(vf, 'r', encoding='utf-8') as f:
 459             oldconfig = yaml.load(f)
 460         if config != oldconfig:
 461             logger.info("Server configuration has changed, rebuild from scratch is required")
 462             vm.destroy()
 463         else:
 464             logger.info("Re-provisioning existing server")
 465             writevf = False
 466     else:
 467         logger.info("No existing server - building from scratch")
 468     if writevf:
 469         with open(vf, 'w', encoding='utf-8') as f:
 470             yaml.dump(config, f)
 471
 472     if config['vm_provider'] == 'libvirt':
 473         found_basebox = False
 474         needs_mutate = False
 475         for box in v.box_list():
 476             if box.name == config['basebox']:
 477                 found_basebox = True
 478                 if box.provider != 'libvirt':
 479                     needs_mutate = True
 480                 continue
 481         if not found_basebox:
 482             if isinstance(config['baseboxurl'], str):
 483                 baseboxurl = config['baseboxurl']
 484             else:
 485                 baseboxurl = config['baseboxurl'][0]
 486             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 487             v.box_add(config['basebox'], baseboxurl)
 488             needs_mutate = True
 489         if needs_mutate:
 490             logger.info('Converting %s to libvirt format', config['basebox'])
 491             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 492             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 493             v.box_remove(config['basebox'], 'virtualbox')
 494
 495     logger.info("Configuring build server VM")
 496     debug_log_vagrant_vm(serverdir, config)
 497     try:
 498         v.up(provision=True)
 499     except fdroidserver.vmtools.FDroidBuildVmException as e:
 500         debug_log_vagrant_vm(serverdir, config)
 501         logger.exception('could not bring buildserver vm up. %s', e)
 502         sys.exit(1)
 503
 504     if config['copy_caches_from_host']:
 505         ssh_config = v.ssh_config()
 506         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 507         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 508         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 509         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 510
 511         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 512             fullpath = os.path.join(os.getenv('HOME'), d)
 513             if os.path.isdir(fullpath):
 514                 # TODO newer versions of vagrant provide `vagrant rsync`
 515                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 516                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 517                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 518                                  fullpath + '/',
 519                                  user + '@' + hostname + ':~/' + d + '/'])
 520
 521         # this file changes every time but should not be cached
 522         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 523         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 524
 525     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 526                          universal_newlines=True)
 527     buildserverid = p.communicate()[0].strip()
 528     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 529     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 530
 531     logger.info("Stopping build server VM")
 532     v.halt()
 533
 534     logger.info("Packaging")
 535     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 536     if os.path.exists(boxfile):
 537         os.remove(boxfile)
 538
 539     vm.package(output=boxfile)
 540
 541     logger.info("Adding box")
 542     vm.box_add('buildserver', boxfile, force=True)
 543
 544     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 545         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 546         sys.exit(1)
 547
 548     if not options.keep_box_file:
 549         logger.debug('box added to vagrant, ' +
 550                      'removing generated box file \'%s\'',
 551                      boxfile)
 552         os.remove(boxfile)
 553
 554
 555 if __name__ == '__main__':
 556     try:
 557         main()
 558     finally:
 559         if tail is not None:
 560             tail.stop()