makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 import textwrap
  17 from clint.textui import progress
  18 from optparse import OptionParser
  19 import fdroidserver.tail
  20 import fdroidserver.vmtools
  21
  22
  23 parser = OptionParser()
  24 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  25                   help="Spew out even more information than normal")
  26 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  27 parser.add_option("-c", "--clean", action="store_true", default=False,
  28                   help="Build from scratch, rather than attempting to update the existing server")
  29 parser.add_option('--skip-cache-update', action="store_true", default=False,
  30                   help="""Skip downloading and checking cache."""
  31                        """This assumes that the cache is already downloaded completely.""")
  32 parser.add_option('--keep-box-file', action="store_true", default=False,
  33                   help="""Box file will not be deleted after adding it to box storage"""
  34                        """ (KVM-only).""")
  35 options, args = parser.parse_args()
  36
  37 logger = logging.getLogger('fdroidserver-makebuildserver')
  38 if options.verbosity >= 2:
  39     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  40     logger.setLevel(logging.DEBUG)
  41 elif options.verbosity == 1:
  42     logging.basicConfig(format='%(message)s', level=logging.INFO)
  43     logger.setLevel(logging.INFO)
  44 elif options.verbosity <= 0:
  45     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  46     logger.setLevel(logging.WARNING)
  47
  48
  49 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  50     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  51     sys.exit(1)
  52
  53 tail = None
  54
  55 # set up default config
  56 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  57 logger.debug('cachedir set to: %s', cachedir)
  58
  59 config = {
  60     'basebox': 'jessie64',
  61     'baseboxurl': [
  62         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  63         'https://f-droid.org/jessie64.box',
  64     ],
  65     'debian_mirror': 'http://http.debian.net/debian/',
  66     'apt_package_cache': False,
  67     'copy_caches_from_host': False,
  68     'boot_timeout': 600,
  69     'cachedir': cachedir,
  70     'cpus': 1,
  71     'memory': 1024,
  72     'hwvirtex': 'off',
  73     'vm_provider': 'virtualbox',
  74 }
  75
  76 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  77     try:
  78         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  79     except subprocess.CalledProcessError as e:
  80         virt = 'none'
  81     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  82         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  83         config['vm_provider'] = 'libvirt'
  84         config['domain'] = 'buildserver_default'
  85     elif virt != 'none':
  86         logger.info('Running in an unsupported VM guest (%s)!', virt)
  87     logger.debug('detected virt: %s', virt)
  88
  89 # load config file, if present
  90 if os.path.exists('makebuildserver.config.py'):
  91     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  92 elif os.path.exists('makebs.config.py'):
  93     # this is the old name for the config file
  94     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  95 if '__builtins__' in config:
  96     del(config['__builtins__'])  # added by compile/exec
  97 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  98
  99 # Update cached files.
 100 cachedir = config['cachedir']
 101 if not os.path.exists(cachedir):
 102     os.makedirs(cachedir, 0o755)
 103     logger.debug('created cachedir %s because it did not exists.', cachedir)
 104
 105 if config['vm_provider'] == 'libvirt':
 106     tmp = cachedir
 107     while tmp != '/':
 108         mode = os.stat(tmp).st_mode
 109         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 110             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 111             logger.critical('  chmod a+X %s', tmp)
 112             sys.exit(1)
 113         tmp = os.path.dirname(tmp)
 114     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 115
 116 if config['apt_package_cache']:
 117     config['aptcachedir'] = cachedir + '/apt/archives'
 118     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 119     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 120     if os.path.isfile(aptcachelock):
 121         logger.info('apt cache dir is locked, removing lock')
 122         os.remove(aptcachelock)
 123     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 124     if os.path.isdir(aptcachepartial):
 125         logger.info('removing partial downloads from apt cache dir')
 126         shutil.rmtree(aptcachepartial)
 127
 128 cachefiles = [
 129     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 130      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 131     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 132      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 133     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 134      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 135     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 136      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 137     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 138      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 139     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 140      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 141     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 142      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 143     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 144      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 145     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 146      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 147     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 148      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 149     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 150      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 151     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 152      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 153     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 154      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 155     ('https://dl.google.com/android/repository/android-14_r04.zip',
 156      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 157     ('https://dl.google.com/android/repository/android-15_r05.zip',
 158      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 159     ('https://dl.google.com/android/repository/android-16_r05.zip',
 160      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 161     ('https://dl.google.com/android/repository/android-17_r03.zip',
 162      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 163     ('https://dl.google.com/android/repository/android-18_r03.zip',
 164      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 165     ('https://dl.google.com/android/repository/android-19_r04.zip',
 166      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 167     ('https://dl.google.com/android/repository/android-20_r02.zip',
 168      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 169     ('https://dl.google.com/android/repository/android-21_r02.zip',
 170      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 171     ('https://dl.google.com/android/repository/android-22_r02.zip',
 172      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 173     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 174      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 175     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 176      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 177     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 178      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 179     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 180      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 181     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 182      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 184      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 185     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 186      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 187     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 188      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 189     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 190      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 192      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 194      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 195     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 196      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 197     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 198      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 199     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 200      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 201     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 202      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 204      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 206      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 208      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 209     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 210      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 211     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 212      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 213     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 214      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 215     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 216      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 217     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 218      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 220      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 221     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 222      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 223     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 224      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 225     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 226      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 228      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 229     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 230      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 231     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 232      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 233     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 234      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 235     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 236      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 237     # the binaries that Google uses are here:
 238     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 239     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 240      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 241     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 242      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 243     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 244      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 245     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 246      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 247     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 248      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 249     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 250      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 251     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 252      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 253     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 254      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 255     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 256      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 257     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 258      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 259     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 260      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 261     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 262      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 263     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 264      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 265     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 266      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 267     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 268      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 269     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 270      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 271     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 272      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 273     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 274      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 275     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 276      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 277     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 278      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 279     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 280      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 281     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 282      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 283     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 284      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 285     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 286      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 287     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 288      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 289     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 290      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 291     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 292      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 293     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 294      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 295     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 296      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 297     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 298      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 299     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 300      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 301     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 302      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 303     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 304      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 305     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 306      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 307     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 308      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 309     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 310      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 311     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 312      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 313     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 314      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 315     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 316      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 317     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 318      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 319 ]
 320
 321
 322 def sha256_for_file(path):
 323     with open(path, 'rb') as f:
 324         s = hashlib.sha256()
 325         while True:
 326             data = f.read(4096)
 327             if not data:
 328                 break
 329             s.update(data)
 330         return s.hexdigest()
 331
 332
 333 def run_via_vagrant_ssh(v, cmdlist):
 334     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 335         cmd = cmdlist
 336     else:
 337         cmd = ' '.join(cmdlist)
 338     v._run_vagrant_command(['ssh', '-c', cmd])
 339
 340
 341 def update_cache(cachedir, cachefiles):
 342     for srcurl, shasum in cachefiles:
 343         filename = os.path.basename(srcurl)
 344         local_filename = os.path.join(cachedir, filename)
 345
 346         if os.path.exists(local_filename):
 347             local_length = os.path.getsize(local_filename)
 348         else:
 349             local_length = -1
 350
 351         resume_header = {}
 352         download = True
 353
 354         try:
 355             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 356             if r.status_code == 200:
 357                 content_length = int(r.headers.get('content-length'))
 358             else:
 359                 content_length = local_length  # skip the download
 360         except requests.exceptions.RequestException as e:
 361             content_length = local_length  # skip the download
 362             logger.warn('%s', e)
 363
 364         if local_length == content_length:
 365             download = False
 366         elif local_length > content_length:
 367             logger.info('deleting corrupt file from cache: %s', local_filename)
 368             os.remove(local_filename)
 369             logger.info("Downloading %s to cache", filename)
 370         elif local_length > -1 and local_length < content_length:
 371             logger.info("Resuming download of %s", local_filename)
 372             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 373         else:
 374             logger.info("Downloading %s to cache", filename)
 375
 376         if download:
 377             r = requests.get(srcurl, headers=resume_header,
 378                              stream=True, verify=False, allow_redirects=True)
 379             content_length = int(r.headers.get('content-length'))
 380             with open(local_filename, 'ab') as f:
 381                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 382                                           expected_size=(content_length / 65536) + 1):
 383                     if chunk:  # filter out keep-alive new chunks
 384                         f.write(chunk)
 385
 386         v = sha256_for_file(local_filename)
 387         if v == shasum:
 388             logger.info("\t...shasum verified for %s", local_filename)
 389         else:
 390             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 391             os.remove(local_filename)
 392             sys.exit(1)
 393
 394
 395 def debug_log_vagrant_vm(vm_dir, config):
 396     if options.verbosity >= 3:
 397         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 398         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 399         logger.debug('> vagrant status')
 400         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 401         logger.debug('> vagrant box list')
 402         subprocess.call(['vagrant', 'box', 'list'])
 403         if config['vm_provider'] == 'libvirt':
 404             logger.debug('> virsh -c qmeu:///system list --all')
 405             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 406             logger.debug('> virsh -c qemu:///system snapshot-list %s', config['domain'])
 407             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', config['domain']])
 408
 409
 410 def main():
 411     global cachedir, cachefiles, config, tail
 412
 413     if options.skip_cache_update:
 414         logger.info('skipping cache update and verification...')
 415     else:
 416         update_cache(cachedir, cachefiles)
 417
 418     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 419     logger.info("Setting executable bit for %s", local_qt_filename)
 420     os.chmod(local_qt_filename, 0o755)
 421
 422     # use VirtualBox software virtualization if hardware is not available,
 423     # like if this is being run in kvm or some other VM platform, like
 424     # http://jenkins.debian.net, the values are 'on' or 'off'
 425     if sys.platform.startswith('darwin'):
 426         # all < 10 year old Macs work, and OSX servers as VM host are very
 427         # rare, but this could also be auto-detected if someone codes it
 428         config['hwvirtex'] = 'on'
 429         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 430     elif os.path.exists('/proc/cpuinfo'):
 431         with open('/proc/cpuinfo') as f:
 432             contents = f.read()
 433         if 'vmx' in contents or 'svm' in contents:
 434             config['hwvirtex'] = 'on'
 435             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 436
 437     serverdir = os.path.join(os.getcwd(), 'buildserver')
 438     logfilename = os.path.join(serverdir, 'up.log')
 439     if not os.path.exists(logfilename):
 440         open(logfilename, 'a').close()  # create blank file
 441     log_cm = vagrant.make_file_cm(logfilename)
 442     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 443
 444     if options.verbosity >= 2:
 445         tail = fdroidserver.tail.Tail(logfilename)
 446         tail.start()
 447
 448     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 449     if options.clean:
 450         vm.destroy()
 451
 452     # Check against the existing Vagrantfile.yaml, and if they differ, we
 453     # need to create a new box:
 454     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 455     writevf = True
 456     if os.path.exists(vf):
 457         logger.info('Halting %s', serverdir)
 458         v.halt()
 459         with open(vf, 'r', encoding='utf-8') as f:
 460             oldconfig = yaml.load(f)
 461         if config != oldconfig:
 462             logger.info("Server configuration has changed, rebuild from scratch is required")
 463             vm.destroy()
 464         else:
 465             logger.info("Re-provisioning existing server")
 466             writevf = False
 467     else:
 468         logger.info("No existing server - building from scratch")
 469     if writevf:
 470         with open(vf, 'w', encoding='utf-8') as f:
 471             yaml.dump(config, f)
 472
 473     if config['vm_provider'] == 'libvirt':
 474         found_basebox = False
 475         needs_mutate = False
 476         for box in v.box_list():
 477             if box.name == config['basebox']:
 478                 found_basebox = True
 479                 if box.provider != 'libvirt':
 480                     needs_mutate = True
 481                 continue
 482         if not found_basebox:
 483             if isinstance(config['baseboxurl'], str):
 484                 baseboxurl = config['baseboxurl']
 485             else:
 486                 baseboxurl = config['baseboxurl'][0]
 487             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 488             v.box_add(config['basebox'], baseboxurl)
 489             needs_mutate = True
 490         if needs_mutate:
 491             logger.info('Converting %s to libvirt format', config['basebox'])
 492             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 493             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 494             v.box_remove(config['basebox'], 'virtualbox')
 495
 496     logger.info("Configuring build server VM")
 497     debug_log_vagrant_vm(serverdir, config)
 498     try:
 499         v.up(provision=True)
 500     except fdroidserver.vmtools.FDroidBuildVmException as e:
 501         debug_log_vagrant_vm(serverdir, config)
 502         logger.exception('could not bring buildserver vm up. %s', e)
 503         sys.exit(1)
 504
 505     if config['copy_caches_from_host']:
 506         ssh_config = v.ssh_config()
 507         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 508         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 509         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 510         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 511
 512         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 513             fullpath = os.path.join(os.getenv('HOME'), d)
 514             if os.path.isdir(fullpath):
 515                 # TODO newer versions of vagrant provide `vagrant rsync`
 516                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 517                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 518                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 519                                  fullpath + '/',
 520                                  user + '@' + hostname + ':~/' + d + '/'])
 521
 522         # this file changes every time but should not be cached
 523         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 524         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 525
 526     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 527                          universal_newlines=True)
 528     buildserverid = p.communicate()[0].strip()
 529     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 530     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 531
 532     logger.info("Stopping build server VM")
 533     v.halt()
 534
 535     logger.info("Packaging")
 536     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 537     if os.path.exists(boxfile):
 538         os.remove(boxfile)
 539
 540     vm.package(output=boxfile)
 541
 542     logger.info("Adding box")
 543     vm.box_add('buildserver', boxfile, force=True)
 544
 545     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 546         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 547         sys.exit(1)
 548
 549     if not options.keep_box_file:
 550         logger.debug('box added to vagrant, ' +
 551                      'removing generated box file \'%s\'',
 552                      boxfile)
 553         os.remove(boxfile)
 554
 555
 556 if __name__ == '__main__':
 557     try:
 558         main()
 559     finally:
 560         if tail is not None:
 561             tail.stop()