makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import stat
   8 import sys
   9 import shutil
  10 import subprocess
  11 import vagrant
  12 import hashlib
  13 import yaml
  14 import json
  15 import logging
  16 from clint.textui import progress
  17 from optparse import OptionParser
  18 import fdroidserver.tail
  19 import fdroidserver.vmtools
  20
  21
  22 parser = OptionParser()
  23 parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
  24                   help="Spew out even more information than normal")
  25 parser.add_option('-q', action='store_const', const=0, dest='verbosity')
  26 parser.add_option("-c", "--clean", action="store_true", default=False,
  27                   help="Build from scratch, rather than attempting to update the existing server")
  28 parser.add_option('--skip-cache-update', action="store_true", default=False,
  29                   help="""Skip downloading and checking cache."""
  30                        """This assumes that the cache is already downloaded completely.""")
  31 parser.add_option('--keep-box-file', action="store_true", default=False,
  32                   help="""Box file will not be deleted after adding it to box storage"""
  33                        """ (KVM-only).""")
  34 options, args = parser.parse_args()
  35
  36 logger = logging.getLogger('fdroidserver-makebuildserver')
  37 if options.verbosity >= 2:
  38     logging.basicConfig(format='%(message)s', level=logging.DEBUG)
  39     logger.setLevel(logging.DEBUG)
  40 elif options.verbosity == 1:
  41     logging.basicConfig(format='%(message)s', level=logging.INFO)
  42     logger.setLevel(logging.INFO)
  43 elif options.verbosity <= 0:
  44     logging.basicConfig(format='%(message)s', level=logging.WARNING)
  45     logger.setLevel(logging.WARNING)
  46
  47
  48 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  49     logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
  50     sys.exit(1)
  51
  52 tail = None
  53
  54 # set up default config
  55 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  56 logger.debug('cachedir set to: %s', cachedir)
  57
  58 config = {
  59     'basebox': 'jessie64',
  60     'baseboxurl': [
  61         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  62         'https://f-droid.org/jessie64.box',
  63     ],
  64     'debian_mirror': 'http://http.debian.net/debian/',
  65     'apt_package_cache': False,
  66     'copy_caches_from_host': False,
  67     'boot_timeout': 600,
  68     'cachedir': cachedir,
  69     'cpus': 1,
  70     'memory': 1024,
  71     'hwvirtex': 'off',
  72     'vm_provider': 'virtualbox',
  73 }
  74
  75 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  76     try:
  77         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  78     except subprocess.CalledProcessError as e:
  79         virt = 'none'
  80     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  81         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  82         config['vm_provider'] = 'libvirt'
  83     elif virt != 'none':
  84         logger.info('Running in an unsupported VM guest (%s)!', virt)
  85     logger.debug('detected virt: %s', virt)
  86
  87 # load config file, if present
  88 if os.path.exists('makebuildserver.config.py'):
  89     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  90 elif os.path.exists('makebs.config.py'):
  91     # this is the old name for the config file
  92     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  93 if '__builtins__' in config:
  94     del(config['__builtins__'])  # added by compile/exec
  95 logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
  96
  97 # Update cached files.
  98 cachedir = config['cachedir']
  99 if not os.path.exists(cachedir):
 100     os.makedirs(cachedir, 0o755)
 101     logger.debug('created cachedir %s because it did not exists.', cachedir)
 102
 103 if config['vm_provider'] == 'libvirt':
 104     tmp = cachedir
 105     while tmp != '/':
 106         mode = os.stat(tmp).st_mode
 107         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
 108             logger.critical('ERROR: %s will not be accessible to the VM!  To fix, run:', tmp)
 109             logger.critical('  chmod a+X %s', tmp)
 110             sys.exit(1)
 111         tmp = os.path.dirname(tmp)
 112     logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
 113
 114 if config['apt_package_cache']:
 115     config['aptcachedir'] = cachedir + '/apt/archives'
 116     logger.debug('aptcachedir is set to %s', config['aptcachedir'])
 117     aptcachelock = os.path.join(config['aptcachedir'], 'lock')
 118     if os.path.isfile(aptcachelock):
 119         logger.info('apt cache dir is locked, removing lock')
 120         os.remove(aptcachelock)
 121     aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
 122     if os.path.isdir(aptcachepartial):
 123         logger.info('removing partial downloads from apt cache dir')
 124         shutil.rmtree(aptcachepartial)
 125
 126 cachefiles = [
 127     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
 128      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
 129     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
 130      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
 131     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 132      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 133     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 134      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 135     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 136      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 137     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 138      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 139     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 140      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 141     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 142      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 143     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 144      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 145     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 146      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 147     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 148      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 149     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 150      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 151     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 152      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 153     ('https://dl.google.com/android/repository/android-14_r04.zip',
 154      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 155     ('https://dl.google.com/android/repository/android-15_r05.zip',
 156      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 157     ('https://dl.google.com/android/repository/android-16_r05.zip',
 158      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 159     ('https://dl.google.com/android/repository/android-17_r03.zip',
 160      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 161     ('https://dl.google.com/android/repository/android-18_r03.zip',
 162      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 163     ('https://dl.google.com/android/repository/android-19_r04.zip',
 164      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 165     ('https://dl.google.com/android/repository/android-20_r02.zip',
 166      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 167     ('https://dl.google.com/android/repository/android-21_r02.zip',
 168      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 169     ('https://dl.google.com/android/repository/android-22_r02.zip',
 170      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 171     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 172      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 173     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 174      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 175     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 176      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 177     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 178      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 179     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 180      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 181     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 182      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 183     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 184      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 185     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 186      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 187     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 188      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 189     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 190      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 191     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 192      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 193     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 194      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 195     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 196      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 197     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 198      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 199     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 200      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 201     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 202      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 203     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 204      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 205     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 206      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 207     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 208      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 209     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 210      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 211     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 212      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 213     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 214      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 215     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 216      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 217     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 218      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 219     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 220      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 221     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 222      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 223     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 224      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 225     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 226      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 227     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 228      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 229     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 230      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 231     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 232      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 233     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 234      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 235     # the binaries that Google uses are here:
 236     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 237     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 238      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 239     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 240      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 241     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 242      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 243     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 244      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 245     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 246      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 247     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 248      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 249     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 250      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 251     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 252      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 253     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 254      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 255     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 256      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 257     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 258      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 259     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 260      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 261     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 262      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 263     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 264      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 265     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 266      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 267     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 268      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 269     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 270      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 271     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 272      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 273     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 274      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 275     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 276      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 277     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 278      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 279     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 280      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 281     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 282      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 283     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 284      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 285     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 286      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 287     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 288      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 289     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 290      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 291     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 292      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 293     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 294      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 295     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 296      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 297     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 298      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 299     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 300      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 301     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 302      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 303     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 304      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 305     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 306      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 307     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 308      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 309     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 310      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 311     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 312      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 313     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 314      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 315     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 316      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 317 ]
 318
 319
 320 def sha256_for_file(path):
 321     with open(path, 'rb') as f:
 322         s = hashlib.sha256()
 323         while True:
 324             data = f.read(4096)
 325             if not data:
 326                 break
 327             s.update(data)
 328         return s.hexdigest()
 329
 330
 331 def run_via_vagrant_ssh(v, cmdlist):
 332     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 333         cmd = cmdlist
 334     else:
 335         cmd = ' '.join(cmdlist)
 336     v._run_vagrant_command(['ssh', '-c', cmd])
 337
 338
 339 def update_cache(cachedir, cachefiles):
 340     for srcurl, shasum in cachefiles:
 341         filename = os.path.basename(srcurl)
 342         local_filename = os.path.join(cachedir, filename)
 343
 344         if os.path.exists(local_filename):
 345             local_length = os.path.getsize(local_filename)
 346         else:
 347             local_length = -1
 348
 349         resume_header = {}
 350         download = True
 351
 352         try:
 353             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 354             if r.status_code == 200:
 355                 content_length = int(r.headers.get('content-length'))
 356             else:
 357                 content_length = local_length  # skip the download
 358         except requests.exceptions.RequestException as e:
 359             content_length = local_length  # skip the download
 360             logger.warn('%s', e)
 361
 362         if local_length == content_length:
 363             download = False
 364         elif local_length > content_length:
 365             logger.info('deleting corrupt file from cache: %s', local_filename)
 366             os.remove(local_filename)
 367             logger.info("Downloading %s to cache", filename)
 368         elif local_length > -1 and local_length < content_length:
 369             logger.info("Resuming download of %s", local_filename)
 370             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 371         else:
 372             logger.info("Downloading %s to cache", filename)
 373
 374         if download:
 375             r = requests.get(srcurl, headers=resume_header,
 376                              stream=True, verify=False, allow_redirects=True)
 377             content_length = int(r.headers.get('content-length'))
 378             with open(local_filename, 'ab') as f:
 379                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 380                                           expected_size=(content_length / 65536) + 1):
 381                     if chunk:  # filter out keep-alive new chunks
 382                         f.write(chunk)
 383
 384         v = sha256_for_file(local_filename)
 385         if v == shasum:
 386             logger.info("\t...shasum verified for %s", local_filename)
 387         else:
 388             logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
 389             os.remove(local_filename)
 390             sys.exit(1)
 391
 392
 393 def debug_log_vagrant_vm(vm_dir, config):
 394     if options.verbosity >= 3:
 395         _vagrant_dir = os.path.join(vm_dir, '.vagrant')
 396         logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
 397         logger.debug('> vagrant status')
 398         subprocess.call(['vagrant', 'status'], cwd=vm_dir)
 399         logger.debug('> vagrant box list')
 400         subprocess.call(['vagrant', 'box', 'list'])
 401         if config['vm_provider'] == 'libvirt':
 402             logger.debug('> virsh -c qmeu:///system list --all')
 403             subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
 404             domain = 'buildserver_default'
 405             logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
 406             subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
 407
 408
 409 def main():
 410     global cachedir, cachefiles, config, tail
 411
 412     if options.skip_cache_update:
 413         logger.info('skipping cache update and verification...')
 414     else:
 415         update_cache(cachedir, cachefiles)
 416
 417     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 418     logger.info("Setting executable bit for %s", local_qt_filename)
 419     os.chmod(local_qt_filename, 0o755)
 420
 421     # use VirtualBox software virtualization if hardware is not available,
 422     # like if this is being run in kvm or some other VM platform, like
 423     # http://jenkins.debian.net, the values are 'on' or 'off'
 424     if sys.platform.startswith('darwin'):
 425         # all < 10 year old Macs work, and OSX servers as VM host are very
 426         # rare, but this could also be auto-detected if someone codes it
 427         config['hwvirtex'] = 'on'
 428         logger.info('platform is darwnin -> hwvirtex = \'on\'')
 429     elif os.path.exists('/proc/cpuinfo'):
 430         with open('/proc/cpuinfo') as f:
 431             contents = f.read()
 432         if 'vmx' in contents or 'svm' in contents:
 433             config['hwvirtex'] = 'on'
 434             logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
 435
 436     serverdir = os.path.join(os.getcwd(), 'buildserver')
 437     logfilename = os.path.join(serverdir, 'up.log')
 438     if not os.path.exists(logfilename):
 439         open(logfilename, 'a').close()  # create blank file
 440     log_cm = vagrant.make_file_cm(logfilename)
 441     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 442
 443     if options.verbosity >= 2:
 444         tail = fdroidserver.tail.Tail(logfilename)
 445         tail.start()
 446
 447     vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
 448     if options.clean:
 449         vm.destroy()
 450
 451     # Check against the existing Vagrantfile.yaml, and if they differ, we
 452     # need to create a new box:
 453     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 454     writevf = True
 455     if os.path.exists(vf):
 456         logger.info('Halting %s', serverdir)
 457         v.halt()
 458         with open(vf, 'r', encoding='utf-8') as f:
 459             oldconfig = yaml.load(f)
 460         if config != oldconfig:
 461             logger.info("Server configuration has changed, rebuild from scratch is required")
 462             vm.destroy()
 463         else:
 464             logger.info("Re-provisioning existing server")
 465             writevf = False
 466     else:
 467         logger.info("No existing server - building from scratch")
 468     if writevf:
 469         with open(vf, 'w', encoding='utf-8') as f:
 470             yaml.dump(config, f)
 471
 472     if config['vm_provider'] == 'libvirt':
 473         found_basebox = False
 474         needs_mutate = False
 475         for box in v.box_list():
 476             if box.name == config['basebox']:
 477                 found_basebox = True
 478                 if box.provider != 'libvirt':
 479                     needs_mutate = True
 480                 continue
 481         if not found_basebox:
 482             if isinstance(config['baseboxurl'], str):
 483                 baseboxurl = config['baseboxurl']
 484             else:
 485                 baseboxurl = config['baseboxurl'][0]
 486             logger.info('Adding %s from %s', config['basebox'], baseboxurl)
 487             v.box_add(config['basebox'], baseboxurl)
 488             needs_mutate = True
 489         if needs_mutate:
 490             logger.info('Converting %s to libvirt format', config['basebox'])
 491             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 492             logger.info('Removing virtualbox format copy of %s', config['basebox'])
 493             v.box_remove(config['basebox'], 'virtualbox')
 494
 495     logger.info("Configuring build server VM")
 496     debug_log_vagrant_vm(serverdir, config)
 497     try:
 498         v.up(provision=True)
 499     except fdroidserver.vmtools.FDroidBuildVmException as e:
 500         debug_log_vagrant_vm(serverdir, config)
 501         logger.exception('could not bring buildserver vm up. %s', e)
 502         sys.exit(1)
 503
 504     if config['copy_caches_from_host']:
 505         ssh_config = v.ssh_config()
 506         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 507         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 508         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 509         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 510
 511         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 512             fullpath = os.path.join(os.getenv('HOME'), d)
 513             if os.path.isdir(fullpath):
 514                 # TODO newer versions of vagrant provide `vagrant rsync`
 515                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 516                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 517                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 518                                  fullpath + '/',
 519                                  user + '@' + hostname + ':~/' + d + '/'])
 520
 521         # this file changes every time but should not be cached
 522         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 523         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 524
 525     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 526                          universal_newlines=True)
 527     buildserverid = p.communicate()[0].strip()
 528     logger.info("Writing buildserver ID ...ID is %s", buildserverid)
 529     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 530
 531     logger.info("Stopping build server VM")
 532     v.halt()
 533
 534     logger.info("Packaging")
 535     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 536     if os.path.exists(boxfile):
 537         os.remove(boxfile)
 538
 539     vm.package(output=boxfile)
 540
 541     logger.info("Adding box")
 542     vm.box_add('buildserver', boxfile, force=True)
 543
 544     if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
 545         logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
 546         sys.exit(1)
 547
 548     if not options.keep_box_file:
 549         logger.debug('box added to vagrant, ' +
 550                      'removing generated box file \'%s\'',
 551                      boxfile)
 552         os.remove(boxfile)
 553
 554
 555 if __name__ == '__main__':
 556     try:
 557         main()
 558     finally:
 559         if tail is not None:
 560             tail.stop()