makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 from clint.textui import progress
  16 from optparse import OptionParser
  17 import fdroidserver.tail
  18
  19
  20 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  21     print('This must be run as ./makebuildserver in fdroidserver.git!')
  22     sys.exit(1)
  23
  24
  25 tail = None
  26
  27 parser = OptionParser()
  28 parser.add_option("-v", "--verbose", action="store_true", default=False,
  29                   help="Spew out even more information than normal")
  30 parser.add_option("-c", "--clean", action="store_true", default=False,
  31                   help="Build from scratch, rather than attempting to update the existing server")
  32 options, args = parser.parse_args()
  33
  34 # set up default config
  35 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  36 config = {
  37     'basebox': 'jessie64',
  38     'baseboxurl': [
  39         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  40         'https://f-droid.org/jessie64.box',
  41     ],
  42     'debian_mirror': 'http://http.debian.net/debian/',
  43     'apt_package_cache': False,
  44     'copy_caches_from_host': False,
  45     'boot_timeout': 600,
  46     'cachedir': cachedir,
  47     'cpus': 1,
  48     'memory': 1024,
  49     'hwvirtex': 'off',
  50     'vm_provider': 'virtualbox',
  51 }
  52
  53 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  54     try:
  55         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  56     except subprocess.CalledProcessError as e:
  57         virt = 'none'
  58     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  59         print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  60         config['vm_provider'] = 'libvirt'
  61         config['domain'] = 'buildserver_default'
  62     elif virt != 'none':
  63         print('Running in an unsupported VM guest (' + virt + ')!')
  64
  65 # load config file, if present
  66 if os.path.exists('makebuildserver.config.py'):
  67     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  68 elif os.path.exists('makebs.config.py'):
  69     # this is the old name for the config file
  70     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  71 if '__builtins__' in config:
  72     del(config['__builtins__'])  # added by compile/exec
  73
  74 # Update cached files.
  75 cachedir = config['cachedir']
  76 if not os.path.exists(cachedir):
  77     os.makedirs(cachedir, 0o755)
  78
  79 if config['vm_provider'] == 'libvirt':
  80     tmp = cachedir
  81     while tmp != '/':
  82         mode = os.stat(tmp).st_mode
  83         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
  84             print('ERROR:', tmp, 'will not be accessible to the VM!  To fix, run:')
  85             print('  chmod a+X', tmp)
  86             sys.exit(1)
  87         tmp = os.path.dirname(tmp)
  88
  89 if config['apt_package_cache']:
  90     config['aptcachedir'] = cachedir + '/apt/archives'
  91
  92 cachefiles = [
  93     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
  94      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
  95     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
  96      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
  97     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
  98      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
  99     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 100      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 101     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 102      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 103     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 104      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 105     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 106      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 107     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 108      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 109     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 110      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 111     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 112      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 113     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 114      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 115     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 116      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 117     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 118      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 119     ('https://dl.google.com/android/repository/android-14_r04.zip',
 120      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 121     ('https://dl.google.com/android/repository/android-15_r05.zip',
 122      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 123     ('https://dl.google.com/android/repository/android-16_r05.zip',
 124      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 125     ('https://dl.google.com/android/repository/android-17_r03.zip',
 126      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 127     ('https://dl.google.com/android/repository/android-18_r03.zip',
 128      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 129     ('https://dl.google.com/android/repository/android-19_r04.zip',
 130      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 131     ('https://dl.google.com/android/repository/android-20_r02.zip',
 132      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 133     ('https://dl.google.com/android/repository/android-21_r02.zip',
 134      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 135     ('https://dl.google.com/android/repository/android-22_r02.zip',
 136      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 137     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 138      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 139     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 140      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 141     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 142      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 143     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 144      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 145     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 146      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 147     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 148      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 149     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 150      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 151     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 152      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 153     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 154      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 155     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 156      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 157     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 158      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 159     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 160      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 161     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 162      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 163     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 164      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 165     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 166      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 167     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 168      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 169     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 170      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 171     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 172      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 173     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 174      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 175     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 176      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 177     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 178      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 179     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 180      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 181     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 182      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 183     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 184      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 185     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 186      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 187     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 188      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 189     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 190      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 191     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 192      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 193     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 194      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 195     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 196      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 197     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 198      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 199     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 200      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 201     # the binaries that Google uses are here:
 202     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 203     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 204      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 205     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 206      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 207     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 208      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 209     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 210      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 211     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 212      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 213     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 214      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 215     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 216      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 217     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 218      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 219     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 220      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 221     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 222      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 223     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 224      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 225     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 226      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 227     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 228      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 229     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 230      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 231     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 232      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 233     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 234      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 235     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 236      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 237     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 238      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 239     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 240      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 241     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 242      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 243     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 244      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 245     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 246      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 247     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 248      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 249     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 250      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 251     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 252      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 253     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 254      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 255     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 256      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 257     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 258      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 259     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 260      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 261     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 262      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 263     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 264      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 265     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 266      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 267     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 268      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 269     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 270      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 271     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 272      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 273     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 274      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 275     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 276      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 277     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 278      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 279     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 280      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 281     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 282      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 283 ]
 284
 285
 286 def sha256_for_file(path):
 287     with open(path, 'rb') as f:
 288         s = hashlib.sha256()
 289         while True:
 290             data = f.read(4096)
 291             if not data:
 292                 break
 293             s.update(data)
 294         return s.hexdigest()
 295
 296
 297 def destroy_current_image(v, serverdir):
 298     global config
 299
 300     # cannot run vagrant without the config in the YAML file
 301     if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')):
 302         v.destroy()
 303     elif options.verbose:
 304         print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!')
 305     if config['vm_provider'] == 'libvirt':
 306         import libvirt
 307         try:
 308             virConnect = libvirt.open('qemu:///system')
 309             virDomain = virConnect.lookupByName(config['domain'])
 310             if virDomain:
 311                 virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE
 312                                         | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA
 313                                         | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM)
 314             storagePool = virConnect.storagePoolLookupByName('default')
 315             if storagePool:
 316                 for vol in storagePool.listAllVolumes():
 317                     vol.delete()
 318         except libvirt.libvirtError as e:
 319             print(e)
 320
 321
 322 def kvm_package(boxfile):
 323     '''
 324     Hack to replace missing `vagrant package` for kvm, based  on the script
 325     `tools/create_box.sh from vagrant-libvirt
 326     '''
 327     import libvirt
 328     virConnect = libvirt.open('qemu:///system')
 329     storagePool = virConnect.storagePoolLookupByName('default')
 330     if storagePool:
 331         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 332         imagepath = vol.path()
 333         # TODO use a libvirt storage pool to ensure the img file is readable
 334         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 335         shutil.copy2(imagepath, 'box.img')
 336         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 337         metadata = """{
 338     "provider": "libvirt",
 339     "format": "qcow2",
 340     "virtual_size": 1000
 341 }
 342 """
 343         vagrantfile = """Vagrant.configure("2") do |config|
 344
 345   config.vm.provider :libvirt do |libvirt|
 346
 347     libvirt.driver = "kvm"
 348     libvirt.host = ""
 349     libvirt.connect_via_ssh = false
 350     libvirt.storage_pool_name = "default"
 351
 352   end
 353 end
 354 """
 355         with open('metadata.json', 'w') as fp:
 356             fp.write(metadata)
 357         with open('Vagrantfile', 'w') as fp:
 358             fp.write(vagrantfile)
 359         with tarfile.open(boxfile, 'w:gz') as tar:
 360             tar.add('metadata.json')
 361             tar.add('Vagrantfile')
 362             tar.add('box.img')
 363         os.remove('metadata.json')
 364         os.remove('Vagrantfile')
 365         os.remove('box.img')
 366
 367
 368 def run_via_vagrant_ssh(v, cmdlist):
 369     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 370         cmd = cmdlist
 371     else:
 372         cmd = ' '.join(cmdlist)
 373     v._run_vagrant_command(['ssh', '-c', cmd])
 374
 375
 376 def main():
 377     global cachedir, cachefiles, config, tail
 378
 379     for srcurl, shasum in cachefiles:
 380         filename = os.path.basename(srcurl)
 381         local_filename = os.path.join(cachedir, filename)
 382
 383         if os.path.exists(local_filename):
 384             local_length = os.path.getsize(local_filename)
 385         else:
 386             local_length = -1
 387
 388         resume_header = {}
 389         download = True
 390
 391         try:
 392             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 393             if r.status_code == 200:
 394                 content_length = int(r.headers.get('content-length'))
 395             else:
 396                 content_length = local_length  # skip the download
 397         except requests.exceptions.RequestException as e:
 398             content_length = local_length  # skip the download
 399             print(e)
 400
 401         if local_length == content_length:
 402             download = False
 403         elif local_length > content_length:
 404             print('deleting corrupt file from cache: ' + local_filename)
 405             os.remove(local_filename)
 406             print("Downloading " + filename + " to cache")
 407         elif local_length > -1 and local_length < content_length:
 408             print("Resuming download of " + local_filename)
 409             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 410         else:
 411             print("Downloading " + filename + " to cache")
 412
 413         if download:
 414             r = requests.get(srcurl, headers=resume_header,
 415                              stream=True, verify=False, allow_redirects=True)
 416             content_length = int(r.headers.get('content-length'))
 417             with open(local_filename, 'ab') as f:
 418                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 419                                           expected_size=(content_length / 65536) + 1):
 420                     if chunk:  # filter out keep-alive new chunks
 421                         f.write(chunk)
 422
 423         v = sha256_for_file(local_filename)
 424         if v == shasum:
 425             print("\t...shasum verified for " + local_filename)
 426         else:
 427             print("Invalid shasum of '" + v + "' detected for " + local_filename)
 428             os.remove(local_filename)
 429             sys.exit(1)
 430
 431     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 432     print("Setting executable bit for " + local_qt_filename)
 433     os.chmod(local_qt_filename, 0o755)
 434
 435     # use VirtualBox software virtualization if hardware is not available,
 436     # like if this is being run in kvm or some other VM platform, like
 437     # http://jenkins.debian.net, the values are 'on' or 'off'
 438     if sys.platform.startswith('darwin'):
 439         # all < 10 year old Macs work, and OSX servers as VM host are very
 440         # rare, but this could also be auto-detected if someone codes it
 441         config['hwvirtex'] = 'on'
 442     elif os.path.exists('/proc/cpuinfo'):
 443         with open('/proc/cpuinfo') as f:
 444             contents = f.read()
 445         if 'vmx' in contents or 'svm' in contents:
 446             config['hwvirtex'] = 'on'
 447
 448     serverdir = os.path.join(os.getcwd(), 'buildserver')
 449     logfilename = os.path.join(serverdir, 'up.log')
 450     if not os.path.exists(logfilename):
 451         open(logfilename, 'a').close()  # create blank file
 452     log_cm = vagrant.make_file_cm(logfilename)
 453     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 454
 455     if options.verbose:
 456         tail = fdroidserver.tail.Tail(logfilename)
 457         tail.start()
 458
 459     if options.clean:
 460         destroy_current_image(v, serverdir)
 461
 462     # Check against the existing Vagrantfile.yaml, and if they differ, we
 463     # need to create a new box:
 464     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 465     writevf = True
 466     if os.path.exists(vf):
 467         print('Halting', serverdir)
 468         v.halt()
 469         with open(vf, 'r', encoding='utf-8') as f:
 470             oldconfig = yaml.load(f)
 471         if config != oldconfig:
 472             print("Server configuration has changed, rebuild from scratch is required")
 473             destroy_current_image(v, serverdir)
 474         else:
 475             print("Re-provisioning existing server")
 476             writevf = False
 477     else:
 478         print("No existing server - building from scratch")
 479     if writevf:
 480         with open(vf, 'w', encoding='utf-8') as f:
 481             yaml.dump(config, f)
 482
 483     if config['vm_provider'] == 'libvirt':
 484         found_basebox = False
 485         needs_mutate = False
 486         for box in v.box_list():
 487             if box.name == config['basebox']:
 488                 found_basebox = True
 489                 if box.provider != 'libvirt':
 490                     needs_mutate = True
 491                 continue
 492         if not found_basebox:
 493             if isinstance(config['baseboxurl'], str):
 494                 baseboxurl = config['baseboxurl']
 495             else:
 496                 baseboxurl = config['baseboxurl'][0]
 497             print('Adding', config['basebox'], 'from', baseboxurl)
 498             v.box_add(config['basebox'], baseboxurl)
 499             needs_mutate = True
 500         if needs_mutate:
 501             print('Converting', config['basebox'], 'to libvirt format')
 502             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 503             print('Removing virtualbox format copy of', config['basebox'])
 504             v.box_remove(config['basebox'], 'virtualbox')
 505
 506     print("Configuring build server VM")
 507     v.up(provision=True)
 508
 509     if config['copy_caches_from_host']:
 510         ssh_config = v.ssh_config()
 511         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 512         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 513         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 514         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 515
 516         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 517             fullpath = os.path.join(os.getenv('HOME'), d)
 518             if os.path.isdir(fullpath):
 519                 # TODO newer versions of vagrant provide `vagrant rsync`
 520                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 521                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 522                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 523                                  fullpath + '/',
 524                                  user + '@' + hostname + ':~/' + d + '/'])
 525
 526         # this file changes every time but should not be cached
 527         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 528         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 529
 530     print("Writing buildserver ID")
 531     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 532                          universal_newlines=True)
 533     buildserverid = p.communicate()[0].strip()
 534     print("...ID is " + buildserverid)
 535     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 536
 537     print("Stopping build server VM")
 538     v.halt()
 539
 540     print("Packaging")
 541     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 542     if os.path.exists(boxfile):
 543         os.remove(boxfile)
 544
 545     if config['vm_provider'] == 'libvirt':
 546         kvm_package(boxfile)
 547     else:
 548         v.package(output=boxfile)
 549
 550     print("Adding box")
 551     v.box_add('buildserver', boxfile, force=True)
 552
 553     os.remove(boxfile)
 554
 555
 556 if __name__ == '__main__':
 557     try:
 558         main()
 559     finally:
 560         if tail is not None:
 561             tail.stop()