makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 from clint.textui import progress
  16 from optparse import OptionParser
  17 import fdroidserver.tail
  18
  19
  20 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  21     print('This must be run as ./makebuildserver in fdroidserver.git!')
  22     sys.exit(1)
  23
  24
  25 tail = None
  26
  27 parser = OptionParser()
  28 parser.add_option("-v", "--verbose", action="store_true", default=False,
  29                   help="Spew out even more information than normal")
  30 parser.add_option("-c", "--clean", action="store_true", default=False,
  31                   help="Build from scratch, rather than attempting to update the existing server")
  32 options, args = parser.parse_args()
  33
  34 # set up default config
  35 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  36 config = {
  37     'basebox': 'jessie64',
  38     'baseboxurl': [
  39         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  40         'https://f-droid.org/jessie64.box',
  41     ],
  42     'debian_mirror': 'http://http.debian.net/debian/',
  43     'apt_package_cache': False,
  44     'copy_caches_from_host': False,
  45     'boot_timeout': 600,
  46     'cachedir': cachedir,
  47     'cpus': 1,
  48     'memory': 1024,
  49     'hwvirtex': 'off',
  50     'vm_provider': 'virtualbox',
  51 }
  52
  53 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  54     try:
  55         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  56     except subprocess.CalledProcessError as e:
  57         virt = 'none'
  58     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  59         print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  60         config['vm_provider'] = 'libvirt'
  61         config['domain'] = 'buildserver_default'
  62     elif virt != 'none':
  63         print('Running in an unsupported VM guest (' + virt + ')!')
  64
  65 # load config file, if present
  66 if os.path.exists('makebuildserver.config.py'):
  67     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  68 elif os.path.exists('makebs.config.py'):
  69     # this is the old name for the config file
  70     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  71 if '__builtins__' in config:
  72     del(config['__builtins__'])  # added by compile/exec
  73
  74 # Update cached files.
  75 cachedir = config['cachedir']
  76 if not os.path.exists(cachedir):
  77     os.makedirs(cachedir, 0o755)
  78
  79 if config['vm_provider'] == 'libvirt':
  80     tmp = cachedir
  81     while tmp != '/':
  82         mode = os.stat(tmp).st_mode
  83         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
  84             print('ERROR:', tmp, 'will not be accessible to the VM!  To fix, run:')
  85             print('  chmod a+X', tmp)
  86             sys.exit(1)
  87         tmp = os.path.dirname(tmp)
  88
  89 if config['apt_package_cache']:
  90     config['aptcachedir'] = cachedir + '/apt/archives'
  91
  92 cachefiles = [
  93     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
  94      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
  95     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
  96      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
  97     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
  98      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
  99     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 100      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 101     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 102      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 103     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 104      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 105     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 106      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 107     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 108      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 109     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 110      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 111     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 112      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 113     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 114      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 115     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 116      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 117     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 118      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 119     ('https://dl.google.com/android/repository/android-14_r04.zip',
 120      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 121     ('https://dl.google.com/android/repository/android-15_r05.zip',
 122      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 123     ('https://dl.google.com/android/repository/android-16_r05.zip',
 124      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 125     ('https://dl.google.com/android/repository/android-17_r03.zip',
 126      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 127     ('https://dl.google.com/android/repository/android-18_r03.zip',
 128      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 129     ('https://dl.google.com/android/repository/android-19_r04.zip',
 130      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 131     ('https://dl.google.com/android/repository/android-20_r02.zip',
 132      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 133     ('https://dl.google.com/android/repository/android-21_r02.zip',
 134      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 135     ('https://dl.google.com/android/repository/android-22_r02.zip',
 136      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 137     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 138      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 139     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 140      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 141     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 142      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 143     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 144      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 145     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 146      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 147     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 148      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 149     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 150      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 151     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 152      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 153     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 154      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 155     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 156      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 157     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 158      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 159     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 160      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 161     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 162      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 163     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 164      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 165     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 166      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 167     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 168      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 169     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 170      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 171     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 172      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 173     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 174      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 175     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 176      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 177     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 178      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 179     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 180      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 181     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 182      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 183     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 184      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 185     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 186      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 187     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 188      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 189     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 190      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 191     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 192      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 193     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 194      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 195     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 196      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 197     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 198      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 199     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 200      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 201     # the binaries that Google uses are here:
 202     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 203     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 204      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 205     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 206      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 207     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 208      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 209     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 210      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 211     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 212      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 213     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 214      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 215     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 216      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 217     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 218      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 219     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 220      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 221     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 222      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 223     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 224      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 225     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 226      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 227     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 228      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 229     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 230      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 231     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 232      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 233     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 234      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 235     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 236      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 237     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 238      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 239     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 240      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 241     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 242      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 243     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 244      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 245     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 246      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 247     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 248      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 249     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 250      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 251     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 252      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 253     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 254      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 255     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 256      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 257     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 258      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 259     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 260      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 261     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 262      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 263     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 264      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 265     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 266      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 267     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 268      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 269     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 270      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 271     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 272      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 273     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 274      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 275     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 276      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 277     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 278      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 279     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 280      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 281     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 282      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 283 ]
 284
 285
 286 def sha256_for_file(path):
 287     with open(path, 'rb') as f:
 288         s = hashlib.sha256()
 289         while True:
 290             data = f.read(4096)
 291             if not data:
 292                 break
 293             s.update(data)
 294         return s.hexdigest()
 295
 296
 297 def destroy_current_image(v, serverdir):
 298     global config
 299
 300     # cannot run vagrant without the config in the YAML file
 301     if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')):
 302         v.destroy()
 303     elif options.verbose:
 304         print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!')
 305     if config['vm_provider'] == 'libvirt':
 306         import libvirt
 307         try:
 308             virConnect = libvirt.open('qemu:///system')
 309             virDomain = virConnect.lookupByName(config['domain'])
 310             if virDomain:
 311                 virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE
 312                                         | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA
 313                                         | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM)
 314             storagePool = virConnect.storagePoolLookupByName('default')
 315             if storagePool:
 316                 for vol in storagePool.listAllVolumes():
 317                     vol.delete()
 318         except libvirt.libvirtError as e:
 319             print(e)
 320
 321
 322 def kvm_package(boxfile):
 323     '''
 324     Hack to replace missing `vagrant package` for kvm, based  on the script
 325     `tools/create_box.sh from vagrant-libvirt
 326     '''
 327     import libvirt
 328     virConnect = libvirt.open('qemu:///system')
 329     storagePool = virConnect.storagePoolLookupByName('default')
 330     if storagePool:
 331         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 332         imagepath = vol.path()
 333         # TODO use a libvirt storage pool to ensure the img file is readable
 334         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 335         shutil.copy2(imagepath, 'box.img')
 336         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 337         metadata = """{
 338     "provider": "libvirt",
 339     "format": "qcow2",
 340     "virtual_size": 1000
 341 }
 342 """
 343         vagrantfile = """Vagrant.configure("2") do |config|
 344   config.ssh.username = "vagrant"
 345   config.ssh.password = "vagrant"
 346
 347   config.vm.provider :libvirt do |libvirt|
 348
 349     libvirt.driver = "kvm"
 350     libvirt.host = ""
 351     libvirt.connect_via_ssh = false
 352     libvirt.storage_pool_name = "default"
 353
 354   end
 355 end
 356 """
 357         with open('metadata.json', 'w') as fp:
 358             fp.write(metadata)
 359         with open('Vagrantfile', 'w') as fp:
 360             fp.write(vagrantfile)
 361         with tarfile.open(boxfile, 'w:gz') as tar:
 362             tar.add('metadata.json')
 363             tar.add('Vagrantfile')
 364             tar.add('box.img')
 365         os.remove('metadata.json')
 366         os.remove('Vagrantfile')
 367         os.remove('box.img')
 368
 369
 370 def run_via_vagrant_ssh(v, cmdlist):
 371     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 372         cmd = cmdlist
 373     else:
 374         cmd = ' '.join(cmdlist)
 375     v._run_vagrant_command(['ssh', '-c', cmd])
 376
 377
 378 def main():
 379     global cachedir, cachefiles, config, tail
 380
 381     for srcurl, shasum in cachefiles:
 382         filename = os.path.basename(srcurl)
 383         local_filename = os.path.join(cachedir, filename)
 384
 385         if os.path.exists(local_filename):
 386             local_length = os.path.getsize(local_filename)
 387         else:
 388             local_length = -1
 389
 390         resume_header = {}
 391         download = True
 392
 393         try:
 394             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 395             if r.status_code == 200:
 396                 content_length = int(r.headers.get('content-length'))
 397             else:
 398                 content_length = local_length  # skip the download
 399         except requests.exceptions.RequestException as e:
 400             content_length = local_length  # skip the download
 401             print(e)
 402
 403         if local_length == content_length:
 404             download = False
 405         elif local_length > content_length:
 406             print('deleting corrupt file from cache: ' + local_filename)
 407             os.remove(local_filename)
 408             print("Downloading " + filename + " to cache")
 409         elif local_length > -1 and local_length < content_length:
 410             print("Resuming download of " + local_filename)
 411             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 412         else:
 413             print("Downloading " + filename + " to cache")
 414
 415         if download:
 416             r = requests.get(srcurl, headers=resume_header,
 417                              stream=True, verify=False, allow_redirects=True)
 418             content_length = int(r.headers.get('content-length'))
 419             with open(local_filename, 'ab') as f:
 420                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 421                                           expected_size=(content_length / 65536) + 1):
 422                     if chunk:  # filter out keep-alive new chunks
 423                         f.write(chunk)
 424
 425         v = sha256_for_file(local_filename)
 426         if v == shasum:
 427             print("\t...shasum verified for " + local_filename)
 428         else:
 429             print("Invalid shasum of '" + v + "' detected for " + local_filename)
 430             os.remove(local_filename)
 431             sys.exit(1)
 432
 433     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 434     print("Setting executable bit for " + local_qt_filename)
 435     os.chmod(local_qt_filename, 0o755)
 436
 437     # use VirtualBox software virtualization if hardware is not available,
 438     # like if this is being run in kvm or some other VM platform, like
 439     # http://jenkins.debian.net, the values are 'on' or 'off'
 440     if sys.platform.startswith('darwin'):
 441         # all < 10 year old Macs work, and OSX servers as VM host are very
 442         # rare, but this could also be auto-detected if someone codes it
 443         config['hwvirtex'] = 'on'
 444     elif os.path.exists('/proc/cpuinfo'):
 445         with open('/proc/cpuinfo') as f:
 446             contents = f.read()
 447         if 'vmx' in contents or 'svm' in contents:
 448             config['hwvirtex'] = 'on'
 449
 450     serverdir = os.path.join(os.getcwd(), 'buildserver')
 451     logfilename = os.path.join(serverdir, 'up.log')
 452     if not os.path.exists(logfilename):
 453         open(logfilename, 'a').close()  # create blank file
 454     log_cm = vagrant.make_file_cm(logfilename)
 455     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 456
 457     if options.verbose:
 458         tail = fdroidserver.tail.Tail(logfilename)
 459         tail.start()
 460
 461     if options.clean:
 462         destroy_current_image(v, serverdir)
 463
 464     # Check against the existing Vagrantfile.yaml, and if they differ, we
 465     # need to create a new box:
 466     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 467     writevf = True
 468     if os.path.exists(vf):
 469         print('Halting', serverdir)
 470         v.halt()
 471         with open(vf, 'r', encoding='utf-8') as f:
 472             oldconfig = yaml.load(f)
 473         if config != oldconfig:
 474             print("Server configuration has changed, rebuild from scratch is required")
 475             destroy_current_image(v, serverdir)
 476         else:
 477             print("Re-provisioning existing server")
 478             writevf = False
 479     else:
 480         print("No existing server - building from scratch")
 481     if writevf:
 482         with open(vf, 'w', encoding='utf-8') as f:
 483             yaml.dump(config, f)
 484
 485     if config['vm_provider'] == 'libvirt':
 486         found_basebox = False
 487         needs_mutate = False
 488         for box in v.box_list():
 489             if box.name == config['basebox']:
 490                 found_basebox = True
 491                 if box.provider != 'libvirt':
 492                     needs_mutate = True
 493                 continue
 494         if not found_basebox:
 495             if isinstance(config['baseboxurl'], str):
 496                 baseboxurl = config['baseboxurl']
 497             else:
 498                 baseboxurl = config['baseboxurl'][0]
 499             print('Adding', config['basebox'], 'from', baseboxurl)
 500             v.box_add(config['basebox'], baseboxurl)
 501             needs_mutate = True
 502         if needs_mutate:
 503             print('Converting', config['basebox'], 'to libvirt format')
 504             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 505             print('Removing virtualbox format copy of', config['basebox'])
 506             v.box_remove(config['basebox'], 'virtualbox')
 507
 508     print("Configuring build server VM")
 509     v.up(provision=True)
 510
 511     if config['copy_caches_from_host']:
 512         ssh_config = v.ssh_config()
 513         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 514         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 515         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 516         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 517
 518         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 519             fullpath = os.path.join(os.getenv('HOME'), d)
 520             if os.path.isdir(fullpath):
 521                 # TODO newer versions of vagrant provide `vagrant rsync`
 522                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 523                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 524                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 525                                  fullpath + '/',
 526                                  user + '@' + hostname + ':~/' + d + '/'])
 527
 528         # this file changes every time but should not be cached
 529         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 530         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 531
 532     print("Writing buildserver ID")
 533     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 534                          universal_newlines=True)
 535     buildserverid = p.communicate()[0].strip()
 536     print("...ID is " + buildserverid)
 537     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 538
 539     print("Stopping build server VM")
 540     v.halt()
 541
 542     print("Packaging")
 543     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 544     if os.path.exists(boxfile):
 545         os.remove(boxfile)
 546
 547     if config['vm_provider'] == 'libvirt':
 548         kvm_package(boxfile)
 549     else:
 550         v.package(output=boxfile)
 551
 552     print("Adding box")
 553     v.box_add('buildserver', boxfile, force=True)
 554
 555     os.remove(boxfile)
 556
 557
 558 if __name__ == '__main__':
 559     try:
 560         main()
 561     finally:
 562         if tail is not None:
 563             tail.stop()