makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 import math
  16 import json
  17 from clint.textui import progress
  18 from optparse import OptionParser
  19 import fdroidserver.tail
  20
  21
  22 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  23     print('This must be run as ./makebuildserver in fdroidserver.git!')
  24     sys.exit(1)
  25
  26
  27 tail = None
  28
  29 parser = OptionParser()
  30 parser.add_option("-v", "--verbose", action="store_true", default=False,
  31                   help="Spew out even more information than normal")
  32 parser.add_option("-c", "--clean", action="store_true", default=False,
  33                   help="Build from scratch, rather than attempting to update the existing server")
  34 options, args = parser.parse_args()
  35
  36 # set up default config
  37 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  38 config = {
  39     'basebox': 'jessie64',
  40     'baseboxurl': [
  41         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  42         'https://f-droid.org/jessie64.box',
  43     ],
  44     'debian_mirror': 'http://http.debian.net/debian/',
  45     'apt_package_cache': False,
  46     'copy_caches_from_host': False,
  47     'boot_timeout': 600,
  48     'cachedir': cachedir,
  49     'cpus': 1,
  50     'memory': 1024,
  51     'hwvirtex': 'off',
  52     'vm_provider': 'virtualbox',
  53 }
  54
  55 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  56     try:
  57         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  58     except subprocess.CalledProcessError as e:
  59         virt = 'none'
  60     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  61         print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  62         config['vm_provider'] = 'libvirt'
  63         config['domain'] = 'buildserver_default'
  64     elif virt != 'none':
  65         print('Running in an unsupported VM guest (' + virt + ')!')
  66
  67 # load config file, if present
  68 if os.path.exists('makebuildserver.config.py'):
  69     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  70 elif os.path.exists('makebs.config.py'):
  71     # this is the old name for the config file
  72     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  73 if '__builtins__' in config:
  74     del(config['__builtins__'])  # added by compile/exec
  75
  76 # Update cached files.
  77 cachedir = config['cachedir']
  78 if not os.path.exists(cachedir):
  79     os.makedirs(cachedir, 0o755)
  80
  81 if config['vm_provider'] == 'libvirt':
  82     tmp = cachedir
  83     while tmp != '/':
  84         mode = os.stat(tmp).st_mode
  85         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
  86             print('ERROR:', tmp, 'will not be accessible to the VM!  To fix, run:')
  87             print('  chmod a+X', tmp)
  88             sys.exit(1)
  89         tmp = os.path.dirname(tmp)
  90
  91 if config['apt_package_cache']:
  92     config['aptcachedir'] = cachedir + '/apt/archives'
  93
  94 cachefiles = [
  95     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
  96      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
  97     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
  98      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
  99     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 100      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 101     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 102      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 103     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 104      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 105     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 106      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 107     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 108      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 109     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 110      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 111     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 112      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 113     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 114      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 115     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 116      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 117     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 118      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 119     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 120      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 121     ('https://dl.google.com/android/repository/android-14_r04.zip',
 122      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 123     ('https://dl.google.com/android/repository/android-15_r05.zip',
 124      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 125     ('https://dl.google.com/android/repository/android-16_r05.zip',
 126      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 127     ('https://dl.google.com/android/repository/android-17_r03.zip',
 128      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 129     ('https://dl.google.com/android/repository/android-18_r03.zip',
 130      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 131     ('https://dl.google.com/android/repository/android-19_r04.zip',
 132      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 133     ('https://dl.google.com/android/repository/android-20_r02.zip',
 134      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 135     ('https://dl.google.com/android/repository/android-21_r02.zip',
 136      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 137     ('https://dl.google.com/android/repository/android-22_r02.zip',
 138      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 139     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 140      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 141     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 142      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 143     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 144      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 145     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 146      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 147     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 148      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 149     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 150      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 151     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 152      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 153     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 154      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 155     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 156      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 157     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 158      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 159     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 160      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 161     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 162      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 163     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 164      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 165     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 166      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 167     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 168      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 169     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 170      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 171     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 172      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 173     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 174      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 175     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 176      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 177     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 178      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 179     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 180      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 181     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 182      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 183     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 184      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 185     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 186      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 187     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 188      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 189     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 190      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 191     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 192      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 193     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 194      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 195     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 196      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 197     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 198      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 199     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 200      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 201     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 202      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 203     # the binaries that Google uses are here:
 204     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 205     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 206      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 207     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 208      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 209     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 210      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 211     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 212      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 213     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 214      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 215     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 216      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 217     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 218      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 219     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 220      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 221     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 222      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 223     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 224      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 225     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 226      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 227     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 228      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 229     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 230      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 231     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 232      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 233     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 234      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 235     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 236      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 237     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 238      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 239     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 240      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 241     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 242      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 243     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 244      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 245     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 246      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 247     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 248      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 249     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 250      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 251     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 252      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 253     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 254      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 255     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 256      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 257     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 258      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 259     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 260      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 261     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 262      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 263     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 264      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 265     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 266      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 267     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 268      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 269     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 270      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 271     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 272      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 273     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 274      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 275     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 276      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 277     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 278      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 279     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 280      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 281     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 282      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 283     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 284      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 285 ]
 286
 287
 288 def sha256_for_file(path):
 289     with open(path, 'rb') as f:
 290         s = hashlib.sha256()
 291         while True:
 292             data = f.read(4096)
 293             if not data:
 294                 break
 295             s.update(data)
 296         return s.hexdigest()
 297
 298
 299 def destroy_current_image(v, serverdir):
 300     global config
 301
 302     # cannot run vagrant without the config in the YAML file
 303     if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')):
 304         v.destroy()
 305     elif options.verbose:
 306         print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!')
 307         subprocess.check_call(['vagrant', 'global-status', '--prune'])
 308
 309     try:
 310         shutil.rmtree(os.path.join(serverdir, '.vagrant'))
 311     except Exception as e:
 312         print("could not delete vagrant dir: %s, %s" % (os.path.join(serverdir, '.vagrant'), e))
 313
 314     if config['vm_provider'] == 'libvirt':
 315         import libvirt
 316         try:
 317             virConnect = libvirt.open('qemu:///system')
 318             virDomain = virConnect.lookupByName(config['domain'])
 319             if virDomain:
 320                 virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE
 321                                         | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA
 322                                         | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM)
 323             storagePool = virConnect.storagePoolLookupByName('default')
 324             if storagePool:
 325                 for vol in storagePool.listAllVolumes():
 326                     vol.delete()
 327         except libvirt.libvirtError as e:
 328             print(e)
 329
 330
 331 def kvm_package(boxfile):
 332     '''
 333     Hack to replace missing `vagrant package` for kvm, based  on the script
 334     `tools/create_box.sh from vagrant-libvirt
 335     '''
 336     import libvirt
 337     virConnect = libvirt.open('qemu:///system')
 338     storagePool = virConnect.storagePoolLookupByName('default')
 339     if storagePool:
 340         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 341         imagepath = vol.path()
 342         # TODO use a libvirt storage pool to ensure the img file is readable
 343         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 344         shutil.copy2(imagepath, 'box.img')
 345         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 346         img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True)
 347         img_info = json.loads(img_info_raw.decode('utf-8'))
 348         metadata = {"provider": "libvirt",
 349                     "format": img_info['format'],
 350                     "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3),
 351                     }
 352
 353         vagrantfile = """Vagrant.configure("2") do |config|
 354   config.ssh.username = "vagrant"
 355   config.ssh.password = "vagrant"
 356
 357   config.vm.provider :libvirt do |libvirt|
 358
 359     libvirt.driver = "kvm"
 360     libvirt.host = ""
 361     libvirt.connect_via_ssh = false
 362     libvirt.storage_pool_name = "default"
 363
 364   end
 365 end
 366 """
 367         with open('metadata.json', 'w') as fp:
 368             fp.write(json.dumps(metadata))
 369         with open('Vagrantfile', 'w') as fp:
 370             fp.write(vagrantfile)
 371         with tarfile.open(boxfile, 'w:gz') as tar:
 372             tar.add('metadata.json')
 373             tar.add('Vagrantfile')
 374             tar.add('box.img')
 375         os.remove('metadata.json')
 376         os.remove('Vagrantfile')
 377         os.remove('box.img')
 378
 379
 380 def run_via_vagrant_ssh(v, cmdlist):
 381     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 382         cmd = cmdlist
 383     else:
 384         cmd = ' '.join(cmdlist)
 385     v._run_vagrant_command(['ssh', '-c', cmd])
 386
 387
 388 def main():
 389     global cachedir, cachefiles, config, tail
 390
 391     for srcurl, shasum in cachefiles:
 392         filename = os.path.basename(srcurl)
 393         local_filename = os.path.join(cachedir, filename)
 394
 395         if os.path.exists(local_filename):
 396             local_length = os.path.getsize(local_filename)
 397         else:
 398             local_length = -1
 399
 400         resume_header = {}
 401         download = True
 402
 403         try:
 404             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 405             if r.status_code == 200:
 406                 content_length = int(r.headers.get('content-length'))
 407             else:
 408                 content_length = local_length  # skip the download
 409         except requests.exceptions.RequestException as e:
 410             content_length = local_length  # skip the download
 411             print(e)
 412
 413         if local_length == content_length:
 414             download = False
 415         elif local_length > content_length:
 416             print('deleting corrupt file from cache: ' + local_filename)
 417             os.remove(local_filename)
 418             print("Downloading " + filename + " to cache")
 419         elif local_length > -1 and local_length < content_length:
 420             print("Resuming download of " + local_filename)
 421             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 422         else:
 423             print("Downloading " + filename + " to cache")
 424
 425         if download:
 426             r = requests.get(srcurl, headers=resume_header,
 427                              stream=True, verify=False, allow_redirects=True)
 428             content_length = int(r.headers.get('content-length'))
 429             with open(local_filename, 'ab') as f:
 430                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 431                                           expected_size=(content_length / 65536) + 1):
 432                     if chunk:  # filter out keep-alive new chunks
 433                         f.write(chunk)
 434
 435         v = sha256_for_file(local_filename)
 436         if v == shasum:
 437             print("\t...shasum verified for " + local_filename)
 438         else:
 439             print("Invalid shasum of '" + v + "' detected for " + local_filename)
 440             os.remove(local_filename)
 441             sys.exit(1)
 442
 443     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 444     print("Setting executable bit for " + local_qt_filename)
 445     os.chmod(local_qt_filename, 0o755)
 446
 447     # use VirtualBox software virtualization if hardware is not available,
 448     # like if this is being run in kvm or some other VM platform, like
 449     # http://jenkins.debian.net, the values are 'on' or 'off'
 450     if sys.platform.startswith('darwin'):
 451         # all < 10 year old Macs work, and OSX servers as VM host are very
 452         # rare, but this could also be auto-detected if someone codes it
 453         config['hwvirtex'] = 'on'
 454     elif os.path.exists('/proc/cpuinfo'):
 455         with open('/proc/cpuinfo') as f:
 456             contents = f.read()
 457         if 'vmx' in contents or 'svm' in contents:
 458             config['hwvirtex'] = 'on'
 459
 460     serverdir = os.path.join(os.getcwd(), 'buildserver')
 461     logfilename = os.path.join(serverdir, 'up.log')
 462     if not os.path.exists(logfilename):
 463         open(logfilename, 'a').close()  # create blank file
 464     log_cm = vagrant.make_file_cm(logfilename)
 465     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 466
 467     if options.verbose:
 468         tail = fdroidserver.tail.Tail(logfilename)
 469         tail.start()
 470
 471     if options.clean:
 472         destroy_current_image(v, serverdir)
 473
 474     # Check against the existing Vagrantfile.yaml, and if they differ, we
 475     # need to create a new box:
 476     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 477     writevf = True
 478     if os.path.exists(vf):
 479         print('Halting', serverdir)
 480         v.halt()
 481         with open(vf, 'r', encoding='utf-8') as f:
 482             oldconfig = yaml.load(f)
 483         if config != oldconfig:
 484             print("Server configuration has changed, rebuild from scratch is required")
 485             destroy_current_image(v, serverdir)
 486         else:
 487             print("Re-provisioning existing server")
 488             writevf = False
 489     else:
 490         print("No existing server - building from scratch")
 491     if writevf:
 492         with open(vf, 'w', encoding='utf-8') as f:
 493             yaml.dump(config, f)
 494
 495     if config['vm_provider'] == 'libvirt':
 496         found_basebox = False
 497         needs_mutate = False
 498         for box in v.box_list():
 499             if box.name == config['basebox']:
 500                 found_basebox = True
 501                 if box.provider != 'libvirt':
 502                     needs_mutate = True
 503                 continue
 504         if not found_basebox:
 505             if isinstance(config['baseboxurl'], str):
 506                 baseboxurl = config['baseboxurl']
 507             else:
 508                 baseboxurl = config['baseboxurl'][0]
 509             print('Adding', config['basebox'], 'from', baseboxurl)
 510             v.box_add(config['basebox'], baseboxurl)
 511             needs_mutate = True
 512         if needs_mutate:
 513             print('Converting', config['basebox'], 'to libvirt format')
 514             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 515             print('Removing virtualbox format copy of', config['basebox'])
 516             v.box_remove(config['basebox'], 'virtualbox')
 517
 518     print("Configuring build server VM")
 519     v.up(provision=True)
 520
 521     if config['copy_caches_from_host']:
 522         ssh_config = v.ssh_config()
 523         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 524         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 525         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 526         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 527
 528         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 529             fullpath = os.path.join(os.getenv('HOME'), d)
 530             if os.path.isdir(fullpath):
 531                 # TODO newer versions of vagrant provide `vagrant rsync`
 532                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 533                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 534                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 535                                  fullpath + '/',
 536                                  user + '@' + hostname + ':~/' + d + '/'])
 537
 538         # this file changes every time but should not be cached
 539         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 540         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 541
 542     print("Writing buildserver ID")
 543     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 544                          universal_newlines=True)
 545     buildserverid = p.communicate()[0].strip()
 546     print("...ID is " + buildserverid)
 547     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 548
 549     print("Stopping build server VM")
 550     v.halt()
 551
 552     print("Packaging")
 553     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 554     if os.path.exists(boxfile):
 555         os.remove(boxfile)
 556
 557     if config['vm_provider'] == 'libvirt':
 558         kvm_package(boxfile)
 559     else:
 560         v.package(output=boxfile)
 561
 562     print("Adding box")
 563     v.box_add('buildserver', boxfile, force=True)
 564
 565     os.remove(boxfile)
 566
 567
 568 if __name__ == '__main__':
 569     try:
 570         main()
 571     finally:
 572         if tail is not None:
 573             tail.stop()