makebuildserver

   1 #!/usr/bin/env python3
   2
   3 import os
   4 import pathlib
   5 import re
   6 import requests
   7 import shutil
   8 import stat
   9 import sys
  10 import subprocess
  11 import tarfile
  12 import vagrant
  13 import hashlib
  14 import yaml
  15 import math
  16 import json
  17 from clint.textui import progress
  18 from optparse import OptionParser
  19 import fdroidserver.tail
  20
  21
  22 if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
  23     print('This must be run as ./makebuildserver in fdroidserver.git!')
  24     sys.exit(1)
  25
  26
  27 tail = None
  28
  29 parser = OptionParser()
  30 parser.add_option("-v", "--verbose", action="store_true", default=False,
  31                   help="Spew out even more information than normal")
  32 parser.add_option("-c", "--clean", action="store_true", default=False,
  33                   help="Build from scratch, rather than attempting to update the existing server")
  34 options, args = parser.parse_args()
  35
  36 # set up default config
  37 cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
  38 config = {
  39     'basebox': 'jessie64',
  40     'baseboxurl': [
  41         pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
  42         'https://f-droid.org/jessie64.box',
  43     ],
  44     'debian_mirror': 'http://http.debian.net/debian/',
  45     'apt_package_cache': False,
  46     'copy_caches_from_host': False,
  47     'boot_timeout': 600,
  48     'cachedir': cachedir,
  49     'cpus': 1,
  50     'memory': 1024,
  51     'hwvirtex': 'off',
  52     'vm_provider': 'virtualbox',
  53 }
  54
  55 if os.path.isfile('/usr/bin/systemd-detect-virt'):
  56     try:
  57         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
  58     except subprocess.CalledProcessError as e:
  59         virt = 'none'
  60     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
  61         print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
  62         config['vm_provider'] = 'libvirt'
  63         config['domain'] = 'buildserver_default'
  64     elif virt != 'none':
  65         print('Running in an unsupported VM guest (' + virt + ')!')
  66
  67 # load config file, if present
  68 if os.path.exists('makebuildserver.config.py'):
  69     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
  70 elif os.path.exists('makebs.config.py'):
  71     # this is the old name for the config file
  72     exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
  73 if '__builtins__' in config:
  74     del(config['__builtins__'])  # added by compile/exec
  75
  76 # Update cached files.
  77 cachedir = config['cachedir']
  78 if not os.path.exists(cachedir):
  79     os.makedirs(cachedir, 0o755)
  80
  81 if config['vm_provider'] == 'libvirt':
  82     tmp = cachedir
  83     while tmp != '/':
  84         mode = os.stat(tmp).st_mode
  85         if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
  86             print('ERROR:', tmp, 'will not be accessible to the VM!  To fix, run:')
  87             print('  chmod a+X', tmp)
  88             sys.exit(1)
  89         tmp = os.path.dirname(tmp)
  90
  91 if config['apt_package_cache']:
  92     config['aptcachedir'] = cachedir + '/apt/archives'
  93
  94 cachefiles = [
  95     ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
  96      '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
  97     ('https://dl.google.com/android/repository/android_m2repository_r47.zip',
  98      'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
  99     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
 100      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
 101     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
 102      'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
 103     ('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
 104      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
 105     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
 106      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
 107     ('https://dl.google.com/android/repository/android-2.1_r03.zip',
 108      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
 109     ('https://dl.google.com/android/repository/android-2.2_r03.zip',
 110      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
 111     ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
 112      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
 113     ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
 114      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
 115     ('https://dl.google.com/android/repository/android-3.0_r02.zip',
 116      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
 117     ('https://dl.google.com/android/repository/android-3.1_r03.zip',
 118      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
 119     ('https://dl.google.com/android/repository/android-3.2_r01.zip',
 120      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
 121     ('https://dl.google.com/android/repository/android-14_r04.zip',
 122      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
 123     ('https://dl.google.com/android/repository/android-15_r05.zip',
 124      '5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
 125     ('https://dl.google.com/android/repository/android-16_r05.zip',
 126      'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
 127     ('https://dl.google.com/android/repository/android-17_r03.zip',
 128      'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
 129     ('https://dl.google.com/android/repository/android-18_r03.zip',
 130      '166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
 131     ('https://dl.google.com/android/repository/android-19_r04.zip',
 132      '5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
 133     ('https://dl.google.com/android/repository/android-20_r02.zip',
 134      'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
 135     ('https://dl.google.com/android/repository/android-21_r02.zip',
 136      'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
 137     ('https://dl.google.com/android/repository/android-22_r02.zip',
 138      '45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
 139     ('https://dl.google.com/android/repository/platform-23_r03.zip',
 140      '4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
 141     ('https://dl.google.com/android/repository/platform-24_r02.zip',
 142      'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
 143     ('https://dl.google.com/android/repository/platform-25_r01.zip',
 144      'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
 145     ('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
 146      '4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
 147     ('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
 148      'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
 149     ('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
 150      '0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
 151     ('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
 152      '7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
 153     ('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
 154      '9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
 155     ('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
 156      'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
 157     ('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
 158      '06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
 159     ('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
 160      'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
 161     ('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
 162      '3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
 163     ('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
 164      '296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
 165     ('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
 166      '12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
 167     ('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
 168      'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
 169     ('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
 170      '859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
 171     ('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
 172      '022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
 173     ('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
 174      '29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
 175     ('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
 176      '3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
 177     ('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
 178      '061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
 179     ('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
 180      '91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
 181     ('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
 182      '56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
 183     ('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
 184      'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
 185     ('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
 186      '82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
 187     ('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
 188      'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
 189     ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
 190      'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
 191     ('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
 192      'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
 193     ('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
 194      '924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
 195     ('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
 196      'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
 197     ('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
 198      '74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
 199     ('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
 200      '671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
 201     ('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
 202      '1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
 203     # the binaries that Google uses are here:
 204     # https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
 205     ('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
 206      'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
 207     ('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
 208      'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
 209     ('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
 210      '360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
 211     ('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
 212      'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
 213     ('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
 214      '097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
 215     ('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
 216      '6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
 217     ('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
 218      '07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
 219     ('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
 220      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
 221     ('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
 222      '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
 223     ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
 224      '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
 225     ('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
 226      '420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
 227     ('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
 228      '010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
 229     ('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
 230      'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
 231     ('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
 232      '3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
 233     ('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
 234      '18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
 235     ('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
 236      'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
 237     ('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
 238      'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
 239     ('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
 240      'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
 241     ('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
 242      '66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
 243     ('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
 244      '8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
 245     ('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
 246      'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
 247     ('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
 248      '0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
 249     ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
 250      '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
 251     ('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
 252      'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
 253     ('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
 254      '39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
 255     ('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
 256      'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
 257     ('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
 258      '5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
 259     ('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
 260      '9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
 261     ('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
 262      'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
 263     ('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
 264      '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
 265     ('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
 266      'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
 267     ('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
 268      '0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
 269     ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
 270      '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
 271     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
 272      '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
 273     ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
 274      'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
 275     ('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
 276      'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
 277     ('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
 278      'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
 279     ('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
 280      '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
 281     ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
 282      '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
 283     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
 284      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 285 ]
 286
 287
 288 def sha256_for_file(path):
 289     with open(path, 'rb') as f:
 290         s = hashlib.sha256()
 291         while True:
 292             data = f.read(4096)
 293             if not data:
 294                 break
 295             s.update(data)
 296         return s.hexdigest()
 297
 298
 299 def destroy_current_image(v, serverdir):
 300     global config
 301
 302     # cannot run vagrant without the config in the YAML file
 303     if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')):
 304         v.destroy()
 305     elif options.verbose:
 306         print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!')
 307
 308     try:
 309         shutil.rmtree(os.path.join(serverdir, '.vagrant'))
 310     except Exception as e:
 311         print("could not delete vagrant dir: %s, %s" % (os.path.join(serverdir, '.vagrant'), e))
 312
 313     if config['vm_provider'] == 'libvirt':
 314         import libvirt
 315         try:
 316             virConnect = libvirt.open('qemu:///system')
 317             virDomain = virConnect.lookupByName(config['domain'])
 318             if virDomain:
 319                 virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE
 320                                         | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA
 321                                         | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM)
 322             storagePool = virConnect.storagePoolLookupByName('default')
 323             if storagePool:
 324                 for vol in storagePool.listAllVolumes():
 325                     vol.delete()
 326         except libvirt.libvirtError as e:
 327             print(e)
 328
 329
 330 def kvm_package(boxfile):
 331     '''
 332     Hack to replace missing `vagrant package` for kvm, based  on the script
 333     `tools/create_box.sh from vagrant-libvirt
 334     '''
 335     import libvirt
 336     virConnect = libvirt.open('qemu:///system')
 337     storagePool = virConnect.storagePoolLookupByName('default')
 338     if storagePool:
 339         vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
 340         imagepath = vol.path()
 341         # TODO use a libvirt storage pool to ensure the img file is readable
 342         subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
 343         shutil.copy2(imagepath, 'box.img')
 344         subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
 345         img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True)
 346         img_info = json.loads(img_info_raw.decode('utf-8'))
 347         metadata = {"provider": "libvirt",
 348                     "format": img_info['format'],
 349                     "virtual_size": math.ceil(img_info['virtual-size'] / 1024.**3),
 350                     }
 351
 352         vagrantfile = """Vagrant.configure("2") do |config|
 353   config.ssh.username = "vagrant"
 354   config.ssh.password = "vagrant"
 355
 356   config.vm.provider :libvirt do |libvirt|
 357
 358     libvirt.driver = "kvm"
 359     libvirt.host = ""
 360     libvirt.connect_via_ssh = false
 361     libvirt.storage_pool_name = "default"
 362
 363   end
 364 end
 365 """
 366         with open('metadata.json', 'w') as fp:
 367             fp.write(json.dumps(metadata))
 368         with open('Vagrantfile', 'w') as fp:
 369             fp.write(vagrantfile)
 370         with tarfile.open(boxfile, 'w:gz') as tar:
 371             tar.add('metadata.json')
 372             tar.add('Vagrantfile')
 373             tar.add('box.img')
 374         os.remove('metadata.json')
 375         os.remove('Vagrantfile')
 376         os.remove('box.img')
 377
 378
 379 def run_via_vagrant_ssh(v, cmdlist):
 380     if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
 381         cmd = cmdlist
 382     else:
 383         cmd = ' '.join(cmdlist)
 384     v._run_vagrant_command(['ssh', '-c', cmd])
 385
 386
 387 def main():
 388     global cachedir, cachefiles, config, tail
 389
 390     for srcurl, shasum in cachefiles:
 391         filename = os.path.basename(srcurl)
 392         local_filename = os.path.join(cachedir, filename)
 393
 394         if os.path.exists(local_filename):
 395             local_length = os.path.getsize(local_filename)
 396         else:
 397             local_length = -1
 398
 399         resume_header = {}
 400         download = True
 401
 402         try:
 403             r = requests.head(srcurl, allow_redirects=True, timeout=60)
 404             if r.status_code == 200:
 405                 content_length = int(r.headers.get('content-length'))
 406             else:
 407                 content_length = local_length  # skip the download
 408         except requests.exceptions.RequestException as e:
 409             content_length = local_length  # skip the download
 410             print(e)
 411
 412         if local_length == content_length:
 413             download = False
 414         elif local_length > content_length:
 415             print('deleting corrupt file from cache: ' + local_filename)
 416             os.remove(local_filename)
 417             print("Downloading " + filename + " to cache")
 418         elif local_length > -1 and local_length < content_length:
 419             print("Resuming download of " + local_filename)
 420             resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
 421         else:
 422             print("Downloading " + filename + " to cache")
 423
 424         if download:
 425             r = requests.get(srcurl, headers=resume_header,
 426                              stream=True, verify=False, allow_redirects=True)
 427             content_length = int(r.headers.get('content-length'))
 428             with open(local_filename, 'ab') as f:
 429                 for chunk in progress.bar(r.iter_content(chunk_size=65536),
 430                                           expected_size=(content_length / 65536) + 1):
 431                     if chunk:  # filter out keep-alive new chunks
 432                         f.write(chunk)
 433
 434         v = sha256_for_file(local_filename)
 435         if v == shasum:
 436             print("\t...shasum verified for " + local_filename)
 437         else:
 438             print("Invalid shasum of '" + v + "' detected for " + local_filename)
 439             os.remove(local_filename)
 440             sys.exit(1)
 441
 442     local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
 443     print("Setting executable bit for " + local_qt_filename)
 444     os.chmod(local_qt_filename, 0o755)
 445
 446     # use VirtualBox software virtualization if hardware is not available,
 447     # like if this is being run in kvm or some other VM platform, like
 448     # http://jenkins.debian.net, the values are 'on' or 'off'
 449     if sys.platform.startswith('darwin'):
 450         # all < 10 year old Macs work, and OSX servers as VM host are very
 451         # rare, but this could also be auto-detected if someone codes it
 452         config['hwvirtex'] = 'on'
 453     elif os.path.exists('/proc/cpuinfo'):
 454         with open('/proc/cpuinfo') as f:
 455             contents = f.read()
 456         if 'vmx' in contents or 'svm' in contents:
 457             config['hwvirtex'] = 'on'
 458
 459     serverdir = os.path.join(os.getcwd(), 'buildserver')
 460     logfilename = os.path.join(serverdir, 'up.log')
 461     if not os.path.exists(logfilename):
 462         open(logfilename, 'a').close()  # create blank file
 463     log_cm = vagrant.make_file_cm(logfilename)
 464     v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
 465
 466     if options.verbose:
 467         tail = fdroidserver.tail.Tail(logfilename)
 468         tail.start()
 469
 470     if options.clean:
 471         destroy_current_image(v, serverdir)
 472
 473     # Check against the existing Vagrantfile.yaml, and if they differ, we
 474     # need to create a new box:
 475     vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 476     writevf = True
 477     if os.path.exists(vf):
 478         print('Halting', serverdir)
 479         v.halt()
 480         with open(vf, 'r', encoding='utf-8') as f:
 481             oldconfig = yaml.load(f)
 482         if config != oldconfig:
 483             print("Server configuration has changed, rebuild from scratch is required")
 484             destroy_current_image(v, serverdir)
 485         else:
 486             print("Re-provisioning existing server")
 487             writevf = False
 488     else:
 489         print("No existing server - building from scratch")
 490     if writevf:
 491         with open(vf, 'w', encoding='utf-8') as f:
 492             yaml.dump(config, f)
 493
 494     if config['vm_provider'] == 'libvirt':
 495         found_basebox = False
 496         needs_mutate = False
 497         for box in v.box_list():
 498             if box.name == config['basebox']:
 499                 found_basebox = True
 500                 if box.provider != 'libvirt':
 501                     needs_mutate = True
 502                 continue
 503         if not found_basebox:
 504             if isinstance(config['baseboxurl'], str):
 505                 baseboxurl = config['baseboxurl']
 506             else:
 507                 baseboxurl = config['baseboxurl'][0]
 508             print('Adding', config['basebox'], 'from', baseboxurl)
 509             v.box_add(config['basebox'], baseboxurl)
 510             needs_mutate = True
 511         if needs_mutate:
 512             print('Converting', config['basebox'], 'to libvirt format')
 513             v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
 514             print('Removing virtualbox format copy of', config['basebox'])
 515             v.box_remove(config['basebox'], 'virtualbox')
 516
 517     print("Configuring build server VM")
 518     v.up(provision=True)
 519
 520     if config['copy_caches_from_host']:
 521         ssh_config = v.ssh_config()
 522         user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
 523         hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
 524         port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
 525         key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
 526
 527         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
 528             fullpath = os.path.join(os.getenv('HOME'), d)
 529             if os.path.isdir(fullpath):
 530                 # TODO newer versions of vagrant provide `vagrant rsync`
 531                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
 532                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
 533                                  'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
 534                                  fullpath + '/',
 535                                  user + '@' + hostname + ':~/' + d + '/'])
 536
 537         # this file changes every time but should not be cached
 538         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
 539         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 540
 541     print("Writing buildserver ID")
 542     p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
 543                          universal_newlines=True)
 544     buildserverid = p.communicate()[0].strip()
 545     print("...ID is " + buildserverid)
 546     run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
 547
 548     print("Stopping build server VM")
 549     v.halt()
 550
 551     print("Packaging")
 552     boxfile = os.path.join(os.getcwd(), 'buildserver.box')
 553     if os.path.exists(boxfile):
 554         os.remove(boxfile)
 555
 556     if config['vm_provider'] == 'libvirt':
 557         kvm_package(boxfile)
 558     else:
 559         v.package(output=boxfile)
 560
 561     print("Adding box")
 562     v.box_add('buildserver', boxfile, force=True)
 563
 564     os.remove(boxfile)
 565
 566
 567 if __name__ == '__main__':
 568     try:
 569         main()
 570     finally:
 571         if tail is not None:
 572             tail.stop()