import pathlib
import re
import requests
-import shutil
import stat
import sys
+import shutil
import subprocess
-import tarfile
import vagrant
import hashlib
import yaml
-import math
import json
-import time
import logging
from clint.textui import progress
from optparse import OptionParser
parser.add_option('--skip-cache-update', action="store_true", default=False,
help="""Skip downloading and checking cache."""
"""This assumes that the cache is already downloaded completely.""")
+parser.add_option('--keep-box-file', action="store_true", default=False,
+ help="""Box file will not be deleted after adding it to box storage"""
+ """ (KVM-only).""")
options, args = parser.parse_args()
logger = logging.getLogger('fdroidserver-makebuildserver')
if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
config['vm_provider'] = 'libvirt'
- config['domain'] = 'buildserver_default'
elif virt != 'none':
logger.info('Running in an unsupported VM guest (%s)!', virt)
-logger.debug('deceted virt: %s', virt)
+ logger.debug('detected virt: %s', virt)
# load config file, if present
if os.path.exists('makebuildserver.config.py'):
if config['apt_package_cache']:
config['aptcachedir'] = cachedir + '/apt/archives'
logger.debug('aptcachedir is set to %s', config['aptcachedir'])
+ aptcachelock = os.path.join(config['aptcachedir'], 'lock')
+ if os.path.isfile(aptcachelock):
+ logger.info('apt cache dir is locked, removing lock')
+ os.remove(aptcachelock)
+ aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
+ if os.path.isdir(aptcachepartial):
+ logger.info('removing partial downloads from apt cache dir')
+ shutil.rmtree(aptcachepartial)
cachefiles = [
- ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
- '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
+ ('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
+ '577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
('https://dl.google.com/android/repository/android_m2repository_r47.zip',
'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
('https://dl.google.com/android/repository/platform-25_r01.zip',
'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
+ ('https://dl.google.com/android/repository/platform-26_r01.zip',
+ '44e7eca5923320db1abb422aa7a3d0ac9ee199a379af6f5c7603e714d7717561'),
('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
'4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
'671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
'1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
+ ('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
+ '152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
+ ('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
+ '7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
# the binaries that Google uses are here:
# https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
'0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
+ ('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
+ '8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
+ ('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
+ '56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
+ ('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
+ 'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
+ ('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
+ '79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
+ ('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
+ 'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
- ('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
- '3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
+ ('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
+ '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
+ ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
+ 'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
]
return s.hexdigest()
-def kvm_package(boxfile):
- '''
- Hack to replace missing `vagrant package` for kvm, based on the script
- `tools/create_box.sh from vagrant-libvirt
- '''
- import libvirt
- virConnect = libvirt.open('qemu:///system')
- storagePool = virConnect.storagePoolLookupByName('default')
- if storagePool:
- vol = storagePool.storageVolLookupByName(config['domain'] + '.img')
- imagepath = vol.path()
- # TODO use a libvirt storage pool to ensure the img file is readable
- subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
- shutil.copy2(imagepath, 'box.img')
- subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
- img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True)
- img_info = json.loads(img_info_raw.decode('utf-8'))
- metadata = {"provider": "libvirt",
- "format": img_info['format'],
- "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3),
- }
-
- vagrantfile = """Vagrant.configure("2") do |config|
- config.ssh.username = "vagrant"
- config.ssh.password = "vagrant"
-
- config.vm.provider :libvirt do |libvirt|
-
- libvirt.driver = "kvm"
- libvirt.host = ""
- libvirt.connect_via_ssh = false
- libvirt.storage_pool_name = "default"
-
- end
-end
-"""
- with open('metadata.json', 'w') as fp:
- fp.write(json.dumps(metadata))
- with open('Vagrantfile', 'w') as fp:
- fp.write(vagrantfile)
- with tarfile.open(boxfile, 'w:gz') as tar:
- tar.add('metadata.json')
- tar.add('Vagrantfile')
- tar.add('box.img')
- os.remove('metadata.json')
- os.remove('Vagrantfile')
- os.remove('box.img')
-
-
def run_via_vagrant_ssh(v, cmdlist):
if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
cmd = cmdlist
if download:
r = requests.get(srcurl, headers=resume_header,
- stream=True, verify=False, allow_redirects=True)
+ stream=True, allow_redirects=True)
content_length = int(r.headers.get('content-length'))
with open(local_filename, 'ab') as f:
for chunk in progress.bar(r.iter_content(chunk_size=65536),
sys.exit(1)
-def debug_log_vagrant_vm(vm_dir, vm_name):
+def debug_log_vagrant_vm(vm_dir, config):
if options.verbosity >= 3:
_vagrant_dir = os.path.join(vm_dir, '.vagrant')
logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
subprocess.call(['vagrant', 'status'], cwd=vm_dir)
logger.debug('> vagrant box list')
subprocess.call(['vagrant', 'box', 'list'])
- logger.debug('> virsh -c qmeu:///system list --all')
- subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
- logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name)
- subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name])
+ if config['vm_provider'] == 'libvirt':
+ logger.debug('> virsh -c qmeu:///system list --all')
+ subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
+ domain = 'buildserver_default'
+ logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
+ subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
def main():
tail = fdroidserver.tail.Tail(logfilename)
tail.start()
- vm = fdroidserver.vmtools.get_build_vm(serverdir)
+ vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
if options.clean:
vm.destroy()
v.box_remove(config['basebox'], 'virtualbox')
logger.info("Configuring build server VM")
- debug_log_vagrant_vm(serverdir, config['domain'])
+ debug_log_vagrant_vm(serverdir, config)
try:
- try:
- v.up(provision=True)
- except subprocess.CalledProcessError as e:
- v.up(provision=True)
- except subprocess.CalledProcessError as e:
- debug_log_vagrant_vm(serverdir, config['domain'])
- logger.critical('could not bring buildserver vm up. %s', e)
+ v.up(provision=True)
+ except fdroidserver.vmtools.FDroidBuildVmException as e:
+ debug_log_vagrant_vm(serverdir, config)
+ logger.exception('could not bring buildserver vm up. %s', e)
sys.exit(1)
if config['copy_caches_from_host']:
if os.path.exists(boxfile):
os.remove(boxfile)
- if config['vm_provider'] == 'libvirt':
- kvm_package(boxfile)
- else:
- v.package(output=boxfile)
+ vm.package(output=boxfile)
logger.info("Adding box")
- v.box_add('buildserver', boxfile, force=True)
+ vm.box_add('buildserver', boxfile, force=True)
- os.remove(boxfile)
+ if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
+ logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
+ sys.exit(1)
+
+ if not options.keep_box_file:
+ logger.debug('box added to vagrant, ' +
+ 'removing generated box file \'%s\'',
+ boxfile)
+ os.remove(boxfile)
if __name__ == '__main__':
~ianmdlvl / fdroidserver.git / blobdiff