From: Hans-Christoph Steiner Date: Mon, 22 May 2017 15:12:34 +0000 (+0200) Subject: refactored kvm_package to vmtools X-Git-Tag: 0.8~56^2~20 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=01b64738239b16d2a4d99c4030cca3a9e775c54d;p=fdroidserver.git refactored kvm_package to vmtools --- diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 05b176b9..c64e9994 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -16,7 +16,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from os import remove as rmfile from os.path import isdir, isfile, join as joinpath, basename, abspath +import math +import json +import tarfile import time import shutil import vagrant @@ -125,6 +129,9 @@ class FDroidBuildVm(): except subprocess.CalledProcessError as e: logger.debug('pruning global vagrant status failed: %s', e) + def package(self, output=None, keep_box_file=None): + self.vgrnt.package(output=output) + class LibvirtBuildVm(FDroidBuildVm): def __init__(self, srvdir): @@ -160,6 +167,69 @@ class LibvirtBuildVm(FDroidBuildVm): except subprocess.CalledProcessError as e: logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e) + def package(self, output=None, keep_box_file=False): + if not output: + output = "buildserver.box" + logger.debug('no output name set for packaging \'%s\',' + + 'defaulting to %s', self.srvname, output) + import libvirt + virConnect = libvirt.open('qemu:///system') + storagePool = virConnect.storagePoolLookupByName('default') + if storagePool: + + if isfile('metadata.json'): + rmfile('metadata.json') + if isfile('Vagrantfile'): + rmfile('Vagrantfile') + if isfile('box.img'): + rmfile('box.img') + + vol = storagePool.storageVolLookupByName(self.srvname + '.img') + imagepath = vol.path() + # TODO use a libvirt storage pool to ensure the img file is readable + subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + shutil.copy2(imagepath, 'box.img') + subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) + img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True) + img_info = json.loads(img_info_raw.decode('utf-8')) + metadata = {"provider": "libvirt", + "format": img_info['format'], + "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)) + 1, + } + + vagrantfile = """Vagrant.configure("2") do |config| + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + + config.vm.provider :libvirt do |libvirt| + + libvirt.driver = "kvm" + libvirt.host = "" + libvirt.connect_via_ssh = false + libvirt.storage_pool_name = "default" + + end +end +""" + with open('metadata.json', 'w') as fp: + fp.write(json.dumps(metadata)) + with open('Vagrantfile', 'w') as fp: + fp.write(vagrantfile) + with tarfile.open(output, 'w:gz') as tar: + tar.add('metadata.json') + tar.add('Vagrantfile') + tar.add('box.img') + + if not keep_box_file: + logger.debug('box packaging complete, removing temporary files.') + rmfile('metadata.json') + rmfile('Vagrantfile') + rmfile('box.img') + + else: + logger.warn('could not connect to storage-pool \'default\',' + + 'skipping packaging buildserver box') + class VirtualboxBuildVm(FDroidBuildVm): pass diff --git a/makebuildserver b/makebuildserver index 9b9ad5ad..469e8f1c 100755 --- a/makebuildserver +++ b/makebuildserver @@ -4,15 +4,12 @@ import os import pathlib import re import requests -import shutil import stat import sys import subprocess -import tarfile import vagrant import hashlib import yaml -import math import json import logging from clint.textui import progress @@ -322,70 +319,6 @@ def sha256_for_file(path): return s.hexdigest() -def kvm_package(boxfile): - ''' - Hack to replace missing `vagrant package` for kvm, based on the script - `tools/create_box.sh from vagrant-libvirt - ''' - import libvirt - virConnect = libvirt.open('qemu:///system') - storagePool = virConnect.storagePoolLookupByName('default') - if storagePool: - - if os.path.isfile('metadata.json'): - os.remove('metadata.json') - if os.path.isfile('Vagrantfile'): - os.remove('Vagrantfile') - if os.path.isfile('box.img'): - os.remove('box.img') - - vol = storagePool.storageVolLookupByName(config['domain'] + '.img') - imagepath = vol.path() - # TODO use a libvirt storage pool to ensure the img file is readable - subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) - shutil.copy2(imagepath, 'box.img') - subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) - img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True) - img_info = json.loads(img_info_raw.decode('utf-8')) - metadata = {"provider": "libvirt", - "format": img_info['format'], - "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3), - } - - vagrantfile = """Vagrant.configure("2") do |config| - config.ssh.username = "vagrant" - config.ssh.password = "vagrant" - - config.vm.provider :libvirt do |libvirt| - - libvirt.driver = "kvm" - libvirt.host = "" - libvirt.connect_via_ssh = false - libvirt.storage_pool_name = "default" - - end -end -""" - - with open('metadata.json', 'w') as fp: - fp.write(json.dumps(metadata)) - with open('Vagrantfile', 'w') as fp: - fp.write(vagrantfile) - with tarfile.open(boxfile, 'w:gz') as tar: - tar.add('metadata.json') - tar.add('Vagrantfile') - tar.add('box.img') - if not options.keep_box_file: - logger.debug('box packaging complete, removing temporary files.') - os.remove('metadata.json') - os.remove('Vagrantfile') - os.remove('box.img') - - else: - logger.warn('could not connect to storage-pool \'default\',' + - 'skipping packaging buildserver box') - - def run_via_vagrant_ssh(v, cmdlist): if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)): cmd = cmdlist @@ -551,10 +484,7 @@ def main(): logger.info("Configuring build server VM") debug_log_vagrant_vm(serverdir, config['domain']) try: - try: - v.up(provision=True) - except subprocess.CalledProcessError as e: - v.up(provision=True) + v.up(provision=True) except subprocess.CalledProcessError as e: debug_log_vagrant_vm(serverdir, config['domain']) logger.critical('could not bring buildserver vm up. %s', e) @@ -595,14 +525,15 @@ def main(): if os.path.exists(boxfile): os.remove(boxfile) - if config['vm_provider'] == 'libvirt': - kvm_package(boxfile) - else: - v.package(output=boxfile) + vm.package(output=boxfile, debug_keep_box_file=options.debug_keep_box_file) logger.info("Adding box") v.box_add('buildserver', boxfile, force=True) + if not 'buildserver' in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'): + logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile) + sys.exit(1) + if not options.keep_box_file: logger.debug('box added to vagrant, ' + 'removing generated box file \'%s\'',