From: Michael Pöhn Date: Sun, 26 Mar 2017 00:41:39 +0000 (+0100) Subject: overhauled and moved destroying builder vm to vmtools.py X-Git-Tag: 0.8~56^2~25 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=92fada803e5aa2ecf7516398bee011666f395521;p=fdroidserver.git overhauled and moved destroying builder vm to vmtools.py --- diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 135e6054..de8c3377 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1110,7 +1110,7 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, this is the 'unsigned' directory. :param repo_dir: The repo directory - used for checking if the build is necessary. - :paaram also_check_dir: An additional location for checking if the build + :param also_check_dir: An additional location for checking if the build is necessary (usually the archive repo) :param test: True if building in test mode, in which case the build will always happen, even if the output already exists. In test mode, the diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py new file mode 100644 index 00000000..a5c0f5ba --- /dev/null +++ b/fdroidserver/vmtools.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# +# vmtools.py - part of the FDroid server tools +# Copyright (C) 2017 Michael Poehn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from os.path import isdir, isfile, join as joinpath, basename, abspath +import time +import shutil +import vagrant +import subprocess +from .common import FDroidException +from logging import getLogger + +logger = getLogger('fdroidserver-vmtools') + + +def get_build_vm(srvdir, provider=None): + """Factory function for getting FDroidBuildVm instances. + + This function tries to figure out what hypervisor should be used + and creates an object for controlling a build VM. + + :param srvdir: path to a directory which contains a Vagrantfile + :param provider: optionally this parameter allows specifiying an + spesific vagrant provider. + :returns: FDroidBuildVm instance. + """ + abssrvdir = abspath(srvdir) + if provider: + if provider == 'libvirt': + logger.debug('build vm provider \'libvirt\' selected') + return LibvirtBuildVm(abssrvdir) + elif provider == 'virtualbox': + logger.debug('build vm provider \'virtualbox\' selected') + return VirtualboxBuildVm(abssrvdir) + else: + logger.warn('unsupported provider \'%s\' requested', provider) + has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant', + 'machines', 'default', 'libvirt')) + has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant', + 'machines', 'default', 'libvirt')) + if has_libvirt_machine and has_vbox_machine: + logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'') + return VirtualboxBuildVm(abssrvdir) + elif has_libvirt_machine: + logger.debug('build vm provider lookup found \'libvirt\'') + return LibvirtBuildVm(abssrvdir) + elif has_vbox_machine: + logger.debug('build vm provider lookup found \'virtualbox\'') + return VirtualboxBuildVm(abssrvdir) + + logger.info('build vm provider lookup could not determine provider, defaulting to \'virtualbox\'') + return VirtualboxBuildVm(abssrvdir) + + +class FDroidBuildVmException(FDroidException): + pass + + +class FDroidBuildVm(): + """Abstract base class for working with FDroids build-servers. + + Use the factory method `fdroidserver.vmtools.get_build_vm()` for + getting correct instances of this class. + + This is intended to be a hypervisor independant, fault tolerant + wrapper around the vagrant functions we use. + """ + + def __init__(self, srvdir): + """Create new server class. + """ + self.srvdir = srvdir + self.srvname = basename(srvdir) + '_default' + self.vgrntfile = joinpath(srvdir, 'Vagrantfile') + if not isdir(srvdir): + raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir)) + if not isfile(self.vgrntfile): + raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile)) + self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm) + + def isUpAndRunning(self): + raise NotImplementedError('TODO implement this') + + def up(self, provision=True): + try: + self.vgrnt.up(provision=provision) + except subprocess.CalledProcessError as e: + logger.info('could not bring vm up: %s', e) + + def destroy(self): + """Remove every trace of this VM from the system. + + This includes deleting: + * hypervisor specific definitions + * vagrant state informations (eg. `.vagrant` folder) + * images related to this vm + """ + try: + self.vgrnt.destroy() + logger.debug('vagrant destroy completed') + except subprocess.CalledProcessError as e: + logger.debug('vagrant destroy failed: %s', e) + vgrntdir = joinpath(self.srvdir, '.vagrant') + try: + shutil.rmtree(vgrntdir) + logger.debug('deleted vagrant dir: %s', vgrntdir) + except Exception as e: + logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e) + try: + subprocess.check_call(['vagrant', 'global-status', '--prune']) + except subprocess.CalledProcessError as e: + logger.debug('pruning global vagrant status failed: %s', e) + + +class LibvirtBuildVm(FDroidBuildVm): + def __init__(self, srvdir): + super().__init__(srvdir) + import libvirt + + try: + self.conn = libvirt.open('qemu:///system') + except libvirt.libvirtError as e: + logger.critical('could not connect to libvirtd: %s', e) + + def destroy(self): + + super().destroy() + + # resorting to virsh instead of libvirt python bindings, because + # this is way more easy and therefore fault tolerant. + # (eg. lookupByName only works on running VMs) + try: + logger.debug('virsh -c qemu:///system destroy', self.srvname) + subprocess.check_call(('virsh', '-c', 'qemu:///system', 'destroy')) + logger.info("...waiting a sec...") + time.sleep(10) + except subprocess.CalledProcessError as e: + logger.info("could not force libvirt domain '%s' off: %s", self.srvname, e) + try: + # libvirt python bindings do not support all flags required + # for undefining domains correctly. + logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', self.srvname) + subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + logger.info("...waiting a sec...") + time.sleep(10) + except subprocess.CalledProcessError as e: + logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e) + + +class VirtualboxBuildVm(FDroidBuildVm): + pass diff --git a/makebuildserver b/makebuildserver index 3b14bd7d..f8b892e6 100755 --- a/makebuildserver +++ b/makebuildserver @@ -19,6 +19,7 @@ import logging from clint.textui import progress from optparse import OptionParser import fdroidserver.tail +import fdroidserver.vmtools parser = OptionParser() @@ -32,7 +33,7 @@ parser.add_option('--skip-cache-update', action="store_true", default=False, """This assumes that the cache is already downloaded completely.""") options, args = parser.parse_args() -logger = logging.getLogger('fdroid-makebuildserver') +logger = logging.getLogger('fdroidserver-makebuildserver') if options.verbosity >= 2: logging.basicConfig(format='%(message)s', level=logging.DEBUG) logger.setLevel(logging.DEBUG) @@ -320,55 +321,6 @@ def sha256_for_file(path): return s.hexdigest() -def destroy_current_image(v, serverdir): - global config - - logger.info('destroying buildserver vm, removing images and vagrant-configs...') - - try: - v.destroy() - logger.debug('vagrant destroy completed') - except subprocess.CalledProcessError as e: - logger.debug('vagrant destroy failed: %s', e) - try: - subprocess.check_call(['vagrant', 'global-status', '--prune']) - except subprocess.CalledProcessError as e: - logger.debug('pruning global vagrant status failed: %s', e) - - try: - shutil.rmtree(os.path.join(serverdir, '.vagrant')) - except Exception as e: - logger.debug("could not delete vagrant dir: %s, %s", os.path.join(serverdir, '.vagrant'), e) - - if config['vm_provider'] == 'libvirt': - import libvirt - try: - conn = libvirt.open('qemu:///system') - try: - dom = conn.lookupByName(config['domain']) - try: - logger.debug('virsh -c qemu:///system destroy %s', config['domain']) - subprocess.check_call(['virsh', '-c', 'qemu:///system', 'destroy', config['domain']]) - logger.info("...waiting a sec...") - time.sleep(10) - except subprocess.CalledProcessError as e: - logger.info("could not force libvirt domain '%s' off: %s", config['domain'], e) - try: - # libvirt python bindings do not support all flags required - # for undefining domains correctly. - logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', config['domain']) - subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', config['domain'], '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) - logger.info("...waiting a sec...") - time.sleep(10) - except subprocess.CalledProcessError as e: - logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e) - except libvirt.libvirtError as e: - logger.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e) - except libvirt.libvirtError as e: - logger.critical('could not connect to libvirtd: %s', e) - sys.exit(1) - - def kvm_package(boxfile): ''' Hack to replace missing `vagrant package` for kvm, based on the script @@ -532,8 +484,9 @@ def main(): tail = fdroidserver.tail.Tail(logfilename) tail.start() + vm = fdroidserver.vmtools.get_build_vm(serverdir) if options.clean: - destroy_current_image(v, serverdir) + vm.destroy() # Check against the existing Vagrantfile.yaml, and if they differ, we # need to create a new box: @@ -546,7 +499,7 @@ def main(): oldconfig = yaml.load(f) if config != oldconfig: logger.info("Server configuration has changed, rebuild from scratch is required") - destroy_current_image(v, serverdir) + vm.destroy() else: logger.info("Re-provisioning existing server") writevf = False