From 4b03c3d42dd91373367bc5af31334e9742f3ecfc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 27 Sep 2016 08:49:32 +0200 Subject: [PATCH] buildserver: replace custom code with python-vagrant I ran into some annoying issues with UTF-8 output in the vagrant logs, and it was hard to solve. So I switched to using python-vagrant, which handles it all for us. Its been around since 2012, has a number of contributors, and is still actively maintained, so it seems like a good bet. I also packaged it for Debian, including a backport in jessie-backports. On Debian/jessie, do `apt-get install python3-vagrant/jessie-backports` --- makebuildserver | 98 +++++++++++++------------------------------------ setup.py | 1 + 2 files changed, 26 insertions(+), 73 deletions(-) diff --git a/makebuildserver b/makebuildserver index 523a232a..4c196ff0 100755 --- a/makebuildserver +++ b/makebuildserver @@ -6,7 +6,7 @@ import requests import stat import sys import subprocess -import time +import vagrant import hashlib import yaml from clint.textui import progress @@ -18,34 +18,7 @@ if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'): sys.exit(1) -def vagrant(params, cwd=None, printout=False): - """Run vagrant. - - :param: list of parameters to pass to vagrant - :cwd: directory to run in, or None for current directory - :printout: True to print output in realtime, False to just - return it - :returns: (ret, out) where ret is the return code, and out - is the stdout (and stderr) from vagrant - """ - p = subprocess.Popen(['vagrant'] + params, cwd=cwd, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True) - out = '' - if printout: - while True: - line = p.stdout.readline() - if len(line) == 0: - break - print(line.rstrip()) - out += line - p.wait() - else: - out = p.communicate()[0] - return (p.returncode, out) - - -boxfile = 'buildserver.box' +boxfile = os.path.join(os.getcwd(), 'buildserver.box') serverdir = 'buildserver' parser = OptionParser() @@ -96,12 +69,6 @@ if '__builtins__' in config: if os.path.exists(boxfile): os.remove(boxfile) -if options.clean: - vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose) - if config['vm_provider'] == 'libvirt': - subprocess.call(['virsh', 'undefine', 'buildserver_default']) - subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img']) - # Update cached files. cachedir = config['cachedir'] if not os.path.exists(cachedir): @@ -394,18 +361,28 @@ elif os.path.exists('/proc/cpuinfo'): if 'vmx' in contents or 'svm' in contents: config['hwvirtex'] = 'on' +logfilename = os.path.join(serverdir, 'up.log') +log_cm = vagrant.make_file_cm(logfilename) +v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm) + +if options.clean: + v.destroy() + if config['vm_provider'] == 'libvirt': + subprocess.call(['virsh', 'undefine', 'buildserver_default']) + subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img']) + # Check against the existing Vagrantfile.yaml, and if they differ, we # need to create a new box: vf = os.path.join(serverdir, 'Vagrantfile.yaml') writevf = True if os.path.exists(vf): print('Halting', serverdir) - vagrant(['halt'], serverdir) + v.halt() with open(vf, 'r', encoding='utf-8') as f: oldconfig = yaml.load(f) if config != oldconfig: print("Server configuration has changed, rebuild from scratch is required") - vagrant(['destroy', '-f'], serverdir) + v.destroy() else: print("Re-provisioning existing server") writevf = False @@ -416,13 +393,12 @@ if writevf: yaml.dump(config, f) if config['vm_provider'] == 'libvirt': - returncode, out = vagrant(['box', 'list'], serverdir, printout=options.verbose) found_basebox = False needs_mutate = False - for line in out.splitlines(): - if line.startswith(config['basebox']): + for box in v.box_list(): + if box.name == config['basebox']: found_basebox = True - if line.split('(')[1].split(',')[0] != 'libvirt': + if box.provider != 'libvirt': needs_mutate = True continue if not found_basebox: @@ -431,24 +407,16 @@ if config['vm_provider'] == 'libvirt': else: baseboxurl = config['baseboxurl'][0] print('Adding', config['basebox'], 'from', baseboxurl) - vagrant(['box', 'add', '--name', config['basebox'], baseboxurl], - serverdir, printout=options.verbose) + v.box_add(config['basebox'], baseboxurl) needs_mutate = True if needs_mutate: print('Converting', config['basebox'], 'to libvirt format') - vagrant(['mutate', config['basebox'], 'libvirt'], - serverdir, printout=options.verbose) + v._call_vagrant_command(['mutate', config['basebox'], 'libvirt']) print('Removing virtualbox format copy of', config['basebox']) - vagrant(['box', 'remove', '--provider', 'virtualbox', config['basebox']], - serverdir, printout=options.verbose) + v.box_remove(config['basebox'], 'virtualbox') print("Configuring build server VM") -returncode, out = vagrant(['up', '--provision'], serverdir, printout=True) -with open(os.path.join(serverdir, 'up.log'), 'w') as log: - log.write(out) -if returncode != 0: - print("Failed to configure server") - sys.exit(1) +v.up(provision=True) print("Writing buildserver ID") p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, @@ -461,28 +429,12 @@ subprocess.call( cwd=serverdir) print("Stopping build server VM") -vagrant(['halt'], serverdir) - -print("Waiting for build server VM to be finished") -ready = False -while not ready: - time.sleep(2) - returncode, out = vagrant(['status'], serverdir) - if returncode != 0: - print("Error while checking status") - sys.exit(1) - for line in out.splitlines(): - if line.startswith("default"): - if line.find("poweroff") != -1 or line.find("shutoff") != 1: - ready = True - else: - print("Status: " + line) +v.halt() print("Packaging") -vagrant(['package', '--output', os.path.join('..', boxfile)], serverdir, - printout=options.verbose) +v.package(output=boxfile) + print("Adding box") -vagrant(['box', 'add', 'buildserver', boxfile, '-f'], - printout=options.verbose) +v.box_add('buildserver', boxfile, force=True) os.remove(boxfile) diff --git a/setup.py b/setup.py index ab74c946..18661e95 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup(name='fdroidserver', 'apache-libcloud >= 0.14.1', 'pyasn1', 'pyasn1-modules', + 'python-vagrant', 'PyYAML', 'requests < 2.11', 'docker-py == 1.9.0', -- 2.30.2