From d4c6fffb301ca1caa8af7d7e9fcba96350fe1354 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 4 Jul 2016 20:49:41 +0200 Subject: [PATCH] buildserver: buildserver/Vagrantfile is configed by .yaml file Vagrantfile is now committed and not changed between configurations. It is configed by translating the python config file's dict to a YAML file, which Vagrantfile now loads and uses. This makes it a lot easier for vagrant users and python programmers to understand, and hopefully makes it easier to maintain and test with. --- .gitignore | 1 - buildserver/.gitignore | 2 +- buildserver/Vagrantfile | 70 +++++++++++++++++++++++++++++ hooks/pre-commit | 2 +- makebuildserver | 97 ++++------------------------------------- 5 files changed, 81 insertions(+), 91 deletions(-) create mode 100644 buildserver/Vagrantfile diff --git a/.gitignore b/.gitignore index cd84eae1..8bff77fa 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ tmp/ tests/repo/icons* # files used in manual testing -/buildserver/Vagrantfile.yaml /config.py /tmp/ /logs/ diff --git a/buildserver/.gitignore b/buildserver/.gitignore index 4a3901ed..d8165d61 100644 --- a/buildserver/.gitignore +++ b/buildserver/.gitignore @@ -1,4 +1,4 @@ .vagrant up.log cache/ -Vagrantfile +Vagrantfile.yaml diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile new file mode 100644 index 00000000..e586e05c --- /dev/null +++ b/buildserver/Vagrantfile @@ -0,0 +1,70 @@ + +require 'yaml' +configfile = YAML.load_file('Vagrantfile.yaml') + +Vagrant.configure("2") do |config| + + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + config.cache.auto_detect = false + config.cache.enable :apt + config.cache.enable :chef + end + + config.vm.box = configfile['basebox'] + config.vm.box_url = configfile['baseboxurl'] + + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--memory", configfile['memory']] + v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] + v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] + end + + config.vm.boot_timeout = configfile['boot_timeout'] + + config.vm.provision :shell, :path => "fixpaths.sh" + + if configfile.has_key? "aptproxy" + config.vm.provision :shell, path: "provision-apt-proxy", + args: [configfile["aptproxy"]] + end + + # buildserver/ is shared to the VM's /vagrant by default so the old + # default does not need a custom mount + if configfile["cachedir"] != "buildserver/cache" + config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', + owner: 'root', group: 'root', create: true + end + + # cache .deb packages on the host via a mount trick + if configfile.has_key? "aptcachedir" + config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives", + owner: 'root', group: 'root', create: true + end + + config.vm.provision "shell", path: "setup-env-vars", + args: ["/home/vagrant/android-sdk"] + config.vm.provision "shell", path: "provision-apt-get-install", + args: [configfile['debian_mirror']] + + config.vm.provision :chef_solo do |chef| + chef.cookbooks_path = "cookbooks" + chef.log_level = :debug + chef.add_recipe "kivy" + end + + config.vm.provision "shell", path: "provision-android-sdk" + config.vm.provision "shell", path: "provision-android-ndk", + args: ["/home/vagrant/android-ndk"] + config.vm.provision "shell", path: "provision-pip", + args: ["compare-locales"] + config.vm.provision "shell", path: "provision-gradle" + config.vm.provision "file", source: "gradle", + destination: "/opt/gradle/bin/gradle" + + # let Ubuntu/trusty's paramiko work with the VM instance + if `uname -v`.include? "14.04" + config.vm.provision "shell", path: "provision-ubuntu-trusty-paramiko" + end + +end diff --git a/hooks/pre-commit b/hooks/pre-commit index 9f0dcb77..1929ee91 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -12,7 +12,7 @@ if [ -z "$files" ]; then PY_TEST_FILES="tests/*.TestCase" SH_FILES="hooks/pre-commit" BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion buildserver/provision-*" - RB_FILES="buildserver/cookbooks/*/recipes/*.rb" + RB_FILES="buildserver/cookbooks/*/recipes/*.rb buildserver/Vagrantfile" else # if actually committing right now, then only run on the files # that are going to be committed at this moment diff --git a/makebuildserver b/makebuildserver index 15a29868..62fee88d 100755 --- a/makebuildserver +++ b/makebuildserver @@ -71,6 +71,7 @@ if os.path.exists('makebuildserver.config.py'): elif os.path.exists('makebs.config.py'): # this is the old name for the config file exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config) +del(config['__builtins__']) # added by compile/exec if not os.path.exists('makebuildserver') or not os.path.exists(serverdir): print('This must be run from the correct directory!') @@ -320,94 +321,15 @@ elif os.path.exists('/proc/cpuinfo'): if 'vmx' in contents or 'svm' in contents: config['hwvirtex'] = 'on' -del(config['__builtins__']) # added by compile/exec -with open(os.path.join(serverdir, 'Vagrantfile.yaml'), 'w') as f: - yaml.dump(config, f) - -# Generate an appropriate Vagrantfile for the buildserver, based on our -# settings... -vagrantfile = """ -require 'yaml' -configfile = YAML.load_file('Vagrantfile.yaml') - -Vagrant.configure("2") do |config| - - if Vagrant.has_plugin?("vagrant-cachier") - config.cache.scope = :box - config.cache.auto_detect = false - config.cache.enable :apt - config.cache.enable :chef - end - - config.vm.box = configfile['basebox'] - config.vm.box_url = configfile['baseboxurl'] - - config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--memory", configfile['memory']] - v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] - v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] - end - - config.vm.boot_timeout = configfile['boot_timeout'] - - config.vm.provision :shell, :path => "fixpaths.sh" - - if configfile.has_key? "aptproxy" - config.vm.provision :shell, path: "provision-apt-proxy", - args: [configfile["aptproxy"]] - end - - # buildserver/ is shared to the VM's /vagrant by default so the old - # default does not need a custom mount - if configfile["cachedir"] != "buildserver/cache" - config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', - owner: 'root', group: 'root', create: true - end - - # cache .deb packages on the host via a mount trick - if configfile.has_key? "aptcachedir" - config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives", - owner: 'root', group: 'root', create: true - end - - config.vm.provision "shell", path: "setup-env-vars", - args: ["/home/vagrant/android-sdk"] - config.vm.provision "shell", path: "provision-apt-get-install", - args: [configfile['debian_mirror']] - - config.vm.provision :chef_solo do |chef| - chef.cookbooks_path = "cookbooks" - chef.log_level = :debug - chef.add_recipe "kivy" - end - - config.vm.provision "shell", path: "provision-android-sdk" - config.vm.provision "shell", path: "provision-android-ndk", - args: ["/home/vagrant/android-ndk"] - config.vm.provision "shell", path: "provision-pip", - args: ["compare-locales"] - config.vm.provision "shell", path: "provision-gradle" - config.vm.provision "file", source: "gradle", - destination: "/opt/gradle/bin/gradle" - - # let Ubuntu/trusty's paramiko work with the VM instance - if `uname -v`.include? "14.04" - config.vm.provision "shell", path: "provision-ubuntu-trusty-paramiko" - end - -end -""" - - -# Check against the existing Vagrantfile, and if they differ, we need to -# create a new box: -vf = os.path.join(serverdir, 'Vagrantfile') +# Check against the existing Vagrantfile.yaml, and if they differ, we +# need to create a new box: +vf = os.path.join(serverdir, 'Vagrantfile.yaml') writevf = True if os.path.exists(vf): vagrant(['halt'], serverdir) - with open(vf, 'r') as f: - oldvf = f.read() - if oldvf != vagrantfile: + with open(vf, 'r', encoding='utf-8') as f: + oldconfig = yaml.load(f) + if config != oldconfig: print("Server configuration has changed, rebuild from scratch is required") vagrant(['destroy', '-f'], serverdir) else: @@ -416,9 +338,8 @@ if os.path.exists(vf): else: print("No existing server - building from scratch") if writevf: - with open(vf, 'w') as f: - f.write(vagrantfile) - + with open(vf, 'w', encoding='utf-8') as f: + yaml.dump(config, f) print("Configuring build server VM") returncode, out = vagrant(['up', '--provision'], serverdir, printout=True) -- 2.30.2