From: Hans-Christoph Steiner <hans@eds.org>
Date: Mon, 4 Jul 2016 18:22:00 +0000 (+0200)
Subject: buildserver: send config to vagrant via YAML file
X-Git-Tag: 0.7.0~35^2~2
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?p=fdroidserver.git;a=commitdiff_plain;h=2e1ec71404a9f9577eb68971fff025d9f6036101

buildserver: send config to vagrant via YAML file

Python can easily output dicts as YAML, and a Vagrantfile is a ruby script,
which can easily read YAML.  Going this route means that Vagrantfile can
ultimately be committed to git, and the configuration will happen all via
Python dicts output as YAML.  That makes it drastically easier to follow
the code, and to make modifications.
---

diff --git a/.gitignore b/.gitignore
index 8bff77fa..cd84eae1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,7 @@ tmp/
 tests/repo/icons*
 
 # files used in manual testing
+/buildserver/Vagrantfile.yaml
 /config.py
 /tmp/
 /logs/
diff --git a/buildserver/provision-apt-proxy b/buildserver/provision-apt-proxy
new file mode 100644
index 00000000..ec9a5ee8
--- /dev/null
+++ b/buildserver/provision-apt-proxy
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo $0
+set -e
+
+rm -f /etc/apt/apt.conf.d/02proxy
+echo "Acquire::ftp::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+echo "Acquire::http::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+echo "Acquire::https::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+
+apt-get update
diff --git a/makebuildserver b/makebuildserver
index 0a0e0d43..15a29868 100755
--- a/makebuildserver
+++ b/makebuildserver
@@ -6,6 +6,7 @@ import sys
 import subprocess
 import time
 import hashlib
+import yaml
 from clint.textui import progress
 from optparse import OptionParser
 
@@ -61,6 +62,7 @@ config = {
     'cachedir': cachedir,
     'cpus': 1,
     'memory': 1024,
+    'hwvirtex': 'off',
 }
 
 # load config file, if present
@@ -85,6 +87,9 @@ cachedir = config['cachedir']
 if not os.path.exists(cachedir):
     os.makedirs(cachedir, 0o755)
 
+if config['apt_package_cache']:
+    config['aptcachedir'] = cachedir + '/apt/archives'
+
 cachefiles = [
     ('https://dl.google.com/android/repository/tools_r25.1.7-linux.zip',
      '3ca053600a86a5a64d5571edfbb1dad27f2bda3bfd2d38e2fe54322610b1ef0b'),
@@ -302,29 +307,29 @@ for srcurl, shasum in cachefiles:
         print("Invalid shasum of '" + v + "' detected for " + local_filename)
         sys.exit(1)
 
-# allow specifying a list/tuple that includes cached local copy
-if type(config['baseboxurl']) in (list, tuple) or config['baseboxurl'][0] in ('(', '['):
-    baseboxurl = config['baseboxurl']
-else:
-    baseboxurl = '"{0}"'.format(config['baseboxurl'])
-
 # use VirtualBox software virtualization if hardware is not available,
 # like if this is being run in kvm or some other VM platform, like
 # http://jenkins.debian.net, the values are 'on' or 'off'
-hwvirtex = 'off'
 if sys.platform.startswith('darwin'):
     # all < 10 year old Macs work, and OSX servers as VM host are very
     # rare, but this could also be auto-detected if someone codes it
-    hwvirtex = 'on'
+    config['hwvirtex'] = 'on'
 elif os.path.exists('/proc/cpuinfo'):
     with open('/proc/cpuinfo') as f:
         contents = f.read()
     if 'vmx' in contents or 'svm' in contents:
-        hwvirtex = 'on'
+        config['hwvirtex'] = 'on'
+
+del(config['__builtins__'])  # added by compile/exec
+with open(os.path.join(serverdir, 'Vagrantfile.yaml'), 'w') as f:
+    yaml.dump(config, f)
 
 # Generate an appropriate Vagrantfile for the buildserver, based on our
 # settings...
 vagrantfile = """
+require 'yaml'
+configfile = YAML.load_file('Vagrantfile.yaml')
+
 Vagrant.configure("2") do |config|
 
   if Vagrant.has_plugin?("vagrant-cachier")
@@ -334,51 +339,41 @@ Vagrant.configure("2") do |config|
     config.cache.enable :chef
   end
 
-  config.vm.box = "{0}"
-  config.vm.box_url = {1}
+  config.vm.box = configfile['basebox']
+  config.vm.box_url = configfile['baseboxurl']
 
   config.vm.provider "virtualbox" do |v|
-    v.customize ["modifyvm", :id, "--memory", "{2}"]
-    v.customize ["modifyvm", :id, "--cpus", "{3}"]
-    v.customize ["modifyvm", :id, "--hwvirtex", "{4}"]
+    v.customize ["modifyvm", :id, "--memory", configfile['memory']]
+    v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
+    v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
   end
 
-  config.vm.boot_timeout = {5}
+  config.vm.boot_timeout = configfile['boot_timeout']
 
   config.vm.provision :shell, :path => "fixpaths.sh"
-""".format(config['basebox'],
-           baseboxurl,
-           config['memory'],
-           config.get('cpus', 1),
-           hwvirtex,
-           config['boot_timeout'])
-if 'aptproxy' in config and config['aptproxy']:
-    vagrantfile += """
-  config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
-""".format(config['aptproxy'])
-
-# buildserver/ is shared to the VM's /vagrant by default so the old default
-# does not need a custom mount
-if cachedir != 'buildserver/cache':
-    vagrantfile += """
-  config.vm.synced_folder '{0}', '/vagrant/cache',
-    owner: 'root', group: 'root', create: true
-""".format(cachedir)
-
-# cache .deb packages on the host via a mount trick
-if config['apt_package_cache']:
-    aptcachedir = cachedir + '/apt/archives'
-    vagrantfile += """
-  config.vm.synced_folder "{0}", "/var/cache/apt/archives",
-    owner: 'root', group: 'root', create: true
-""".format(aptcachedir)
 
-vagrantfile += """
+  if configfile.has_key? "aptproxy"
+    config.vm.provision :shell, path: "provision-apt-proxy",
+      args: [configfile["aptproxy"]]
+  end
+
+  # buildserver/ is shared to the VM's /vagrant by default so the old
+  # default does not need a custom mount
+  if configfile["cachedir"] != "buildserver/cache"
+    config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
+      owner: 'root', group: 'root', create: true
+  end
+
+  # cache .deb packages on the host via a mount trick
+  if configfile.has_key? "aptcachedir"
+    config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives",
+      owner: 'root', group: 'root', create: true
+  end
 
   config.vm.provision "shell", path: "setup-env-vars",
     args: ["/home/vagrant/android-sdk"]
   config.vm.provision "shell", path: "provision-apt-get-install",
-    args: ["{0}"]
+    args: [configfile['debian_mirror']]
 
   config.vm.provision :chef_solo do |chef|
     chef.cookbooks_path = "cookbooks"
@@ -401,7 +396,7 @@ vagrantfile += """
   end
 
 end
-""".format(config['debian_mirror'])
+"""
 
 
 # Check against the existing Vagrantfile, and if they differ, we need to