From: Ciaran Gultnieks Date: Fri, 31 Aug 2012 13:48:50 +0000 (+0100) Subject: Integrated latest build server stuff (except snapshots) X-Git-Tag: 0.1~777 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=d9f404ba073ad413fdaeba8136dd6aeb0728a497;p=fdroidserver.git Integrated latest build server stuff (except snapshots) --- diff --git a/builder/.gitignore b/builder/.gitignore deleted file mode 100644 index 80b72f60..00000000 --- a/builder/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -sshconfig -.vagrant -*.log diff --git a/builder/Vagrantfile b/builder/Vagrantfile deleted file mode 100644 index 1487a8df..00000000 --- a/builder/Vagrantfile +++ /dev/null @@ -1,7 +0,0 @@ -Vagrant::Config.run do |config| - - config.vm.box = "buildserver" - - config.vm.customize ["modifyvm", :id, "--memory", "768"] - -end diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 4d323809..8e30fead 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -1,7 +1,7 @@ Vagrant::Config.run do |config| - config.vm.box = "debian6-32" - config.vm.box_url = "/shares/software/OS and Boot/debian6-32.box" + config.vm.box = "precise32" + config.vm.box_url = "/shares/software/OS and Boot/precise32.box" config.vm.customize ["modifyvm", :id, "--memory", "1024"] diff --git a/buildserver/cookbooks/android-sdk/recipes/default.rb b/buildserver/cookbooks/android-sdk/recipes/default.rb index 91b5a084..80c10b8f 100644 --- a/buildserver/cookbooks/android-sdk/recipes/default.rb +++ b/buildserver/cookbooks/android-sdk/recipes/default.rb @@ -17,8 +17,18 @@ script "setup-android-sdk" do mv android-sdk-linux #{sdk_loc} rm android-sdk_r16-linux.tgz #{sdk_loc}/tools/android update sdk --no-ui -t platform-tool - #{sdk_loc}/tools/android update sdk --no-ui -t platform - #{sdk_loc}/tools/android update sdk --no-ui -t tool,platform-tool + #{sdk_loc}/tools/android update sdk --no-ui -t tool + #{sdk_loc}/tools/android update sdk --no-ui -t android-3 + #{sdk_loc}/tools/android update sdk --no-ui -t android-4 + #{sdk_loc}/tools/android update sdk --no-ui -t android-7 + #{sdk_loc}/tools/android update sdk --no-ui -t android-8 + #{sdk_loc}/tools/android update sdk --no-ui -t android-10 + #{sdk_loc}/tools/android update sdk --no-ui -t android-11 + #{sdk_loc}/tools/android update sdk --no-ui -t android-13 + #{sdk_loc}/tools/android update sdk --no-ui -t android-14 + #{sdk_loc}/tools/android update sdk --no-ui -t android-15 + #{sdk_loc}/tools/android update sdk --no-ui -t android-16 + #{sdk_loc}/tools/android update sdk --no-ui -t addon-google_apis-google-16 " not_if do File.exists?("#{sdk_loc}") diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb index 756d6bcc..4782f125 100644 --- a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -1,5 +1,5 @@ -%w{ant ant-contrib maven2 javacc python git-core mercurial subversion bzr}.each do |pkg| +%w{ant ant-contrib maven javacc python git-core mercurial subversion bzr git-svn make perlmagick}.each do |pkg| package pkg do action :install end diff --git a/docs/fdroid.texi b/docs/fdroid.texi index c7c91cb0..b0010357 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -979,7 +979,9 @@ the ability to execute anything) for other applications in the repository. @end enumerate Through complete isolation, the repurcussions are at least limited to the -application in question. +application in question. Not only is the build environment fresh for each +build, and thrown away afterwards, but it is also isolated from the signing +environment. Aside from security issues, there are some applications which have strange requirements such as custom versions of the NDK. It would be impractical (or @@ -994,14 +996,14 @@ Some things may not work properly yet. Talk to CiaranG if you're trying to use this and have problems. In addition to the basic setup previously described, you will also need -a Vagrant-compatible Debian Squeeze base box called 'debian6-32'. You can -create one of these for yourself from standard Debian installation media, as +a Vagrant-compatible Ubuntu Precise base box called 'precise32'. You can +create one of these for yourself from standard Ubuntu installation media, as the specification for what's required to be Vagrant-compatible is very well defined. This is the sensible and secure way to do it, since you know what's -in it. If you insist on taking a shortcut, ask CiaranG for his on the forum -or in IRC. +in it. If you insist on taking a shortcut, ask CiaranG about it on IRC. -With this base box installed, you can then do: +With this base box installed, you can then go to the @code{fdroidserver} +directory and run this: @example ./makebuildserver.sh diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 99f057a9..bfe1abdf 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -36,13 +36,13 @@ from common import VCSException def build_server(app, thisbuild, build_dir, output_dir): """Do a build on the build server.""" - import paramiko + import ssh # Destroy the builder vm if it already exists... # TODO: need to integrate the snapshot stuff so it doesn't have to # keep wasting time doing this unnecessarily. if os.path.exists(os.path.join('builder', '.vagrant')): - if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0: + if subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder') != 0: raise BuildException("Failed to destroy build server") # Start up the virtual maachine... @@ -55,65 +55,114 @@ def build_server(app, thisbuild, build_dir, output_dir): vagranthost = 'default' # Host in ssh config file # Load and parse the SSH config... - sshconfig = paramiko.SSHConfig() + sshconfig = ssh.SSHConfig() sshf = open('builder/sshconfig', 'r') sshconfig.parse(sshf) sshf.close() sshconfig = sshconfig.lookup(vagranthost) # Open SSH connection... - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - print sshconfig - ssh.connect(sshconfig['hostname'], username=sshconfig['user'], + sshs = ssh.SSHClient() + sshs.set_missing_host_key_policy(ssh.AutoAddPolicy()) + sshs.connect(sshconfig['hostname'], username=sshconfig['user'], port=int(sshconfig['port']), timeout=10, look_for_keys=False, key_filename=sshconfig['identityfile']) # Get an SFTP connection... - ftp = ssh.open_sftp() + ftp = sshs.open_sftp() ftp.get_channel().settimeout(15) # Put all the necessary files in place... ftp.chdir('/home/vagrant') - ftp.put('build.py', 'build.py') - ftp.put('common.py', 'common.py') - ftp.put('config.buildserver.py', 'config.py') + + # Helper to copy the contents of a directory to the server... + def send_dir(path): + root = os.path.dirname(path) + main = os.path.basename(path) + ftp.mkdir(main) + for r, d, f in os.walk(path): + rr = os.path.relpath(r, root) + ftp.chdir(rr) + for dd in d: + ftp.mkdir(dd) + for ff in f: + ftp.put(os.path.join(root, rr, ff), ff) + for i in range(len(rr.split('/'))): + ftp.chdir('..') + ftp.chdir('..') + + serverpath = os.path.abspath(os.path.dirname(__file__)) + ftp.put(os.path.join(serverpath, 'build.py'), 'build.py') + ftp.put(os.path.join(serverpath, 'common.py'), 'common.py') + ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py') + + # Copy the metadata - just the file for this app... ftp.mkdir('metadata') ftp.chdir('metadata') ftp.put(os.path.join('metadata', app['id'] + '.txt'), app['id'] + '.txt') ftp.chdir('..') + # Create the build directory... ftp.mkdir('build') ftp.chdir('build') ftp.mkdir('extlib') - ftp.mkdir(app['id']) - ftp.chdir('..') - def send_dir(path): - lastdir = path - for r, d, f in os.walk(path): - ftp.chdir(r) - for dd in d: - ftp.mkdir(dd) - for ff in f: - ftp.put(os.path.join(r, ff), ff) - for i in range(len(r.split('/'))): + # Copy the main app source code + if os.path.exists(build_dir): + send_dir(build_dir) + # Copy any extlibs that are required... + if thisbuild.has_key('extlibs'): + ftp.chdir('build') + ftp.chdir('extlib') + for lib in thisbuild['extlibs'].split(';'): + lp = lib.split('/') + for d in lp[:-1]: + ftp.mkdir(d) + ftp.chdir(d) + ftp.put(os.path.join('build/extlib', lib), lp[-1]) + for _ in lp[:-1]: + ftp.chdir('..') + ftp.chdir('..') + ftp.chdir('..') + # Copy any srclibs that are required... + if thisbuild.has_key('srclibs'): + ftp.chdir('build') + ftp.chdir('extlib') + for lib in thisbuild['srclibs'].split(';'): + lp = lib.split('@').split('/') + for d in lp[:-1]: + ftp.mkdir(d) + ftp.chdir(d) + ftp.put(os.path.join('build/extlib', lib), lp[-1]) + for _ in lp[:-1]: ftp.chdir('..') - send_dir(build_dir) - # TODO: send relevant extlib and srclib directories too + ftp.chdir('..') + ftp.chdir('..') # Execute the build script... - ssh.exec_command('python build.py --on-server -p ' + + chan = sshs.get_transport().open_session() + stdoutf = chan.makefile('rb') + stderrf = chan.makefile_stderr('rb') + chan.exec_command('python build.py --on-server -p ' + app['id'] + ' --vercode ' + thisbuild['vercode']) + returncode = chan.recv_exit_status() + output = stdoutf.read() + error = stderrf.read() + if returncode != 0: + raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) # Retrieve the built files... + ftp.chdir('/home/vagrant/unsigned') apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' - ftp.chdir('/home/vagrant/unsigned') - ftp.get(apkfile, os.path.join(output_dir, apkfile)) - ftp.get(tarball, os.path.join(output_dir, tarball)) + try: + ftp.get(apkfile, os.path.join(output_dir, apkfile)) + ftp.get(tarball, os.path.join(output_dir, tarball)) + except: + raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) + ftp.close() # Get rid of the virtual machine... - if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0: + if subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder') != 0: # Not a very helpful message yet! raise BuildException("Failed to destroy") @@ -304,6 +353,10 @@ def trybuild(app, thisbuild, build_dir, output_dir, extlib_dir, tmp_dir, print "Building version " + thisbuild['version'] + ' of ' + app['id'] if server: + # When using server mode, still keep a local cache of the repo, by + # grabbing the source now. + vcs.gotorevision(thisbuild['commit']) + build_server(app, thisbuild, build_dir, output_dir) else: build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force, verbose) @@ -396,6 +449,17 @@ def main(): os.makedirs(build_dir) extlib_dir = os.path.join(build_dir, 'extlib') + # Create the 'builder' vagrant directory if we don't have it... + if options.server: + if not os.path.exists('builder'): + os.mkdir('builder') + vf = open('builder/Vagrantfile', 'w') + vf.write('Vagrant::Config.run do |config|\n') + vf.write('config.vm.box = "buildserver"\n') + vf.write('config.vm.customize ["modifyvm", :id, "--memory", "768"]\n') + vf.write('end\n') + vf.close() + # Filter apps and build versions according to command-line options, etc... if options.package: apps = [app for app in apps if app['id'] == options.package] diff --git a/makebuildserver.sh b/makebuildserver.sh index 31594172..5e2a7c00 100755 --- a/makebuildserver.sh +++ b/makebuildserver.sh @@ -3,6 +3,7 @@ set -e rm -f buildserver.box cd buildserver vagrant up +sleep 5 vagrant ssh -c "sudo shutdown -h now" cd .. # Just to wait until it's shut down!