2 # -*- coding: utf-8 -*-
4 # build.py - part of the FDroid server tools
5 # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
28 from xml.dom.minidom import Document
29 from optparse import OptionParser
32 from common import BuildException
33 from common import VCSException
36 def build_server(app, thisbuild, build_dir, output_dir):
37 """Do a build on the build server."""
41 # Destroy the builder vm if it already exists...
42 # TODO: need to integrate the snapshot stuff so it doesn't have to
43 # keep wasting time doing this unnecessarily.
44 if os.path.exists(os.path.join('builder', '.vagrant')):
45 if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
46 raise BuildException("Failed to destroy build server")
48 # Start up the virtual maachine...
49 if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
50 # Not a very helpful message yet!
51 raise BuildException("Failed to set up build server")
52 # Get SSH configuration settings for us to connect...
53 subprocess.call('vagrant ssh-config >sshconfig',
54 cwd='builder', shell=True)
55 vagranthost = 'default' # Host in ssh config file
57 # Load and parse the SSH config...
58 sshconfig = paramiko.SSHConfig()
59 sshf = open('builder/sshconfig', 'r')
62 sshconfig = sshconfig.lookup(vagranthost)
64 # Open SSH connection...
65 ssh = paramiko.SSHClient()
66 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
68 ssh.connect(sshconfig['hostname'], username=sshconfig['user'],
69 port=int(sshconfig['port']), timeout=10, look_for_keys=False,
70 key_filename=sshconfig['identityfile'])
72 # Get an SFTP connection...
74 ftp.get_channel().settimeout(15)
76 # Put all the necessary files in place...
77 ftp.chdir('/home/vagrant')
78 ftp.put('build.py', 'build.py')
79 ftp.put('common.py', 'common.py')
80 ftp.put('config.buildserver.py', 'config.py')
83 ftp.put(os.path.join('metadata', app['id'] + '.txt'),
93 for r, d, f in os.walk(path):
98 ftp.put(os.path.join(r, ff), ff)
99 for i in range(len(r.split('/'))):
102 # TODO: send relevant extlib and srclib directories too
104 # Execute the build script...
105 ssh.exec_command('python build.py --on-server -p ' +
106 app['id'] + ' --vercode ' + thisbuild['vercode'])
108 # Retrieve the built files...
109 apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
110 tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
111 ftp.chdir('/home/vagrant/unsigned')
112 ftp.get(apkfile, os.path.join(output_dir, apkfile))
113 ftp.get(tarball, os.path.join(output_dir, tarball))
115 # Get rid of the virtual machine...
116 if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
117 # Not a very helpful message yet!
118 raise BuildException("Failed to destroy")
121 def build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force):
122 """Do a build locally."""
124 # Prepare the source code...
125 root_dir = common.prepare_source(vcs, app, thisbuild,
126 build_dir, extlib_dir, sdk_path, ndk_path,
129 # Scan before building...
130 buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
131 if len(buildprobs) > 0:
132 print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
133 for problem in buildprobs:
134 print '...' + problem
136 raise BuildException("Can't build due to " +
137 str(len(buildprobs)) + " scanned problems")
139 # Build the source tarball right before we build the release...
140 tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
141 tarball = tarfile.open(os.path.join(tmp_dir,
142 tarname + '.tar.gz'), "w:gz")
144 for vcs_dir in ['.svn', '.git', '.hg', '.bzr']:
145 if f.endswith(vcs_dir):
148 tarball.add(build_dir, tarname, exclude=tarexc)
151 # Build native stuff if required...
152 if thisbuild.get('buildjni') not in (None, 'no'):
153 jni_components = thisbuild.get('buildjni')
154 if jni_components == 'yes':
155 jni_components = ['']
157 jni_components = jni_components.split(';')
158 ndkbuild = os.path.join(ndk_path, "ndk-build")
159 for d in jni_components:
161 print "Running ndk-build in " + root_dir + '/' + d
162 p = subprocess.Popen([ndkbuild], cwd=root_dir + '/' + d,
163 stdout=subprocess.PIPE)
164 output = p.communicate()[0]
165 if p.returncode != 0:
167 raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']))
169 # Build the release...
170 if thisbuild.has_key('maven'):
171 p = subprocess.Popen(['mvn', 'clean', 'install',
172 '-Dandroid.sdk.path=' + sdk_path],
173 cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
176 antcommands = ['debug','install']
177 elif thisbuild.has_key('antcommand'):
178 antcommands = [thisbuild['antcommand']]
180 antcommands = ['release']
181 p = subprocess.Popen(['ant'] + antcommands, cwd=root_dir,
182 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183 output, error = p.communicate()
184 if p.returncode != 0:
185 raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
188 print "Build successful"
190 # Find the apk name in the output...
191 if thisbuild.has_key('bindir'):
192 bindir = os.path.join(build_dir, thisbuild['bindir'])
194 bindir = os.path.join(root_dir, 'bin')
195 if thisbuild.get('initfun', 'no') == "yes":
196 # Special case (again!) for funambol...
197 src = ("funambol-android-sync-client-" +
198 thisbuild['version'] + "-unsigned.apk")
199 src = os.path.join(bindir, src)
200 elif thisbuild.has_key('maven'):
201 src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk",
202 output, re.S|re.M).group(1)
203 src = os.path.join(bindir, src) + '.apk'
204 #[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk
206 src = re.match(r".*^.*Creating (\S+) for release.*$.*", output,
208 src = os.path.join(bindir, src)
210 # By way of a sanity check, make sure the version and version
211 # code in our new apk match what we expect...
212 print "Checking " + src
213 p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools',
215 'dump', 'badging', src],
216 stdout=subprocess.PIPE)
217 output = p.communicate()[0]
218 if thisbuild.get('novcheck', 'no') == "yes":
219 vercode = thisbuild['vercode']
220 version = thisbuild['version']
224 for line in output.splitlines():
225 if line.startswith("package:"):
226 pat = re.compile(".*versionCode='([0-9]*)'.*")
227 vercode = re.match(pat, line).group(1)
228 pat = re.compile(".*versionName='([^']*)'.*")
229 version = re.match(pat, line).group(1)
230 if version == None or vercode == None:
231 raise BuildException("Could not find version information in build in output")
233 # Some apps (e.g. Timeriffic) have had the bonkers idea of
234 # including the entire changelog in the version number. Remove
235 # it so we can compare. (TODO: might be better to remove it
236 # before we compile, in fact)
237 index = version.find(" //")
239 version = version[:index]
241 if (version != thisbuild['version'] or
242 vercode != thisbuild['vercode']):
243 raise BuildException(("Unexpected version/version code in output"
246 % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
249 # Copy the unsigned apk to our destination directory for further
250 # processing (by publish.py)...
251 dest = os.path.join(output_dir, app['id'] + '_' +
252 thisbuild['vercode'] + '.apk')
253 shutil.copyfile(src, dest)
255 # Move the source tarball into the output directory...
256 if output_dir != tmp_dir:
257 tarfilename = tarname + '.tar.gz'
258 shutil.move(os.path.join(tmp_dir, tarfilename),
259 os.path.join(output_dir, tarfilename))
262 def trybuild(app, thisbuild, build_dir, output_dir, extlib_dir, tmp_dir,
263 repo_dir, vcs, test, server, install, force):
265 Build a particular version of an application, if it needs building.
267 Returns True if the build was done, False if it wasn't necessary.
270 dest = os.path.join(output_dir, app['id'] + '_' +
271 thisbuild['vercode'] + '.apk')
272 dest_repo = os.path.join(repo_dir, app['id'] + '_' +
273 thisbuild['vercode'] + '.apk')
275 if os.path.exists(dest) or (not test and os.path.exists(dest_repo)):
278 if thisbuild['commit'].startswith('!'):
281 print "Building version " + thisbuild['version'] + ' of ' + app['id']
284 build_server(app, thisbuild, build_dir, output_dir)
286 build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force)
290 def parse_commandline():
291 """Parse the command line. Returns options, args."""
293 parser = OptionParser()
294 parser.add_option("-v", "--verbose", action="store_true", default=False,
295 help="Spew out even more information than normal")
296 parser.add_option("-p", "--package", default=None,
297 help="Build only the specified package")
298 parser.add_option("-c", "--vercode", default=None,
299 help="Build only the specified version code")
300 parser.add_option("-s", "--stop", action="store_true", default=False,
301 help="Make the build stop on exceptions")
302 parser.add_option("-t", "--test", action="store_true", default=False,
303 help="Test mode - put output in the tmp directory only.")
304 parser.add_option("--server", action="store_true", default=False,
305 help="Use build server")
306 parser.add_option("--on-server", action="store_true", default=False,
307 help="Specify that we're running on the build server")
308 parser.add_option("-f", "--force", action="store_true", default=False,
309 help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
310 parser.add_option("--install", action="store_true", default=False,
311 help="Use 'ant debug install' to build and install a " +
312 "debug version on your device or emulator. " +
313 "Implies --force and --test")
314 parser.add_option("--all", action="store_true", default=False,
315 help="Use with --install, when not using --package"
316 " to confirm you really want to build and install everything.")
317 options, args = parser.parse_args()
319 # The --install option implies --test and --force...
322 print "Can't install when building on a build server."
324 if not options.package and not options.all:
325 print "This would build and install everything in the repo to the device."
326 print "You probably want to use --package and maybe also --vercode."
327 print "If you really want to install everything, use --all."
332 if options.force and not options.test:
333 print "Force is only allowed in test mode"
343 # Read configuration...
344 execfile('config.py', globals())
345 options, args = parse_commandline()
348 apps = common.read_metadata(options.verbose)
351 if not os.path.isdir(log_dir):
352 print "Creating log directory"
356 if not os.path.isdir(tmp_dir):
357 print "Creating temporary directory"
363 output_dir = 'unsigned'
364 if not os.path.isdir(output_dir):
365 print "Creating output directory"
366 os.makedirs(output_dir)
371 if not os.path.isdir(build_dir):
372 print "Creating build directory"
373 os.makedirs(build_dir)
374 extlib_dir = os.path.join(build_dir, 'extlib')
376 # Filter apps and build versions according to command-line options, etc...
378 apps = [app for app in apps if app['id'] == options.package]
380 print "No such package"
382 apps = [app for app in apps if (options.force or not app['Disabled']) and
383 app['builds'] and len(app['Repo Type']) > 0 and len(app['builds']) > 0]
385 print "Nothing to do - all apps are disabled or have no builds defined."
389 app['builds'] = [b for b in app['builds']
390 if str(b['vercode']) == options.vercode]
392 # Build applications...
397 build_dir = 'build/' + app['id']
399 # Set up vcs interface and make sure we have the latest code...
400 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
402 for thisbuild in app['builds']:
404 if trybuild(app, thisbuild, build_dir, output_dir, extlib_dir,
405 tmp_dir, repo_dir, vcs, options.test, options.server,
406 options.install, options.force):
407 build_succeeded.append(app)
408 except BuildException as be:
411 print "Could not build app %s due to BuildException: %s" % (app['id'], be)
412 logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
413 logfile.write(str(be))
415 failed_apps[app['id']] = be
416 except VCSException as vcse:
419 print "VCS error while building app %s: %s" % (app['id'], vcse)
420 failed_apps[app['id']] = vcse
421 except Exception as e:
424 print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
425 failed_apps[app['id']] = e
427 for app in build_succeeded:
428 print "success: %s" % (app['id'])
430 for fa in failed_apps:
431 print "Build for app %s failed:\n%s" % (fa, failed_apps[fa])
434 if len(build_succeeded) > 0:
435 print str(len(build_succeeded)) + ' builds succeeded'
436 if len(failed_apps) > 0:
437 print str(len(failed_apps)) + ' builds failed'
440 if __name__ == "__main__":