-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
#
# build.py - part of the FDroid server tools
# Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com
import time
import json
from ConfigParser import ConfigParser
-from optparse import OptionParser, OptionError
-from distutils.version import LooseVersion
+from argparse import ArgumentParser
import logging
import common
+import net
import metadata
+import scanner
from common import FDroidException, BuildException, VCSException, FDroidPopen, SdkToolsPopen
try:
raise BuildException("Error getting ssh config")
vagranthost = 'default' # Host in ssh config file
sshconfig = paramiko.SSHConfig()
- sshf = open('builder/sshconfig', 'r')
+ sshf = open(os.path.join('builder', 'sshconfig'), 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
shutil.rmtree('builder')
os.mkdir('builder')
- p = subprocess.Popen('vagrant --version', shell=True,
+ p = subprocess.Popen(['vagrant', '--version'],
stdout=subprocess.PIPE)
- vver = p.communicate()[0]
- if vver.startswith('Vagrant version 1.2'):
- with open('builder/Vagrantfile', 'w') as vf:
- vf.write('Vagrant.configure("2") do |config|\n')
- vf.write('config.vm.box = "buildserver"\n')
- vf.write('end\n')
- else:
- with open('builder/Vagrantfile', 'w') as vf:
- vf.write('Vagrant::Config.run do |config|\n')
- vf.write('config.vm.box = "buildserver"\n')
- vf.write('end\n')
+ vver = p.communicate()[0].strip().split(' ')[1]
+ if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4:
+ raise BuildException("Unsupported vagrant version {0}".format(vver))
+
+ with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf:
+ vf.write('Vagrant.configure("2") do |config|\n')
+ vf.write('config.vm.box = "buildserver"\n')
+ vf.write('config.vm.synced_folder ".", "/vagrant", disabled: true\n')
+ vf.write('end\n')
logging.info("Starting new build server")
retcode, _ = vagrant(['up'], cwd='builder')
# Note that 'force' here also implies test mode.
-def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
+def build_server(app, build, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
try:
except NameError:
raise BuildException("Paramiko is required to use the buildserver")
if options.verbose:
- logging.getLogger("paramiko").setLevel(logging.DEBUG)
+ logging.getLogger("paramiko").setLevel(logging.INFO)
else:
logging.getLogger("paramiko").setLevel(logging.WARN)
logging.info("Preparing server for build...")
serverpath = os.path.abspath(os.path.dirname(__file__))
- ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
- ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
- ftp.put(os.path.join(serverpath, 'metadata.py'), 'metadata.py')
+ ftp.mkdir('fdroidserver')
+ ftp.chdir('fdroidserver')
+ ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid')
+ ftp.chmod('fdroid', 0o755)
+ send_dir(os.path.join(serverpath))
+ ftp.chdir(homedir)
+
ftp.put(os.path.join(serverpath, '..', 'buildserver',
'config.buildserver.py'), 'config.py')
ftp.chmod('config.py', 0o600)
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
- ftp.put(os.path.join('metadata', app['id'] + '.txt'),
- app['id'] + '.txt')
+ ftp.put(os.path.join('metadata', app.id + '.txt'),
+ app.id + '.txt')
# And patches if there are any...
- if os.path.exists(os.path.join('metadata', app['id'])):
- send_dir(os.path.join('metadata', app['id']))
+ if os.path.exists(os.path.join('metadata', app.id)):
+ send_dir(os.path.join('metadata', app.id))
ftp.chdir(homedir)
# Create the build directory...
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
- if thisbuild['extlibs']:
+ if build.extlibs:
ftp.chdir(homedir + '/build/extlib')
- for lib in thisbuild['extlibs']:
+ for lib in build.extlibs:
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
- if thisbuild['srclibs']:
- for lib in thisbuild['srclibs']:
+ if build.srclibs:
+ for lib in build.srclibs:
srclibpaths.append(
- common.getsrclib(lib, 'build/srclib', srclibpaths,
- basepath=True, prepare=False))
+ common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
# (no need if it's a srclib)
if (not basesrclib) and os.path.exists(build_dir):
ftp.chdir(homedir + '/build')
- fv = '.fdroidvcs-' + app['id']
+ fv = '.fdroidvcs-' + app.id
ftp.put(os.path.join('build', fv), fv)
send_dir(build_dir)
logging.info("Starting build...")
chan = sshs.get_transport().open_session()
chan.get_pty()
- cmdline = 'python build.py --on-server'
+ cmdline = os.path.join(homedir, 'fdroidserver', 'fdroid')
+ cmdline += ' build --on-server'
if force:
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
- cmdline += " %s:%s" % (app['id'], thisbuild['vercode'])
+ cmdline += " %s:%s" % (app.id, build.vercode)
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
while not chan.exit_status_ready():
if returncode != 0:
raise BuildException(
"Build.py failed on server for {0}:{1}".format(
- app['id'], thisbuild['version']), output)
+ app.id, build.version), output)
# Retrieve the built files...
logging.info("Retrieving build output...")
ftp.chdir(homedir + '/tmp')
else:
ftp.chdir(homedir + '/unsigned')
- apkfile = common.getapkname(app, thisbuild)
- tarball = common.getsrcname(app, thisbuild)
+ apkfile = common.getapkname(app, build)
+ tarball = common.getsrcname(app, build)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
if not options.notarball:
except:
raise BuildException(
"Build failed for %s:%s - missing output files".format(
- app['id'], thisbuild['version']), output)
+ app.id, build.version), output)
ftp.close()
finally:
def adapt_gradle(build_dir):
+ filename = 'build.gradle'
for root, dirs, files in os.walk(build_dir):
- if 'build.gradle' in files:
- path = os.path.join(root, 'build.gradle')
- logging.debug("Adapting build.gradle at %s" % path)
-
- FDroidPopen(['sed', '-i',
- r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"'
- + config['build_tools'] + '"@g', path])
-
-
-def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver):
+ for filename in files:
+ if not filename.endswith('.gradle'):
+ continue
+ path = os.path.join(root, filename)
+ if not os.path.isfile(path):
+ continue
+ logging.debug("Adapting %s at %s" % (filename, path))
+ common.regsub_file(r"""(\s*)buildToolsVersion([\s=]+).*""",
+ r"""\1buildToolsVersion\2'%s'""" % config['build_tools'],
+ path)
+
+
+def capitalize_intact(string):
+ """Like str.capitalize(), but leave the rest of the string intact without
+ switching it to lowercase."""
+ if len(string) == 0:
+ return string
+ if len(string) == 1:
+ return string.upper()
+ return string[0].upper() + string[1:]
+
+
+def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally."""
- if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
- if not config['ndk_path']:
- logging.critical("$ANDROID_NDK is not set!")
+ ndk_path = build.ndk_path()
+ if build.buildjni and build.buildjni != ['no']:
+ if not ndk_path:
+ logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r10e')
+ logging.critical("Configured versions:")
+ for k, v in config['ndk_paths'].iteritems():
+ if k.endswith("_orig"):
+ continue
+ logging.critical(" %s: %s" % (k, v))
sys.exit(3)
- elif not os.path.isdir(config['sdk_path']):
- logging.critical("$ANDROID_NDK points to a non-existing directory!")
+ elif not os.path.isdir(ndk_path):
+ logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
sys.exit(3)
+ # Set up environment vars that depend on each build
+ for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
+ common.env[n] = ndk_path
+
+ common.reset_env_path()
+ # Set up the current NDK to the PATH
+ common.add_to_env_path(ndk_path)
+
# Prepare the source code...
- root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
+ root_dir, srclibpaths = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
- extlib_dir, onserver)
+ extlib_dir, onserver, refresh)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
- if thisbuild['type'] == 'maven':
+ gradletasks = []
+ bmethod = build.build_method()
+ if bmethod == 'maven':
logging.info("Cleaning Maven project...")
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
- if '@' in thisbuild['maven']:
- maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1])
+ if '@' in build.maven:
+ maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
- elif thisbuild['type'] == 'gradle':
+ elif bmethod == 'gradle':
logging.info("Cleaning Gradle project...")
- cmd = [config['gradle'], 'clean']
+
+ if build.preassemble:
+ gradletasks += build.preassemble
+
+ flavours = build.gradle
+ if flavours == ['yes']:
+ flavours = []
+
+ flavours_cmd = ''.join([capitalize_intact(f) for f in flavours])
+
+ gradletasks += ['assemble' + flavours_cmd + 'Release']
adapt_gradle(build_dir)
for name, number, libpath in srclibpaths:
adapt_gradle(libpath)
+ cmd = [config['gradle']]
+ if build.gradleprops:
+ cmd += ['-P' + kv for kv in build.gradleprops]
+
+ cmd += ['clean']
+
p = FDroidPopen(cmd, cwd=root_dir)
- elif thisbuild['type'] == 'kivy':
+ elif bmethod == 'kivy':
pass
- elif thisbuild['type'] == 'ant':
+ elif bmethod == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
- (app['id'], thisbuild['version']), p.output)
+ (app.id, build.version), p.output)
for root, dirs, files in os.walk(build_dir):
- # Don't remove possibly necessary 'gradle' dirs if 'gradlew' is not there
- if 'gradlew' in files:
- logging.debug("Getting rid of Gradle wrapper stuff in %s" % root)
- os.remove(os.path.join(root, 'gradlew'))
- if 'gradlew.bat' in files:
- os.remove(os.path.join(root, 'gradlew.bat'))
- if 'gradle' in dirs:
- shutil.rmtree(os.path.join(root, 'gradle'))
-
- if not options.skipscan:
+
+ def del_dirs(dl):
+ for d in dl:
+ if d in dirs:
+ shutil.rmtree(os.path.join(root, d))
+
+ def del_files(fl):
+ for f in fl:
+ if f in files:
+ os.remove(os.path.join(root, f))
+
+ if 'build.gradle' in files:
+ # Even when running clean, gradle stores task/artifact caches in
+ # .gradle/ as binary files. To avoid overcomplicating the scanner,
+ # manually delete them, just like `gradle clean` should have removed
+ # the build/ dirs.
+ del_dirs(['build', '.gradle'])
+ del_files(['gradlew', 'gradlew.bat'])
+
+ if 'pom.xml' in files:
+ del_dirs(['target'])
+
+ if any(f in files for f in ['ant.properties', 'project.properties', 'build.xml']):
+ del_dirs(['bin', 'gen'])
+
+ if 'jni' in dirs:
+ del_dirs(['obj'])
+
+ if options.skipscan:
+ if build.scandelete:
+ raise BuildException("Refusing to skip source scan since scandelete is present")
+ else:
# Scan before building...
logging.info("Scanning source for common problems...")
- count = common.scan_source(build_dir, root_dir, thisbuild)
+ count = scanner.scan_source(build_dir, root_dir, build)
if count > 0:
if force:
- logging.warn('Scanner found %d problems:' % count)
+ logging.warn('Scanner found %d problems' % count)
else:
raise BuildException("Can't build due to %d errors while scanning" % count)
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
- tarname = common.getsrcname(app, thisbuild)
+ tarname = common.getsrcname(app, build)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
tarball.close()
# Run a build command if one is required...
- if thisbuild['build']:
+ if build.build:
logging.info("Running 'build' commands in %s" % root_dir)
- cmd = common.replace_config_vars(thisbuild['build'])
+ cmd = common.replace_config_vars(build.build, build)
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
- (app['id'], thisbuild['version']), p.output)
+ (app.id, build.version), p.output)
# Build native stuff if required...
- if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
+ if build.buildjni and build.buildjni != ['no']:
logging.info("Building the native code")
- jni_components = thisbuild['buildjni']
+ jni_components = build.buildjni
if jni_components == ['yes']:
jni_components = ['']
- cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"]
+ cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
else:
logging.info("Building native code in the main project")
- manifest = root_dir + '/' + d + '/AndroidManifest.xml'
+ manifest = os.path.join(root_dir, d, 'AndroidManifest.xml')
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
- raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.output)
+ raise BuildException("NDK build failed for %s:%s" % (app.id, build.version), p.output)
p = None
# Build the release...
- if thisbuild['type'] == 'maven':
+ if bmethod == 'maven':
logging.info("Building Maven project...")
- if '@' in thisbuild['maven']:
- maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1])
+ if '@' in build.maven:
+ maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
else:
maven_dir = root_dir
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true',
'package']
- if thisbuild['target']:
- target = thisbuild["target"].split('-')[1]
- FDroidPopen(['sed', '-i',
- 's@<platform>[0-9]*</platform>@<platform>'
- + target + '</platform>@g',
- 'pom.xml'],
- cwd=root_dir)
- if '@' in thisbuild['maven']:
- FDroidPopen(['sed', '-i',
- 's@<platform>[0-9]*</platform>@<platform>'
- + target + '</platform>@g',
- 'pom.xml'],
- cwd=maven_dir)
+ if build.target:
+ target = build.target.split('-')[1]
+ common.regsub_file(r'<platform>[0-9]*</platform>',
+ r'<platform>%s</platform>' % target,
+ os.path.join(root_dir, 'pom.xml'))
+ if '@' in build.maven:
+ common.regsub_file(r'<platform>[0-9]*</platform>',
+ r'<platform>%s</platform>' % target,
+ os.path.join(maven_dir, 'pom.xml'))
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
- elif thisbuild['type'] == 'kivy':
+ elif bmethod == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
- distdir = 'python-for-android/dist/fdroid'
+ distdir = os.path.join('python-for-android', 'dist', 'fdroid')
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
- cmd += ' ANDROIDNDK=' + config['ndk_path']
- cmd += ' ANDROIDNDKVER=r9'
+ cmd += ' ANDROIDNDK=' + ndk_path
+ cmd += ' ANDROIDNDKVER=' + build.ndk
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
- p = FDroidPopen(cmd, cwd='python-for-android', shell=True)
+ p = subprocess.Popen(cmd, cwd='python-for-android', shell=True)
if p.returncode != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
- if cid != app['id']:
+ if cid != app.id:
raise BuildException("Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
cmd = ['./build.py'
'--dir', root_dir,
'--name', bconfig.get('app', 'title'),
- '--package', app['id'],
+ '--package', app.id,
'--version', bconfig.get('app', 'version'),
'--orientation', orientation
]
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
- elif thisbuild['type'] == 'gradle':
+ elif bmethod == 'gradle':
logging.info("Building Gradle project...")
- flavours = thisbuild['gradle']
- if flavours == ['yes']:
- flavours = []
- commands = [config['gradle']]
- if thisbuild['preassemble']:
- commands += thisbuild['preassemble']
+ cmd = [config['gradle']]
+ if build.gradleprops:
+ cmd += ['-P' + kv for kv in build.gradleprops]
- flavours_cmd = ''.join(flavours)
- if flavours_cmd:
- flavours_cmd = flavours_cmd[0].upper() + flavours_cmd[1:]
+ cmd += gradletasks
- commands += ['assemble' + flavours_cmd + 'Release']
-
- # Avoid having to use lintOptions.abortOnError false
- if thisbuild['gradlepluginver'] >= LooseVersion('0.7'):
- with open(os.path.join(root_dir, 'build.gradle'), "a") as f:
- f.write("\nandroid { lintOptions { checkReleaseBuilds false } }\n")
-
- p = FDroidPopen(commands, cwd=root_dir)
+ p = FDroidPopen(cmd, cwd=root_dir)
- elif thisbuild['type'] == 'ant':
+ elif bmethod == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
- if thisbuild['antcommands']:
- cmd += thisbuild['antcommands']
+ if build.antcommands:
+ cmd += build.antcommands
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
- raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.output)
- logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id'])
+ raise BuildException("Build failed for %s:%s" % (app.id, build.version), p.output)
+ logging.info("Successfully built version " + build.version + ' of ' + app.id)
- if thisbuild['type'] == 'maven':
+ omethod = build.output_method()
+ if omethod == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if any(
a in line for a in ('.apk', '.ap_', '.jar'))])
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
- elif thisbuild['type'] == 'kivy':
- src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format(
- bconfig.get('app', 'title'), bconfig.get('app', 'version'))
- elif thisbuild['type'] == 'gradle':
+ elif omethod == 'kivy':
+ src = os.path.join('python-for-android', 'dist', 'default', 'bin',
+ '{0}-{1}-release.apk'.format(
+ bconfig.get('app', 'title'),
+ bconfig.get('app', 'version')))
+ elif omethod == 'gradle':
+ src = None
+ for apks_dir in [
+ os.path.join(root_dir, 'build', 'outputs', 'apk'),
+ os.path.join(root_dir, 'build', 'apk'),
+ ]:
+ for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
+ apks = glob.glob(os.path.join(apks_dir, apkglob))
+
+ if len(apks) > 1:
+ raise BuildException('More than one resulting apks found in %s' % apks_dir,
+ '\n'.join(apks))
+ if len(apks) == 1:
+ src = apks[0]
+ break
+ if src is not None:
+ break
- if thisbuild['gradlepluginver'] >= LooseVersion('0.11'):
- apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk')
- else:
- apks_dir = os.path.join(root_dir, 'build', 'apk')
+ if src is None:
+ raise BuildException('Failed to find any output apks')
- apks = glob.glob(os.path.join(apks_dir, '*-release-unsigned.apk'))
- if len(apks) > 1:
- raise BuildException('More than one resulting apks found in %s' % apks_dir,
- '\n'.join(apks))
- if len(apks) < 1:
- raise BuildException('Failed to find gradle output in %s' % apks_dir)
- src = apks[0]
- elif thisbuild['type'] == 'ant':
+ elif omethod == 'ant':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
- elif thisbuild['type'] == 'raw':
- src = os.path.join(root_dir, thisbuild['output'])
- src = os.path.normpath(src)
+ elif omethod == 'raw':
+ globpath = os.path.join(root_dir, build.output)
+ apks = glob.glob(globpath)
+ if len(apks) > 1:
+ raise BuildException('Multiple apks match %s' % globpath, '\n'.join(apks))
+ if len(apks) < 1:
+ raise BuildException('No apks match %s' % globpath)
+ src = os.path.normpath(apks[0])
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
- if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
+ if build.buildjni and build.buildjni != ['no']:
if nativecode is None:
raise BuildException("Native code should have been built but none was packaged")
- if thisbuild['novcheck']:
- vercode = thisbuild['vercode']
- version = thisbuild['version']
+ if build.novcheck:
+ vercode = build.vercode
+ version = build.version
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
- if foundid != app['id']:
- raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id'])
+ if foundid != app.id:
+ raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id)
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
if index != -1:
version = version[:index]
- if (version != thisbuild['version'] or
- vercode != thisbuild['vercode']):
+ if (version != build.version or
+ vercode != build.vercode):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
- % (version, str(vercode), thisbuild['version'],
- str(thisbuild['vercode']))
+ % (version, str(vercode), build.version,
+ str(build.vercode))
)
# Add information for 'fdroid verify' to be able to reproduce the build
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
- dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
+ dest = os.path.join(output_dir, common.getapkname(app, build))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
os.path.join(output_dir, tarname))
-def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
- tmp_dir, repo_dir, vcs, test, server, force, onserver):
+def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
+ tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh):
"""
Build a particular version of an application, if it needs building.
:returns: True if the build was done, False if it wasn't necessary.
"""
- dest_apk = common.getapkname(app, thisbuild)
+ dest_apk = common.getapkname(app, build)
dest = os.path.join(output_dir, dest_apk)
dest_repo = os.path.join(repo_dir, dest_apk)
if os.path.exists(dest_also):
return False
- if thisbuild['disable'] and not options.force:
+ if build.disable and not options.force:
return False
logging.info("Building version %s (%s) of %s" % (
- thisbuild['version'], thisbuild['vercode'], app['id']))
+ build.version, build.vercode, app.id))
if server:
# When using server mode, still keep a local cache of the repo, by
# grabbing the source now.
- vcs.gotorevision(thisbuild['commit'])
+ vcs.gotorevision(build.commit)
- build_server(app, thisbuild, vcs, build_dir, output_dir, force)
+ build_server(app, build, vcs, build_dir, output_dir, force)
else:
- build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver)
+ build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
return True
def parse_commandline():
- """Parse the command line. Returns options, args."""
-
- parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
- parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
- parser.add_option("-q", "--quiet", action="store_true", default=False,
- help="Restrict output to warnings and errors")
- parser.add_option("-l", "--latest", action="store_true", default=False,
- help="Build only the latest version of each package")
- parser.add_option("-s", "--stop", action="store_true", default=False,
- help="Make the build stop on exceptions")
- parser.add_option("-t", "--test", action="store_true", default=False,
- help="Test mode - put output in the tmp directory only, and always build, even if the output already exists.")
- parser.add_option("--server", action="store_true", default=False,
- help="Use build server")
- parser.add_option("--resetserver", action="store_true", default=False,
- help="Reset and create a brand new build server, even if the existing one appears to be ok.")
- parser.add_option("--on-server", dest="onserver", action="store_true", default=False,
- help="Specify that we're running on the build server")
- parser.add_option("--skip-scan", dest="skipscan", action="store_true", default=False,
- help="Skip scanning the source code for binaries and other problems")
- parser.add_option("--no-tarball", dest="notarball", action="store_true", default=False,
- help="Don't create a source tarball, useful when testing a build")
- parser.add_option("-f", "--force", action="store_true", default=False,
- help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
- parser.add_option("-a", "--all", action="store_true", default=False,
- help="Build all applications available")
- parser.add_option("-w", "--wiki", default=False, action="store_true",
- help="Update the wiki")
- options, args = parser.parse_args()
+ """Parse the command line. Returns options, parser."""
+
+ parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
+ common.setup_global_opts(parser)
+ parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
+ parser.add_argument("-l", "--latest", action="store_true", default=False,
+ help="Build only the latest version of each package")
+ parser.add_argument("-s", "--stop", action="store_true", default=False,
+ help="Make the build stop on exceptions")
+ parser.add_argument("-t", "--test", action="store_true", default=False,
+ help="Test mode - put output in the tmp directory only, and always build, even if the output already exists.")
+ parser.add_argument("--server", action="store_true", default=False,
+ help="Use build server")
+ parser.add_argument("--resetserver", action="store_true", default=False,
+ help="Reset and create a brand new build server, even if the existing one appears to be ok.")
+ parser.add_argument("--on-server", dest="onserver", action="store_true", default=False,
+ help="Specify that we're running on the build server")
+ parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False,
+ help="Skip scanning the source code for binaries and other problems")
+ parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False,
+ help="Don't create a source tarball, useful when testing a build")
+ parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True,
+ help="Don't refresh the repository, useful when testing a build with no internet connection")
+ parser.add_argument("-f", "--force", action="store_true", default=False,
+ help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
+ parser.add_argument("-a", "--all", action="store_true", default=False,
+ help="Build all applications available")
+ parser.add_argument("-w", "--wiki", default=False, action="store_true",
+ help="Update the wiki")
+ options = parser.parse_args()
# Force --stop with --on-server to get correct exit code
if options.onserver:
options.stop = True
if options.force and not options.test:
- raise OptionError("Force is only allowed in test mode", "force")
+ parser.error("option %s: Force is only allowed in test mode" % "force")
- return options, args
+ return options, parser
options = None
config = None
global options, config
- options, args = parse_commandline()
- if not args and not options.all:
- raise OptionError("If you really want to build all the apps, use --all", "all")
+ options, parser = parse_commandline()
+
+ metadata_files = glob.glob('.fdroid.*[a-z]') # ignore files ending in ~
+ if os.path.isdir('metadata'):
+ pass
+ elif len(metadata_files) == 0:
+ raise FDroidException("No app metadata found, nothing to process!")
+ elif len(metadata_files) > 1:
+ raise FDroidException("Only one local metadata file allowed! Found: "
+ + " ".join(metadata_files))
+
+ if not options.appid and not options.all:
+ parser.error("option %s: If you really want to build all the apps, use --all" % "all")
config = common.read_config(options)
if config['build_server_always']:
options.server = True
if options.resetserver and not options.server:
- raise OptionError("Using --resetserver without --server makes no sense", "resetserver")
+ parser.error("option %s: Using --resetserver without --server makes no sense" % "resetserver")
log_dir = 'logs'
if not os.path.isdir(log_dir):
# Read all app and srclib metadata
allapps = metadata.read_metadata(xref=not options.onserver)
- apps = common.read_app_args(args, allapps, True)
+ apps = common.read_app_args(options.appid, allapps, True)
for appid, app in apps.items():
- if (app['Disabled'] and not options.force) or not app['Repo Type'] or not app['builds']:
+ if (app.Disabled and not options.force) or not app.RepoType or not app.builds:
del apps[appid]
if not apps:
if options.latest:
for app in apps.itervalues():
- for build in reversed(app['builds']):
- if build['disable'] and not options.force:
+ for build in reversed(app.builds):
+ if build.disable and not options.force:
continue
- app['builds'] = [build]
+ app.builds = [build]
break
if options.wiki:
first = True
- for thisbuild in app['builds']:
+ for build in app.builds:
wikilog = None
try:
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
- if app['Repo Type'] == 'srclib':
- build_dir = os.path.join('build', 'srclib', app['Repo'])
+ if app.RepoType == 'srclib':
+ build_dir = os.path.join('build', 'srclib', app.Repo)
else:
build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
logging.debug("Getting {0} vcs interface for {1}"
- .format(app['Repo Type'], app['Repo']))
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+ .format(app.RepoType, app.Repo))
+ vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
first = False
- logging.debug("Checking " + thisbuild['version'])
- if trybuild(app, thisbuild, build_dir, output_dir,
+ logging.debug("Checking " + build.version)
+ if trybuild(app, build, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test,
options.server, options.force,
- options.onserver):
+ options.onserver, options.refresh):
+
+ if app.Binaries is not None:
+ # This is an app where we build from source, and
+ # verify the apk contents against a developer's
+ # binary. We get that binary now, and save it
+ # alongside our built one in the 'unsigend'
+ # directory.
+ url = app.Binaries
+ url = url.replace('%v', build.version)
+ url = url.replace('%c', str(build.vercode))
+ logging.info("...retrieving " + url)
+ of = "{0}_{1}.apk.binary".format(app.id, build.vercode)
+ of = os.path.join(output_dir, of)
+ net.download_file(url, local_filename=of)
+
build_succeeded.append(app)
wikilog = "Build succeeded"
- except BuildException as be:
- logfile = open(os.path.join(log_dir, appid + '.log'), 'a+')
- logfile.write(str(be))
- logfile.close()
- print("Could not build app %s due to BuildException: %s" % (appid, be))
- if options.stop:
- sys.exit(1)
- failed_apps[appid] = be
- wikilog = be.get_wikitext()
except VCSException as vcse:
reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse)
logging.error("VCS error while building app %s: %s" % (
sys.exit(1)
failed_apps[appid] = vcse
wikilog = str(vcse)
+ except FDroidException as e:
+ with open(os.path.join(log_dir, appid + '.log'), 'a+') as f:
+ f.write(str(e))
+ logging.error("Could not build app %s: %s" % (appid, e))
+ if options.stop:
+ sys.exit(1)
+ failed_apps[appid] = e
+ wikilog = e.get_wikitext()
except Exception as e:
logging.error("Could not build app %s due to unknown error: %s" % (
appid, traceback.format_exc()))
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
- lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode']
+ lastbuildpage = appid + '/lastbuild_' + build.vercode
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')
logging.error("Error while attempting to publish build log")
for app in build_succeeded:
- logging.info("success: %s" % (app['id']))
+ logging.info("success: %s" % (app.id))
if not options.verbose:
for fa in failed_apps:
~ianmdlvl / fdroidserver.git / blobdiff