import json
from ConfigParser import ConfigParser
from argparse import ArgumentParser
-from distutils.version import LooseVersion
import logging
import common
# Note that 'force' here also implies test mode.
-def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
+def build_server(app, build, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
try:
except NameError:
raise BuildException("Paramiko is required to use the buildserver")
if options.verbose:
- logging.getLogger("paramiko").setLevel(logging.DEBUG)
+ logging.getLogger("paramiko").setLevel(logging.INFO)
else:
logging.getLogger("paramiko").setLevel(logging.WARN)
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
- ftp.put(os.path.join('metadata', app['id'] + '.txt'),
- app['id'] + '.txt')
+ ftp.put(os.path.join('metadata', app.id + '.txt'),
+ app.id + '.txt')
# And patches if there are any...
- if os.path.exists(os.path.join('metadata', app['id'])):
- send_dir(os.path.join('metadata', app['id']))
+ if os.path.exists(os.path.join('metadata', app.id)):
+ send_dir(os.path.join('metadata', app.id))
ftp.chdir(homedir)
# Create the build directory...
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
- if thisbuild['extlibs']:
+ if build.extlibs:
ftp.chdir(homedir + '/build/extlib')
- for lib in thisbuild['extlibs']:
+ for lib in build.extlibs:
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
- if thisbuild['srclibs']:
- for lib in thisbuild['srclibs']:
+ if build.srclibs:
+ for lib in build.srclibs:
srclibpaths.append(
common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False))
# (no need if it's a srclib)
if (not basesrclib) and os.path.exists(build_dir):
ftp.chdir(homedir + '/build')
- fv = '.fdroidvcs-' + app['id']
+ fv = '.fdroidvcs-' + app.id
ftp.put(os.path.join('build', fv), fv)
send_dir(build_dir)
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
- cmdline += " %s:%s" % (app['id'], thisbuild['vercode'])
+ cmdline += " %s:%s" % (app.id, build.vercode)
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
while not chan.exit_status_ready():
if returncode != 0:
raise BuildException(
"Build.py failed on server for {0}:{1}".format(
- app['id'], thisbuild['version']), output)
+ app.id, build.version), output)
# Retrieve the built files...
logging.info("Retrieving build output...")
ftp.chdir(homedir + '/tmp')
else:
ftp.chdir(homedir + '/unsigned')
- apkfile = common.getapkname(app, thisbuild)
- tarball = common.getsrcname(app, thisbuild)
+ apkfile = common.getapkname(app, build)
+ tarball = common.getsrcname(app, build)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
if not options.notarball:
except:
raise BuildException(
"Build failed for %s:%s - missing output files".format(
- app['id'], thisbuild['version']), output)
+ app.id, build.version), output)
ftp.close()
finally:
if not os.path.isfile(path):
continue
logging.debug("Adapting %s at %s" % (filename, path))
- common.regsub_file(r"""(\s*)buildToolsVersion([\s=]+)['"].*""",
+ common.regsub_file(r"""(\s*)buildToolsVersion([\s=]+).*""",
r"""\1buildToolsVersion\2'%s'""" % config['build_tools'],
path)
return string[0].upper() + string[1:]
-def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
+def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally."""
- if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
- if not thisbuild['ndk_path']:
- logging.critical("Android NDK version '%s' could not be found!" % thisbuild['ndk'])
+ ndk_path = build.ndk_path()
+ if build.buildjni and build.buildjni != ['no']:
+ if not ndk_path:
+ logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r10e')
logging.critical("Configured versions:")
for k, v in config['ndk_paths'].iteritems():
if k.endswith("_orig"):
continue
logging.critical(" %s: %s" % (k, v))
sys.exit(3)
- elif not os.path.isdir(thisbuild['ndk_path']):
- logging.critical("Android NDK '%s' is not a directory!" % thisbuild['ndk_path'])
+ elif not os.path.isdir(ndk_path):
+ logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
sys.exit(3)
# Set up environment vars that depend on each build
for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
- common.env[n] = thisbuild['ndk_path']
+ common.env[n] = ndk_path
common.reset_env_path()
# Set up the current NDK to the PATH
- common.add_to_env_path(thisbuild['ndk_path'])
+ common.add_to_env_path(ndk_path)
# Prepare the source code...
- root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
+ root_dir, srclibpaths = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, onserver, refresh)
# different from the default ones
p = None
gradletasks = []
- if thisbuild['type'] == 'maven':
+ method = build.method()
+ if method == 'maven':
logging.info("Cleaning Maven project...")
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
- if '@' in thisbuild['maven']:
- maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1])
+ if '@' in build.maven:
+ maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
- elif thisbuild['type'] == 'gradle':
+ elif method == 'gradle':
logging.info("Cleaning Gradle project...")
- if thisbuild['preassemble']:
- gradletasks += thisbuild['preassemble']
+ if build.preassemble:
+ gradletasks += build.preassemble
- flavours = thisbuild['gradle']
+ flavours = build.gradle
if flavours == ['yes']:
flavours = []
adapt_gradle(libpath)
cmd = [config['gradle']]
- if thisbuild['gradleprops']:
- cmd += ['-P'+kv for kv in thisbuild['gradleprops']]
+ if build.gradleprops:
+ cmd += ['-P'+kv for kv in build.gradleprops]
for task in gradletasks:
parts = task.split(':')
p = FDroidPopen(cmd, cwd=root_dir)
- elif thisbuild['type'] == 'kivy':
+ elif method == 'kivy':
pass
- elif thisbuild['type'] == 'ant':
+ elif method == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
- (app['id'], thisbuild['version']), p.output)
+ (app.id, build.version), p.output)
for root, dirs, files in os.walk(build_dir):
del_dirs(['obj'])
if options.skipscan:
- if thisbuild['scandelete']:
+ if build.scandelete:
raise BuildException("Refusing to skip source scan since scandelete is present")
else:
# Scan before building...
logging.info("Scanning source for common problems...")
- count = scanner.scan_source(build_dir, root_dir, thisbuild)
+ count = scanner.scan_source(build_dir, root_dir, build)
if count > 0:
if force:
logging.warn('Scanner found %d problems' % count)
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
- tarname = common.getsrcname(app, thisbuild)
+ tarname = common.getsrcname(app, build)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
tarball.close()
# Run a build command if one is required...
- if thisbuild['build']:
+ if build.build:
logging.info("Running 'build' commands in %s" % root_dir)
- cmd = common.replace_config_vars(thisbuild['build'], thisbuild)
+ cmd = common.replace_config_vars(build.build, build)
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
- (app['id'], thisbuild['version']), p.output)
+ (app.id, build.version), p.output)
# Build native stuff if required...
- if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
+ if build.buildjni and build.buildjni != ['no']:
logging.info("Building the native code")
- jni_components = thisbuild['buildjni']
+ jni_components = build.buildjni
if jni_components == ['yes']:
jni_components = ['']
- cmd = [os.path.join(thisbuild['ndk_path'], "ndk-build"), "-j1"]
+ cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
- raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.output)
+ raise BuildException("NDK build failed for %s:%s" % (app.id, build.version), p.output)
p = None
# Build the release...
- if thisbuild['type'] == 'maven':
+ if method == 'maven':
logging.info("Building Maven project...")
- if '@' in thisbuild['maven']:
- maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1])
+ if '@' in build.maven:
+ maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
else:
maven_dir = root_dir
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true',
'package']
- if thisbuild['target']:
- target = thisbuild["target"].split('-')[1]
+ if build.target:
+ target = build.target.split('-')[1]
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(root_dir, 'pom.xml'))
- if '@' in thisbuild['maven']:
+ if '@' in build.maven:
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(maven_dir, 'pom.xml'))
bindir = os.path.join(root_dir, 'target')
- elif thisbuild['type'] == 'kivy':
+ elif method == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
- cmd += ' ANDROIDNDK=' + thisbuild['ndk_path']
- cmd += ' ANDROIDNDKVER=' + thisbuild['ndk']
+ cmd += ' ANDROIDNDK=' + ndk_path
+ cmd += ' ANDROIDNDKVER=' + build.ndk
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
- if cid != app['id']:
+ if cid != app.id:
raise BuildException("Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
cmd = ['./build.py'
'--dir', root_dir,
'--name', bconfig.get('app', 'title'),
- '--package', app['id'],
+ '--package', app.id,
'--version', bconfig.get('app', 'version'),
'--orientation', orientation
]
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
- elif thisbuild['type'] == 'gradle':
+ elif method == 'gradle':
logging.info("Building Gradle project...")
- # Avoid having to use lintOptions.abortOnError false
- if thisbuild['gradlepluginver'] >= LooseVersion('0.7'):
- with open(os.path.join(root_dir, 'build.gradle'), "a") as f:
- f.write("\nandroid { lintOptions { checkReleaseBuilds false } }\n")
-
cmd = [config['gradle']]
- if thisbuild['gradleprops']:
- cmd += ['-P'+kv for kv in thisbuild['gradleprops']]
+ if build.gradleprops:
+ cmd += ['-P'+kv for kv in build.gradleprops]
cmd += gradletasks
p = FDroidPopen(cmd, cwd=root_dir)
- elif thisbuild['type'] == 'ant':
+ elif method == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
- if thisbuild['antcommands']:
- cmd += thisbuild['antcommands']
+ if build.antcommands:
+ cmd += build.antcommands
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
- raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.output)
- logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id'])
+ raise BuildException("Build failed for %s:%s" % (app.id, build.version), p.output)
+ logging.info("Successfully built version " + build.version + ' of ' + app.id)
- if thisbuild['type'] == 'maven':
+ if method == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if any(
a in line for a in ('.apk', '.ap_', '.jar'))])
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
- elif thisbuild['type'] == 'kivy':
+ elif method == 'kivy':
src = os.path.join('python-for-android', 'dist', 'default', 'bin',
'{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'),
bconfig.get('app', 'version')))
- elif thisbuild['type'] == 'gradle':
+ elif method == 'gradle':
+ src = None
+ for apks_dir in [
+ os.path.join(root_dir, 'build', 'outputs', 'apk'),
+ os.path.join(root_dir, 'build', 'apk'),
+ ]:
+ for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
+ apks = glob.glob(os.path.join(apks_dir, apkglob))
+
+ if len(apks) > 1:
+ raise BuildException('More than one resulting apks found in %s' % apks_dir,
+ '\n'.join(apks))
+ if len(apks) == 1:
+ src = apks[0]
+ break
+ if src is not None:
+ break
- if thisbuild['gradlepluginver'] >= LooseVersion('0.11'):
- apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk')
- else:
- apks_dir = os.path.join(root_dir, 'build', 'apk')
-
- apks = glob.glob(os.path.join(apks_dir, '*-release-unsigned.apk'))
- if len(apks) > 1:
- raise BuildException('More than one resulting apks found in %s' % apks_dir,
- '\n'.join(apks))
- if len(apks) < 1:
- raise BuildException('Failed to find gradle output in %s' % apks_dir)
- src = apks[0]
- elif thisbuild['type'] == 'ant':
+ if src is None:
+ raise BuildException('Failed to find any output apks')
+
+ elif method == 'ant':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
- elif thisbuild['type'] == 'raw':
- src = os.path.join(root_dir, thisbuild['output'])
+ elif method == 'raw':
+ src = os.path.join(root_dir, build.output)
src = os.path.normpath(src)
# Make sure it's not debuggable...
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
- if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
+ if build.buildjni and build.buildjni != ['no']:
if nativecode is None:
raise BuildException("Native code should have been built but none was packaged")
- if thisbuild['novcheck']:
- vercode = thisbuild['vercode']
- version = thisbuild['version']
+ if build.novcheck:
+ vercode = build.vercode
+ version = build.version
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
- if foundid != app['id']:
- raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id'])
+ if foundid != app.id:
+ raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id)
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
if index != -1:
version = version[:index]
- if (version != thisbuild['version'] or
- vercode != thisbuild['vercode']):
+ if (version != build.version or
+ vercode != build.vercode):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
- % (version, str(vercode), thisbuild['version'],
- str(thisbuild['vercode']))
+ % (version, str(vercode), build.version,
+ str(build.vercode))
)
# Add information for 'fdroid verify' to be able to reproduce the build
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
- dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
+ dest = os.path.join(output_dir, common.getapkname(app, build))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
os.path.join(output_dir, tarname))
-def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
+def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh):
"""
Build a particular version of an application, if it needs building.
:returns: True if the build was done, False if it wasn't necessary.
"""
- dest_apk = common.getapkname(app, thisbuild)
+ dest_apk = common.getapkname(app, build)
dest = os.path.join(output_dir, dest_apk)
dest_repo = os.path.join(repo_dir, dest_apk)
if os.path.exists(dest_also):
return False
- if thisbuild['disable'] and not options.force:
+ if build.disable and not options.force:
return False
logging.info("Building version %s (%s) of %s" % (
- thisbuild['version'], thisbuild['vercode'], app['id']))
+ build.version, build.vercode, app.id))
if server:
# When using server mode, still keep a local cache of the repo, by
# grabbing the source now.
- vcs.gotorevision(thisbuild['commit'])
+ vcs.gotorevision(build.commit)
- build_server(app, thisbuild, vcs, build_dir, output_dir, force)
+ build_server(app, build, vcs, build_dir, output_dir, force)
else:
- build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
+ build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
return True
apps = common.read_app_args(options.appid, allapps, True)
for appid, app in apps.items():
- if (app['Disabled'] and not options.force) or not app['Repo Type'] or not app['builds']:
+ if (app.Disabled and not options.force) or not app.RepoType or not app.builds:
del apps[appid]
if not apps:
if options.latest:
for app in apps.itervalues():
- for build in reversed(app['builds']):
- if build['disable'] and not options.force:
+ for build in reversed(app.builds):
+ if build.disable and not options.force:
continue
- app['builds'] = [build]
+ app.builds = [build]
break
if options.wiki:
first = True
- for thisbuild in app['builds']:
+ for build in app.builds:
wikilog = None
try:
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
- if app['Repo Type'] == 'srclib':
- build_dir = os.path.join('build', 'srclib', app['Repo'])
+ if app.RepoType == 'srclib':
+ build_dir = os.path.join('build', 'srclib', app.Repo)
else:
build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
logging.debug("Getting {0} vcs interface for {1}"
- .format(app['Repo Type'], app['Repo']))
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+ .format(app.RepoType, app.Repo))
+ vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
first = False
- logging.debug("Checking " + thisbuild['version'])
- if trybuild(app, thisbuild, build_dir, output_dir,
+ logging.debug("Checking " + build.version)
+ if trybuild(app, build, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test,
options.server, options.force,
options.onserver, options.refresh):
- if app.get('Binaries', None):
+ if app.Binaries is not None:
# This is an app where we build from source, and
# verify the apk contents against a developer's
# binary. We get that binary now, and save it
# alongside our built one in the 'unsigend'
# directory.
- url = app['Binaries']
- url = url.replace('%v', thisbuild['version'])
- url = url.replace('%c', str(thisbuild['vercode']))
+ url = app.Binaries
+ url = url.replace('%v', build.version)
+ url = url.replace('%c', str(build.vercode))
logging.info("...retrieving " + url)
- of = "{0}_{1}.apk.binary".format(app['id'], thisbuild['vercode'])
+ of = "{0}_{1}.apk.binary".format(app.id, build.vercode)
of = os.path.join(output_dir, of)
net.download_file(url, local_filename=of)
build_succeeded.append(app)
wikilog = "Build succeeded"
- except BuildException as be:
- with open(os.path.join(log_dir, appid + '.log'), 'a+') as f:
- f.write(str(be))
- print("Could not build app %s due to BuildException: %s" % (appid, be))
- if options.stop:
- sys.exit(1)
- failed_apps[appid] = be
- wikilog = be.get_wikitext()
except VCSException as vcse:
reason = str(vcse).split('\n', 1)[0] if options.verbose else str(vcse)
logging.error("VCS error while building app %s: %s" % (
sys.exit(1)
failed_apps[appid] = vcse
wikilog = str(vcse)
+ except FDroidException as e:
+ with open(os.path.join(log_dir, appid + '.log'), 'a+') as f:
+ f.write(str(e))
+ logging.error("Could not build app %s: %s" % (appid, e))
+ if options.stop:
+ sys.exit(1)
+ failed_apps[appid] = e
+ wikilog = e.get_wikitext()
except Exception as e:
logging.error("Could not build app %s due to unknown error: %s" % (
appid, traceback.format_exc()))
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
- lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode']
+ lastbuildpage = appid + '/lastbuild_' + build.vercode
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')
logging.error("Error while attempting to publish build log")
for app in build_succeeded:
- logging.info("success: %s" % (app['id']))
+ logging.info("success: %s" % (app.id))
if not options.verbose:
for fa in failed_apps: