From: Daniel Martí Date: Tue, 21 Jun 2016 10:12:54 +0000 (+0000) Subject: Merge branch 'some-build-fixes' into 'master' X-Git-Tag: 0.7.0~44 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=d11135b44173adb25f3e3f50cb3888019a7f4fa6;hp=323ad5e0ad1df6c252a5e576be58a74c53015e14;p=fdroidserver.git Merge branch 'some-build-fixes' into 'master' Some build fixes This includes a couple of fixes discussed with @mvdan including removing the `ANDROID_NDK_HOME` env var from the buildserver and removing the default behavior to force the build-tools version. See merge request !136 --- diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 3c9b0a6a..9d4a54c0 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -3,26 +3,23 @@ set -e -if [ -z $ANDROID_NDK_HOME ]; then - echo "ANDROID_NDK_HOME env var must be set!" - exit 1 -fi +NDK_BASE=$1 -test -e $ANDROID_NDK_HOME || mkdir -p $ANDROID_NDK_HOME -cd $ANDROID_NDK_HOME +test -e $NDK_BASE || mkdir -p $NDK_BASE +cd $NDK_BASE -if [ ! -e $ANDROID_NDK_HOME/r9b ]; then +if [ ! -e $NDK_BASE/r9b ]; then if [ `uname -m` == 'x86_64' ] ; then SUFFIX='_64' else SUFFIX='' fi - tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 - tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 + tar xjf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 + tar xjf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 mv android-ndk-r9b r9b fi -if [ ! -e $ANDROID_NDK_HOME/r10e ]; then +if [ ! -e $NDK_BASE/r10e ]; then if [ `uname -m` == 'x86_64' ] ; then SUFFIX='_64' else @@ -32,5 +29,5 @@ if [ ! -e $ANDROID_NDK_HOME/r10e ]; then mv android-ndk-r10e r10e fi -chmod -R a+rX $ANDROID_NDK_HOME/ -find $ANDROID_NDK_HOME/ -type f -executable -print0 | xargs -0 chmod a+x +chmod -R a+rX $NDK_BASE/ +find $NDK_BASE/ -type f -executable -print0 | xargs -0 chmod a+x diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index 5560ecb1..907b2489 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -43,7 +43,7 @@ for f in `ls -1 build-tools*.zip`; do done ${ANDROID_HOME}/tools/android --silent update sdk --no-ui --all \ - --filter platform-tools,extra-android-m2repository${cached} < /dev/null < $bsenv echo export ANDROID_HOME=$1 >> $bsenv -echo export ANDROID_NDK_HOME=$2 >> $bsenv echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv chmod 0644 $bsenv diff --git a/examples/config.py b/examples/config.py index 56ed7b86..1a3367e8 100644 --- a/examples/config.py +++ b/examples/config.py @@ -25,6 +25,10 @@ # Build tools version to be used # build_tools = "23.0.3" +# Force all build to use the above version of build -tools, good for testing +# builds without having all of the possible build-tools installed. +# force_build_tools = True + # Command or path to binary for running Ant # ant = "ant" @@ -232,10 +236,10 @@ The repository of older versions of applications from the main demo repository. # --server option on dedicated secure build server hosts. # build_server_always = True -# By default, fdroid will use YAML and the custom .txt metadata formats. It +# By default, fdroid will use YAML .yml and the custom .txt metadata formats. It # is also possible to have metadata in JSON and XML by adding 'json' and # 'xml'. -# accepted_formats = ['txt', 'yaml'] +# accepted_formats = ['txt', 'yml'] # Limit in number of characters that fields can take up # Only the fields listed here are supported, defaults shown diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 7709a6bf..bad025a6 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -431,8 +431,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force): release_vm() -def adapt_gradle(build_dir): - filename = 'build.gradle' +def force_gradle_build_tools(build_dir, build_tools): for root, dirs, files in os.walk(build_dir): for filename in files: if not filename.endswith('.gradle'): @@ -440,9 +439,9 @@ def adapt_gradle(build_dir): path = os.path.join(root, filename) if not os.path.isfile(path): continue - logging.debug("Adapting %s at %s" % (filename, path)) + logging.debug("Forcing build-tools %s in %s" % (build_tools, path)) common.regsub_file(r"""(\s*)buildToolsVersion([\s=]+).*""", - r"""\1buildToolsVersion\2'%s'""" % config['build_tools'], + r"""\1buildToolsVersion\2'%s'""" % build_tools, path) @@ -512,9 +511,10 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, gradletasks += ['assemble' + flavours_cmd + 'Release'] - adapt_gradle(build_dir) - for name, number, libpath in srclibpaths: - adapt_gradle(libpath) + if config['force_build_tools']: + force_gradle_build_tools(build_dir, config['build_tools']) + for name, number, libpath in srclibpaths: + force_gradle_build_tools(libpath, config['build_tools']) cmd = [config['gradle']] if build.gradleprops: diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 17df02ae..f27177fe 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -59,6 +59,7 @@ default_config = { 'r10e': "$ANDROID_NDK", }, 'build_tools': "23.0.3", + 'force_build_tools': False, 'java_paths': None, 'ant': "ant", 'mvn3': "mvn", diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index ded6cd1a..9ba9b1fd 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -783,9 +783,10 @@ def read_metadata(xref=True): + glob.glob('.fdroid.json') + glob.glob('.fdroid.xml') + glob.glob('.fdroid.yml')): + packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if packageName in apps: + raise MetaDataException("Found multiple metadata files for " + packageName) app = parse_metadata(metadatapath) - if app.id in apps: - raise MetaDataException("Found multiple metadata files for " + app.id) check_metadata(app) apps[app.id] = app diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index 81d88e9b..d32e1442 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -46,7 +46,7 @@ cd $WORKSPACE echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py -./makebuildserver +./makebuildserver --verbose # this can be handled in the jenkins job, or here: if [ -e fdroiddata ]; then diff --git a/makebuildserver b/makebuildserver index 05616452..44c43256 100755 --- a/makebuildserver +++ b/makebuildserver @@ -377,10 +377,8 @@ if config['apt_package_cache']: vagrantfile += """ - config.vm.provision "shell" do |shell| - shell.path = "setup-env-vars" - shell.args = ["/home/vagrant/android-sdk", "/home/vagrant/android-ndk"] - end + config.vm.provision "shell", path: "setup-env-vars", + args: ["/home/vagrant/android-sdk"] config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" @@ -397,7 +395,8 @@ vagrantfile += """ end config.vm.provision "shell", path: "provision-android-sdk" - config.vm.provision "shell", path: "provision-android-ndk" + config.vm.provision "shell", path: "provision-android-ndk", + args: ["/home/vagrant/android-ndk"] config.vm.provision "shell", path: "provision-gradle" config.vm.provision "file", source: "gradle", destination: "/opt/gradle/bin/gradle" diff --git a/tests/build.TestCase b/tests/build.TestCase index 369c3836..9391a042 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -46,7 +46,7 @@ class BuildTest(unittest.TestCase): self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.isfile(path)) - def test_adapt_gradle(self): + def test_force_gradle_build_tools(self): testsbase = os.path.join(os.path.dirname(__file__), '..', '.testfiles') if not os.path.exists(testsbase): os.makedirs(testsbase) @@ -54,9 +54,7 @@ class BuildTest(unittest.TestCase): shutil.copytree(os.path.join(os.path.dirname(__file__), 'source-files'), os.path.join(testsdir, 'source-files')) teststring = 'FAKE_VERSION_FOR_TESTING' - fdroidserver.build.config = {} - fdroidserver.build.config['build_tools'] = teststring - fdroidserver.build.adapt_gradle(testsdir) + fdroidserver.build.force_gradle_build_tools(testsdir, teststring) pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8')) for p in ('source-files/fdroid/fdroidclient/build.gradle', 'source-files/Zillode/syncthing-silk/build.gradle', @@ -67,6 +65,7 @@ class BuildTest(unittest.TestCase): filedata = f.read() self.assertIsNotNone(pattern.search(filedata)) + if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False,