Merge branch 'some-build-fixes' into 'master'

Some build fixes

This includes a couple of fixes discussed with @mvdan including removing the `ANDROID_NDK_HOME` env var from the buildserver and removing the default behavior to force the build-tools version.

See merge request !136

diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk
index 3c9b0a6a704fec761ef285daafed737bfc584d52..9d4a54c0ba2fb32ee75ccb577f3f9e5e74598cdc 100644 (file)
--- a/buildserver/provision-android-ndk
+++ b/buildserver/provision-android-ndk
@@ -3,26 +3,23 @@
 
 set -e
 
-if [ -z $ANDROID_NDK_HOME ]; then
-    echo "ANDROID_NDK_HOME env var must be set!"
-    exit 1
-fi
+NDK_BASE=$1
 
-test -e $ANDROID_NDK_HOME || mkdir -p $ANDROID_NDK_HOME
-cd $ANDROID_NDK_HOME
+test -e $NDK_BASE || mkdir -p $NDK_BASE
+cd $NDK_BASE
 
-if [ ! -e $ANDROID_NDK_HOME/r9b ]; then
+if [ ! -e $NDK_BASE/r9b ]; then
     if [ `uname -m` == 'x86_64' ] ; then
         SUFFIX='_64'
     else
         SUFFIX=''
     fi
-    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
-    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
+    tar xjf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
+    tar xjf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
     mv android-ndk-r9b r9b
 fi
 
-if [ ! -e $ANDROID_NDK_HOME/r10e ]; then
+if [ ! -e $NDK_BASE/r10e ]; then
     if [ `uname -m` == 'x86_64' ] ; then
        SUFFIX='_64'
     else
@@ -32,5 +29,5 @@ if [ ! -e $ANDROID_NDK_HOME/r10e ]; then
     mv android-ndk-r10e r10e
 fi
 
-chmod -R a+rX $ANDROID_NDK_HOME/
-find $ANDROID_NDK_HOME/ -type f -executable -print0 | xargs -0 chmod a+x
+chmod -R a+rX $NDK_BASE/
+find $NDK_BASE/ -type f -executable -print0 | xargs -0 chmod a+x

diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk
index 5560ecb18cf9b39d3f062de829c1f4643623ca10..907b2489abf721e9645b38d635ebc617488d3c8d 100644 (file)
--- a/buildserver/provision-android-sdk
+++ b/buildserver/provision-android-sdk
@@ -43,7 +43,7 @@ for f in `ls -1 build-tools*.zip`; do
 done
 
 ${ANDROID_HOME}/tools/android --silent update sdk --no-ui --all \
-    --filter platform-tools,extra-android-m2repository${cached} <<EOH
+    --filter platform-tools,extra-android-m2repository${cached} > /dev/null <<EOH
 y
 
 EOH

diff --git a/buildserver/setup-env-vars b/buildserver/setup-env-vars
index 83d433eb0f958783d52db70795afdc0014741a2b..00220a234ccca6f5aa00b7bcf45966266d178ae2 100644 (file)
--- a/buildserver/setup-env-vars
+++ b/buildserver/setup-env-vars
@@ -10,7 +10,6 @@ bsenv=/etc/profile.d/bsenv.sh
 echo "# generated on "`date` > $bsenv
 
 echo export ANDROID_HOME=$1 >> $bsenv
-echo export ANDROID_NDK_HOME=$2 >> $bsenv
 echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv
 
 chmod 0644 $bsenv

diff --git a/examples/config.py b/examples/config.py
index 56ed7b86cd3c4172da07c5c9316a97bc5a771a06..1a3367e86a9fc2c24e1366ea941e899c4ca78aa3 100644 (file)
--- a/examples/config.py
+++ b/examples/config.py
@@ -25,6 +25,10 @@
 # Build tools version to be used
 # build_tools = "23.0.3"
 
+# Force all build to use the above version of build -tools, good for testing
+# builds without having all of the possible build-tools installed.
+# force_build_tools = True
+
 # Command or path to binary for running Ant
 # ant = "ant"
 
@@ -232,10 +236,10 @@ The repository of older versions of applications from the main demo repository.
 # --server option on dedicated secure build server hosts.
 # build_server_always = True
 
-# By default, fdroid will use YAML and the custom .txt metadata formats. It
+# By default, fdroid will use YAML .yml and the custom .txt metadata formats. It
 # is also possible to have metadata in JSON and XML by adding 'json' and
 # 'xml'.
-# accepted_formats = ['txt', 'yaml']
+# accepted_formats = ['txt', 'yml']
 
 # Limit in number of characters that fields can take up
 # Only the fields listed here are supported, defaults shown

diff --git a/fdroidserver/build.py b/fdroidserver/build.py
index 7709a6bf104772ba1c16afe947b1e34d8bf0745e..bad025a6d9a37258aedad1bada262e1a82301f6e 100644 (file)
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@ -431,8 +431,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force):
         release_vm()
 
 
-def adapt_gradle(build_dir):
-    filename = 'build.gradle'
+def force_gradle_build_tools(build_dir, build_tools):
     for root, dirs, files in os.walk(build_dir):
         for filename in files:
             if not filename.endswith('.gradle'):
@@ -440,9 +439,9 @@ def adapt_gradle(build_dir):
             path = os.path.join(root, filename)
             if not os.path.isfile(path):
                 continue
-            logging.debug("Adapting %s at %s" % (filename, path))
+            logging.debug("Forcing build-tools %s in %s" % (build_tools, path))
             common.regsub_file(r"""(\s*)buildToolsVersion([\s=]+).*""",
-                               r"""\1buildToolsVersion\2'%s'""" % config['build_tools'],
+                               r"""\1buildToolsVersion\2'%s'""" % build_tools,
                                path)
 
 
@@ -512,9 +511,10 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
 
         gradletasks += ['assemble' + flavours_cmd + 'Release']
 
-        adapt_gradle(build_dir)
-        for name, number, libpath in srclibpaths:
-            adapt_gradle(libpath)
+        if config['force_build_tools']:
+            force_gradle_build_tools(build_dir, config['build_tools'])
+            for name, number, libpath in srclibpaths:
+                force_gradle_build_tools(libpath, config['build_tools'])
 
         cmd = [config['gradle']]
         if build.gradleprops:

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 17df02aec47117e0b360763f6f349d90a3406595..f27177fe1f4ad44d28a3add0f8d750c3c36398f6 100644 (file)
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -59,6 +59,7 @@ default_config = {
         'r10e': "$ANDROID_NDK",
     },
     'build_tools': "23.0.3",
+    'force_build_tools': False,
     'java_paths': None,
     'ant': "ant",
     'mvn3': "mvn",

diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py
index ded6cd1a4057c9e345f715ab89a7b5fc9b534339..9ba9b1fdef17d8a1ecb0acb9c2a9c5dd418d80c6 100644 (file)
--- a/fdroidserver/metadata.py
+++ b/fdroidserver/metadata.py
@@ -783,9 +783,10 @@ def read_metadata(xref=True):
                                + glob.glob('.fdroid.json')
                                + glob.glob('.fdroid.xml')
                                + glob.glob('.fdroid.yml')):
+        packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+        if packageName in apps:
+            raise MetaDataException("Found multiple metadata files for " + packageName)
         app = parse_metadata(metadatapath)
-        if app.id in apps:
-            raise MetaDataException("Found multiple metadata files for " + app.id)
         check_metadata(app)
         apps[app.id] = app
 

diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver
index 81d88e9bdb1426040713048338ab166e2416e91b..d32e14423ac642343cfe661f954c0e9ccf1adcb0 100755 (executable)
--- a/jenkins-build-makebuildserver
+++ b/jenkins-build-makebuildserver
@@ -46,7 +46,7 @@ cd $WORKSPACE
 echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
 echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
 echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
-./makebuildserver
+./makebuildserver --verbose
 
 # this can be handled in the jenkins job, or here:
 if [ -e fdroiddata ]; then

diff --git a/makebuildserver b/makebuildserver
index 0561645263e20855735ebf0e40f4768b78bb19a1..44c432569c029548b1b232757453f1e6f0627872 100755 (executable)
--- a/makebuildserver
+++ b/makebuildserver
@@ -377,10 +377,8 @@ if config['apt_package_cache']:
 
 vagrantfile += """
 
-  config.vm.provision "shell" do |shell|
-    shell.path = "setup-env-vars"
-    shell.args = ["/home/vagrant/android-sdk", "/home/vagrant/android-ndk"]
-  end
+  config.vm.provision "shell", path: "setup-env-vars",
+    args: ["/home/vagrant/android-sdk"]
 
   config.vm.provision :chef_solo do |chef|
     chef.cookbooks_path = "cookbooks"
@@ -397,7 +395,8 @@ vagrantfile += """
   end
 
   config.vm.provision "shell", path: "provision-android-sdk"
-  config.vm.provision "shell", path: "provision-android-ndk"
+  config.vm.provision "shell", path: "provision-android-ndk",
+    args: ["/home/vagrant/android-ndk"]
   config.vm.provision "shell", path: "provision-gradle"
   config.vm.provision "file", source: "gradle",
     destination: "/opt/gradle/bin/gradle"

diff --git a/tests/build.TestCase b/tests/build.TestCase
index 369c3836f250996e2ee0cd995e760aeb4b0f6ebb..9391a0423e01ef768621ce03d29d08e3772256f1 100755 (executable)
--- a/tests/build.TestCase
+++ b/tests/build.TestCase
@@ -46,7 +46,7 @@ class BuildTest(unittest.TestCase):
                 self.assertTrue(os.path.exists(path))
                 self.assertTrue(os.path.isfile(path))
 
-    def test_adapt_gradle(self):
+    def test_force_gradle_build_tools(self):
         testsbase = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
         if not os.path.exists(testsbase):
             os.makedirs(testsbase)
@@ -54,9 +54,7 @@ class BuildTest(unittest.TestCase):
         shutil.copytree(os.path.join(os.path.dirname(__file__), 'source-files'),
                         os.path.join(testsdir, 'source-files'))
         teststring = 'FAKE_VERSION_FOR_TESTING'
-        fdroidserver.build.config = {}
-        fdroidserver.build.config['build_tools'] = teststring
-        fdroidserver.build.adapt_gradle(testsdir)
+        fdroidserver.build.force_gradle_build_tools(testsdir, teststring)
         pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
         for p in ('source-files/fdroid/fdroidclient/build.gradle',
                   'source-files/Zillode/syncthing-silk/build.gradle',
@@ -67,6 +65,7 @@ class BuildTest(unittest.TestCase):
                 filedata = f.read()
             self.assertIsNotNone(pattern.search(filedata))
 
+
 if __name__ == "__main__":
     parser = optparse.OptionParser()
     parser.add_option("-v", "--verbose", action="store_true", default=False,