chiark / gitweb /
Merge branch 'update-vagrantfile' into 'master'
authorCiaran Gultnieks <ciaran@ciarang.com>
Fri, 7 Nov 2014 14:44:58 +0000 (14:44 +0000)
committerCiaran Gultnieks <ciaran@ciarang.com>
Fri, 7 Nov 2014 14:44:58 +0000 (14:44 +0000)
Update Vagrantfile and docs to clarify v1.4.3 is ok

Saw in the server docs that we were recommending 1.3.x and saying 1.4.x was broken. I've confirmed that 1.4.x works, and updated things accordingly. Higher version might work, but figured minimal change to build stuff was best :)

See merge request !24

31 files changed:
MANIFEST.in
buildserver/config.buildserver.py
buildserver/cookbooks/android-sdk/recipes/default.rb
buildserver/cookbooks/fdroidbuild-general/recipes/default.rb
buildserver/cookbooks/gradle/recipes/default.rb
buildserver/cookbooks/gradle/recipes/gradle
docs/fdroid.texi
examples/config.py
fdroidserver/build.py
fdroidserver/checkupdates.py
fdroidserver/common.py
fdroidserver/import.py
fdroidserver/init.py
fdroidserver/lint.py
fdroidserver/metadata.py
fdroidserver/publish.py
fdroidserver/stats.py
fdroidserver/update.py
fdroidserver/verify.py
jenkins-build
makebuildserver
setup.py
tests/getsig/getsig.java [moved from fdroidserver/getsig/getsig.java with 100% similarity]
tests/getsig/make.sh [moved from fdroidserver/getsig/make.sh with 100% similarity]
tests/getsig/run.sh [moved from fdroidserver/getsig/run.sh with 100% similarity]
tests/run-tests
tests/update.TestCase [new file with mode: 0755]
tests/urzip-badcert.apk [new file with mode: 0644]
tests/urzip-badsig.apk [new file with mode: 0644]
updateplugin [deleted file]
wp-fdroid/wp-fdroid.php

index 29dd42e47f386c4153b8c658e2d29a11dd5ae0b8..2cf1078c5ab82af0e14b80953d6b71aaeaaa3372 100644 (file)
@@ -24,11 +24,14 @@ include examples/config.py
 include examples/fdroid-icon.png
 include examples/makebs.config.py
 include examples/opensc-fdroid.cfg
-include fdroidserver/getsig/run.sh
-include fdroidserver/getsig/make.sh
-include fdroidserver/getsig/getsig.java
+include tests/getsig/run.sh
+include tests/getsig/make.sh
+include tests/getsig/getsig.java
+include tests/getsig/getsig.class
 include tests/run-tests
+include tests/update.TestCase
 include tests/urzip.apk
+include tests/urzip-badsig.apk
 include wp-fdroid/AndroidManifest.xml
 include wp-fdroid/android-permissions.php
 include wp-fdroid/readme.txt
index ffc9cca08d429d1389431e126f2aab86c8d52cb4..29fbe07d699fe1c912c9d03ce63ae65df9ac6436 100644 (file)
@@ -1,6 +1,6 @@
 sdk_path = "/home/vagrant/android-sdk"
 ndk_path = "/home/vagrant/android-ndk"
-build_tools = "20.0.0"
+build_tools = "21.0.2"
 ant = "ant"
 mvn3 = "mvn"
 gradle = "gradle"
index 7074382c5d83c5d017f57532ab9bcf3ac38c8724..e21400376c360d998283581a0887651a9a17cac0 100644 (file)
@@ -26,7 +26,7 @@ end
 script "add_build_tools" do
   interpreter "bash"
   user user
-  ver = "20.0.0"
+  ver = "21.0.2"
   cwd "/tmp"
   code "
     if [ -f /vagrant/cache/build-tools/#{ver}.tar.gz ] ; then
@@ -66,7 +66,7 @@ end
 
 %w{android-3 android-4 android-5 android-6 android-7 android-8 android-9
    android-10 android-11 android-12 android-13 android-14 android-15
-   android-16 android-17 android-18 android-19 android-20
+   android-16 android-17 android-18 android-19 android-20 android-21
    extra-android-support extra-android-m2repository}.each do |sdk|
 
   script "add_sdk_#{sdk}" do
index 15c031e164f7d0a1a2e4014a0f0c8887de86e750..d289cdb450eead86d51075c3ee143b99d731456f 100644 (file)
@@ -5,7 +5,7 @@ execute "apt-get-update" do
   command "apt-get update"
 end
 
-%w{ant ant-contrib autoconf autopoint bison cmake expect libtool libsaxonb-java libssl1.0.0 libssl-dev maven openjdk-7-jdk javacc python python-magic git-core mercurial subversion bzr git-svn make perlmagick pkg-config zip yasm imagemagick gettext realpath transfig texinfo curl librsvg2-bin xsltproc vorbis-tools swig quilt faketime optipng python-gnupg}.each do |pkg|
+%w{ant ant-contrib autoconf autopoint bison cmake expect libtool libsaxonb-java libssl1.0.0 libssl-dev maven openjdk-7-jdk javacc python python-magic git-core mercurial subversion bzr git-svn make perlmagick pkg-config zip yasm imagemagick gettext realpath transfig texinfo curl librsvg2-bin xsltproc vorbis-tools swig quilt faketime optipng python-gnupg python3-gnupg}.each do |pkg|
   package pkg do
     action :install
   end
index 06055e243d3a79afa8aa98afd6b496f071f0fa43..792813d48e9a44242a27fd4fa63ca746a01f3016 100644 (file)
@@ -18,7 +18,7 @@ script "add-gradle-verdir" do
   not_if "test -d /opt/gradle/versions"
 end
 
-%w{1.4 1.6 1.7 1.8 1.9 1.10 1.11 1.12}.each do |ver|
+%w{1.4 1.6 1.7 1.8 1.9 1.10 1.11 1.12 2.1}.each do |ver|
   script "install-gradle-#{ver}" do
     cwd "/tmp"
     interpreter "bash"
index 89169b245af235339317eedbf0bfa401a37f8b0f..f8a381929dcb254606b801a8dc7b9cc8ce8c38af 100755 (executable)
@@ -23,11 +23,14 @@ contains() {
 
 # key-value pairs of what gradle version each gradle plugin version
 # should accept
-d_plugin_k=(0.12 0.11 0.10  0.9  0.8 0.7 0.6 0.5 0.4 0.3 0.2)
-d_plugin_v=(1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
+d_plugin_k=(0.14 0.13 0.12 0.11 0.10  0.9  0.8 0.7 0.6 0.5 0.4 0.3 0.2)
+d_plugin_v=( 2.1  2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
 
 for v in ${d_plugin_v}; do
-       contains $v "${v_all[*]}" && v_def=$v && break
+       if contains $v "${v_all[*]}"; then
+               v_def=$v
+               break
+       fi
 done
 
 # Latest takes priority
index c25fc1a106b814f3510e0718ae1b4d979d7aa137..c57689c7e0903f62f485347dd34b40aa8857895f 100644 (file)
@@ -885,7 +885,7 @@ which architecture or platform the apk is designed to run on.
 If specified, the package version code in the AndroidManifest.xml is
 replaced with the version code for the build. See also forceversion.
 
-@item rm=relpath1,relpath2,...
+@item rm=<path1>[,<path2>,...]
 Specifies the relative paths of files or directories to delete before
 the build is done. The paths are relative to the base of the build
 directory - i.e. the root of the directory structure checked out from
@@ -895,7 +895,7 @@ AndroidManifest.xml.
 Multiple files/directories can be specified by separating them with ','.
 Directories will be recursively deleted.
 
-@item extlibs=a,b,...
+@item extlibs=<lib1>[,<lib2>,...]
 Comma-separated list of external libraries (jar files) from the
 @code{build/extlib} library, which will be placed in the @code{libs} directory
 of the project.
@@ -948,7 +948,7 @@ You can use $$SDK$$, $$NDK$$ and $$MVN3$$ to substitute the paths to the
 android SDK and NDK directories, and Maven 3 executable respectively e.g.
 for when you need to run @code{android update project} explicitly.
 
-@item scanignore=path1,path2,...
+@item scanignore=<path1>[,<path2>,...]
 Enables one or more files/paths to be excluded from the scan process.
 This should only be used where there is a very good reason, and
 probably accompanied by a comment explaining why it is necessary.
@@ -956,7 +956,7 @@ probably accompanied by a comment explaining why it is necessary.
 When scanning the source tree for problems, matching files whose relative
 paths start with any of the paths given here are ignored.
 
-@item scandelete=path1,path2,...
+@item scandelete=<path1>[,<path2>,...]
 Similar to scanignore=, but instead of ignoring files under the given paths,
 it tells f-droid to delete the matching files directly.
 
@@ -988,23 +988,25 @@ actually not required or used, remove the directory instead (using
 isn't used nor built will result in an error saying that native
 libraries were expected in the resulting package.
 
-@item gradle=<flavour>
-Build with Gradle instead of Ant, specifying what flavour to assemble.
-If <flavour> is 'yes' or 'main', no flavour will be used. Note
-that this will not work on projects with flavours, since it will build
-all flavours and there will be no 'main' build.
+@item gradle=<flavour1>[,<flavour2>,...]
+Build with Gradle instead of Ant, specifying what flavours to use. Flavours
+are case sensitive since the path to the output apk is as well.
+
+If only one flavour is given and it is 'yes' or 'main', no flavour will be
+used. Note that for projects with flavours, you must specify at least one
+valid flavour since 'yes' or 'main' will build all of them separately.
 
 @item maven=yes[@@<dir>]
 Build with Maven instead of Ant. An extra @@<dir> tells f-droid to run Maven
 inside that relative subdirectory. Sometimes it is needed to use @@.. so that
 builds happen correctly.
 
-@item preassemble=<task1> <task2>
-Space-separated list of Gradle tasks to be run before the assemble task
-in a Gradle project build.
+@item preassemble=<task1>[,<task2>,...]
+List of Gradle tasks to be run before the assemble task in a Gradle project
+build.
 
-@item antcommand=xxx
-Specify an alternate Ant command (target) instead of the default
+@item antcommands=<target1>[,<target2>,...]
+Specify an alternate set of Ant commands (target) instead of the default
 'release'. It can't be given any flags, such as the path to a build.xml.
 
 @item output=path/to/output.apk
index 5f8dfdcfd919a90445c9b0917095d9c904f4b4f3..e1272649b02b82433ea1f276befb49f90b12eb82 100644 (file)
@@ -9,7 +9,7 @@
 # Override the path to the Android NDK, $ANDROID_NDK by default
 # ndk_path = "/path/to/android-ndk"
 # Build tools version to be used
-build_tools = "20.0.0"
+build_tools = "21.0.2"
 
 # Command for running Ant
 # ant = "/path/to/ant"
@@ -169,6 +169,12 @@ update_stats = False
 # calculation purposes.
 stats_ignore = []
 
+# Server stats logs are retrieved from. Required when update_stats is True.
+stats_server = "example.com"
+
+# User stats logs are retrieved from. Required when update_stats is True.
+stats_user = "bob"
+
 # Use the following to push stats to a Carbon instance:
 stats_to_carbon = False
 carbon_host = '0.0.0.0'
index b38250e4b5fd4f123b93ef24e26fe6e35ce1421f..9c8acfa2ab846ad9e0354d87c00343aff6d58520 100644 (file)
@@ -522,25 +522,6 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
         tarball.add(build_dir, tarname, exclude=tarexc)
         tarball.close()
 
-    if onserver:
-        manifest = os.path.join(root_dir, 'AndroidManifest.xml')
-        if os.path.exists(manifest):
-            homedir = os.path.expanduser('~')
-            with open(os.path.join(homedir, 'buildserverid'), 'r') as f:
-                buildserverid = f.read()
-            with open(os.path.join(homedir, 'fdroidserverid'), 'r') as f:
-                fdroidserverid = f.read()
-            with open(manifest, 'r') as f:
-                manifestcontent = f.read()
-            manifestcontent = manifestcontent.replace('</manifest>',
-                                                      '<fdroid buildserverid="'
-                                                      + buildserverid + '"'
-                                                      + ' fdroidserverid="'
-                                                      + fdroidserverid + '"'
-                                                      + '/></manifest>')
-            with open(manifest, 'w') as f:
-                f.write(manifestcontent)
-
     # Run a build command if one is required...
     if thisbuild['build']:
         logging.info("Running 'build' commands in %s" % root_dir)
@@ -680,14 +661,13 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
 
     elif thisbuild['type'] == 'gradle':
         logging.info("Building Gradle project...")
-        flavours = thisbuild['gradle'].split(',')
-
-        if len(flavours) == 1 and flavours[0] in ['main', 'yes', '']:
-            flavours[0] = ''
+        flavours = thisbuild['gradle']
+        if flavours == ['yes']:
+            flavours = []
 
         commands = [config['gradle']]
         if thisbuild['preassemble']:
-            commands += thisbuild['preassemble'].split()
+            commands += thisbuild['preassemble']
 
         flavours_cmd = ''.join(flavours)
         if flavours_cmd:
@@ -705,8 +685,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
     elif thisbuild['type'] == 'ant':
         logging.info("Building Ant project...")
         cmd = ['ant']
-        if thisbuild['antcommand']:
-            cmd += [thisbuild['antcommand']]
+        if thisbuild['antcommands']:
+            cmd += thisbuild['antcommands']
         else:
             cmd += ['release']
         p = FDroidPopen(cmd, cwd=root_dir)
@@ -833,6 +813,19 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
                                 str(thisbuild['vercode']))
                              )
 
+    # Add information for 'fdroid verify' to be able to reproduce the build
+    # environment.
+    if onserver:
+        metadir = os.path.join(tmp_dir, 'META-INF')
+        if not os.path.exists(metadir):
+            os.mkdir(metadir)
+        homedir = os.path.expanduser('~')
+        for fn in ['buildserverid', 'fdroidserverid']:
+            shutil.copyfile(os.path.join(homedir, fn),
+                            os.path.join(metadir, fn))
+            subprocess.call(['jar', 'uf', os.path.abspath(src),
+                            'META-INF/' + fn], cwd=tmp_dir)
+
     # Copy the unsigned apk to our destination directory for further
     # processing (by publish.py)...
     dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
index 4c1b57287e8b81b5e49ab0f4d0bb45b4e91c6811..21bac4cec84ba070910667fd7eb16009896c674b 100644 (file)
@@ -109,14 +109,12 @@ def check_tags(app, pattern):
 
         vcs.gotorevision(None)
 
-        flavour = None
+        flavours = []
         if len(app['builds']) > 0:
             if app['builds'][-1]['subdir']:
                 build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
             if app['builds'][-1]['gradle']:
-                flavour = app['builds'][-1]['gradle']
-        if flavour == 'yes':
-            flavour = None
+                flavours = app['builds'][-1]['gradle']
 
         hpak = None
         htag = None
@@ -136,7 +134,7 @@ def check_tags(app, pattern):
             vcs.gotorevision(tag)
 
             # Only process tags where the manifest exists...
-            paths = common.manifest_paths(build_dir, flavour)
+            paths = common.manifest_paths(build_dir, flavours)
             version, vercode, package = \
                 common.parse_androidmanifests(paths, app['Update Check Ignore'])
             if not package or package != appid or not version or not vercode:
@@ -196,20 +194,17 @@ def check_repomanifest(app, branch=None):
         elif repotype == 'bzr':
             vcs.gotorevision(None)
 
-        flavour = None
-
+        flavours = []
         if len(app['builds']) > 0:
             if app['builds'][-1]['subdir']:
                 build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
             if app['builds'][-1]['gradle']:
-                flavour = app['builds'][-1]['gradle']
-        if flavour == 'yes':
-            flavour = None
+                flavours = app['builds'][-1]['gradle']
 
         if not os.path.isdir(build_dir):
             return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
 
-        paths = common.manifest_paths(build_dir, flavour)
+        paths = common.manifest_paths(build_dir, flavours)
 
         version, vercode, package = \
             common.parse_androidmanifests(paths, app['Update Check Ignore'])
@@ -319,15 +314,13 @@ def check_changed_subdir(app):
     if not os.path.isdir(build_dir):
         return None
 
-    flavour = None
+    flavours = []
     if len(app['builds']) > 0 and app['builds'][-1]['gradle']:
-        flavour = app['builds'][-1]['gradle']
-    if flavour == 'yes':
-        flavour = None
+        flavours = app['builds'][-1]['gradle']
 
     for d in dirs_with_manifest(build_dir):
         logging.debug("Trying possible dir %s." % d)
-        m_paths = common.manifest_paths(d, flavour)
+        m_paths = common.manifest_paths(d, flavours)
         package = common.parse_androidmanifests(m_paths, app['Update Check Ignore'])[2]
         if package and package == appid:
             logging.debug("Manifest exists in possible dir %s." % d)
@@ -352,18 +345,15 @@ def fetch_autoname(app, tag):
     except VCSException:
         return None
 
-    flavour = None
+    flavours = []
     if len(app['builds']) > 0:
         if app['builds'][-1]['subdir']:
             app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
         if app['builds'][-1]['gradle']:
-            flavour = app['builds'][-1]['gradle']
-    if flavour == 'yes':
-        flavour = None
+            flavours = app['builds'][-1]['gradle']
 
-    logging.debug("...fetch auto name from " + app_dir +
-                  ((" (flavour: %s)" % flavour) if flavour else ""))
-    new_name = common.fetch_real_name(app_dir, flavour)
+    logging.debug("...fetch auto name from " + app_dir)
+    new_name = common.fetch_real_name(app_dir, flavours)
     commitmsg = None
     if new_name:
         logging.debug("...got autoname '" + new_name + "'")
@@ -375,7 +365,7 @@ def fetch_autoname(app, tag):
         logging.debug("...couldn't get autoname")
 
     if app['Current Version'].startswith('@string/'):
-        cv = common.version_name(app['Current Version'], app_dir, flavour)
+        cv = common.version_name(app['Current Version'], app_dir, flavours)
         if app['Current Version'] != cv:
             app['Current Version'] = cv
             if not commitmsg:
index 6e9cf1ec1b8070f85c89725340f61da1c8528b2d..033cba8b64c8cc25182313c64de92d40b67169a3 100644 (file)
@@ -39,38 +39,55 @@ options = None
 env = None
 
 
-def get_default_config():
-    return {
-        'sdk_path': os.getenv("ANDROID_HOME") or "",
-        'ndk_path': os.getenv("ANDROID_NDK") or "",
-        'build_tools': "20.0.0",
-        'ant': "ant",
-        'mvn3': "mvn",
-        'gradle': 'gradle',
-        'sync_from_local_copy_dir': False,
-        'update_stats': False,
-        'stats_ignore': [],
-        'stats_to_carbon': False,
-        'repo_maxage': 0,
-        'build_server_always': False,
-        'keystore': os.path.join(os.getenv("HOME"), '.local', 'share', 'fdroidserver', 'keystore.jks'),
-        'smartcardoptions': [],
-        'char_limits': {
-            'Summary': 50,
-            'Description': 1500
-        },
-        'keyaliases': {},
-        'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
-        'repo_name': "My First FDroid Repo Demo",
-        'repo_icon': "fdroid-icon.png",
-        'repo_description': '''
-            This is a repository of apps to be used with FDroid. Applications in this
-            repository are either official binaries built by the original application
-            developers, or are binaries built from source by the admin of f-droid.org
-            using the tools on https://gitlab.com/u/fdroid.
-            ''',
-        'archive_older': 0,
-    }
+default_config = {
+    'sdk_path': "$ANDROID_HOME",
+    'ndk_path': "$ANDROID_NDK",
+    'build_tools': "21.0.2",
+    'ant': "ant",
+    'mvn3': "mvn",
+    'gradle': 'gradle',
+    'sync_from_local_copy_dir': False,
+    'update_stats': False,
+    'stats_ignore': [],
+    'stats_server': None,
+    'stats_user': None,
+    'stats_to_carbon': False,
+    'repo_maxage': 0,
+    'build_server_always': False,
+    'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
+    'smartcardoptions': [],
+    'char_limits': {
+        'Summary': 50,
+        'Description': 1500
+    },
+    'keyaliases': {},
+    'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
+    'repo_name': "My First FDroid Repo Demo",
+    'repo_icon': "fdroid-icon.png",
+    'repo_description': '''
+        This is a repository of apps to be used with FDroid. Applications in this
+        repository are either official binaries built by the original application
+        developers, or are binaries built from source by the admin of f-droid.org
+        using the tools on https://gitlab.com/u/fdroid.
+        ''',
+    'archive_older': 0,
+}
+
+
+def fill_config_defaults(thisconfig):
+    for k, v in default_config.items():
+        if k not in thisconfig:
+            thisconfig[k] = v
+
+    # Expand paths (~users and $vars)
+    for k in ['sdk_path', 'ndk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
+        v = thisconfig[k]
+        orig = v
+        v = os.path.expanduser(v)
+        v = os.path.expandvars(v)
+        if orig != v:
+            thisconfig[k] = v
+            thisconfig[k + '_orig'] = orig
 
 
 def read_config(opts, config_file='config.py'):
@@ -109,17 +126,7 @@ def read_config(opts, config_file='config.py'):
         if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
             logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
 
-    defconfig = get_default_config()
-    for k, v in defconfig.items():
-        if k not in config:
-            config[k] = v
-
-    # Expand environment variables
-    for k, v in config.items():
-        if type(v) != str:
-            continue
-        v = os.path.expanduser(v)
-        config[k] = os.path.expandvars(v)
+    fill_config_defaults(config)
 
     if not test_sdk_exists(config):
         sys.exit(3)
@@ -188,32 +195,31 @@ def read_config(opts, config_file='config.py'):
     return config
 
 
-def test_sdk_exists(c):
-    if c['sdk_path'] is None:
-        # c['sdk_path'] is set to the value of ANDROID_HOME by default
-        logging.error('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
+def test_sdk_exists(thisconfig):
+    if thisconfig['sdk_path'] == default_config['sdk_path']:
+        logging.error('No Android SDK found!')
         logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
         logging.error('\texport ANDROID_HOME=/opt/android-sdk')
         return False
-    if not os.path.exists(c['sdk_path']):
-        logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
+    if not os.path.exists(thisconfig['sdk_path']):
+        logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
         return False
-    if not os.path.isdir(c['sdk_path']):
-        logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
+    if not os.path.isdir(thisconfig['sdk_path']):
+        logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
         return False
     for d in ['build-tools', 'platform-tools', 'tools']:
-        if not os.path.isdir(os.path.join(c['sdk_path'], d)):
+        if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
             logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
-                c['sdk_path'], d))
+                thisconfig['sdk_path'], d))
             return False
     return True
 
 
-def test_build_tools_exists(c):
-    if not test_sdk_exists(c):
+def test_build_tools_exists(thisconfig):
+    if not test_sdk_exists(thisconfig):
         return False
-    build_tools = os.path.join(c['sdk_path'], 'build-tools')
-    versioned_build_tools = os.path.join(build_tools, c['build_tools'])
+    build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
+    versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
     if not os.path.isdir(versioned_build_tools):
         logging.critical('Android Build Tools path "'
                          + versioned_build_tools + '" does not exist!')
@@ -823,7 +829,7 @@ def retrieve_string(app_dir, string, xmlfiles=None):
 
 
 # Return list of existing files that will be used to find the highest vercode
-def manifest_paths(app_dir, flavour):
+def manifest_paths(app_dir, flavours):
 
     possible_manifests = \
         [os.path.join(app_dir, 'AndroidManifest.xml'),
@@ -831,7 +837,9 @@ def manifest_paths(app_dir, flavour):
          os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
          os.path.join(app_dir, 'build.gradle')]
 
-    if flavour:
+    for flavour in flavours:
+        if flavour == 'yes':
+            continue
         possible_manifests.append(
             os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
 
@@ -839,11 +847,11 @@ def manifest_paths(app_dir, flavour):
 
 
 # Retrieve the package name. Returns the name, or None if not found.
-def fetch_real_name(app_dir, flavour):
+def fetch_real_name(app_dir, flavours):
     app_search = re.compile(r'.*<application.*').search
     name_search = re.compile(r'.*android:label="([^"]+)".*').search
     app_found = False
-    for f in manifest_paths(app_dir, flavour):
+    for f in manifest_paths(app_dir, flavours):
         if not has_extension(f, 'xml'):
             continue
         logging.debug("fetch_real_name: Checking manifest at " + f)
@@ -864,8 +872,8 @@ def fetch_real_name(app_dir, flavour):
 
 
 # Retrieve the version name
-def version_name(original, app_dir, flavour):
-    for f in manifest_paths(app_dir, flavour):
+def version_name(original, app_dir, flavours):
+    for f in manifest_paths(app_dir, flavours):
         if not has_extension(f, 'xml'):
             continue
         string = retrieve_string(app_dir, original)
@@ -1180,13 +1188,15 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     if build['subdir']:
         localprops += [os.path.join(root_dir, 'local.properties')]
     for path in localprops:
-        if not os.path.isfile(path):
-            continue
-        logging.info("Updating properties file at %s" % path)
-        f = open(path, 'r')
-        props = f.read()
-        f.close()
-        props += '\n'
+        props = ""
+        if os.path.isfile(path):
+            logging.info("Updating local.properties file at %s" % path)
+            f = open(path, 'r')
+            props += f.read()
+            f.close()
+            props += '\n'
+        else:
+            logging.info("Creating local.properties file at %s" % path)
         # Fix old-fashioned 'sdk-location' by copying
         # from sdk.dir, if necessary
         if build['oldsdkloc']:
@@ -1196,7 +1206,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         else:
             props += "sdk.dir=%s\n" % config['sdk_path']
             props += "sdk-location=%s\n" % config['sdk_path']
-        if 'ndk_path' in config:
+        if config['ndk_path']:
             # Add ndk location
             props += "ndk.dir=%s\n" % config['ndk_path']
             props += "ndk-location=%s\n" % config['ndk_path']
@@ -1207,11 +1217,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         f.write(props)
         f.close()
 
-    flavour = None
+    flavours = []
     if build['type'] == 'gradle':
-        flavour = build['gradle']
-        if flavour in ['main', 'yes', '']:
-            flavour = None
+        flavours = build['gradle']
 
         version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
         gradlepluginver = None
@@ -1254,7 +1262,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     # Insert version code and number into the manifest if necessary
     if build['forceversion']:
         logging.info("Changing the version name")
-        for path in manifest_paths(root_dir, flavour):
+        for path in manifest_paths(root_dir, flavours):
             if not os.path.isfile(path):
                 continue
             if has_extension(path, 'xml'):
@@ -1273,7 +1281,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
                     raise BuildException("Failed to amend build.gradle")
     if build['forcevercode']:
         logging.info("Changing the version code")
-        for path in manifest_paths(root_dir, flavour):
+        for path in manifest_paths(root_dir, flavours):
             if not os.path.isfile(path):
                 continue
             if has_extension(path, 'xml'):
@@ -1652,8 +1660,12 @@ def FDroidPopen(commands, cwd=None, shell=False, output=True):
     logging.debug("> %s" % ' '.join(commands))
 
     result = PopenResult()
-    p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
-                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    p = None
+    try:
+        p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
+                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    except OSError, e:
+        raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e))
 
     stdout_queue = Queue.Queue()
     stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
@@ -1773,3 +1785,37 @@ def place_srclib(root_dir, number, libpath):
                 o.write(line)
         if not placed:
             o.write('android.library.reference.%d=%s\n' % (number, relpath))
+
+
+def compare_apks(apk1, apk2, tmp_dir):
+    """Compare two apks
+
+    Returns None if the apk content is the same (apart from the signing key),
+    otherwise a string describing what's different, or what went wrong when
+    trying to do the comparison.
+    """
+
+    thisdir = os.path.join(tmp_dir, 'this_apk')
+    thatdir = os.path.join(tmp_dir, 'that_apk')
+    for d in [thisdir, thatdir]:
+        if os.path.exists(d):
+            shutil.rmtree(d)
+        os.mkdir(d)
+
+    if subprocess.call(['jar', 'xf',
+                        os.path.abspath(apk1)],
+                       cwd=thisdir) != 0:
+        return("Failed to unpack " + apk1)
+    if subprocess.call(['jar', 'xf',
+                        os.path.abspath(apk2)],
+                       cwd=thatdir) != 0:
+        return("Failed to unpack " + apk2)
+
+    p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir,
+                    output=False)
+    lines = p.output.splitlines()
+    if len(lines) != 1 or 'META-INF' not in lines[0]:
+        return("Unexpected diff output - " + p.output)
+
+    # If we get here, it seems like they're the same!
+    return None
index be9fe12f460036dbaf93137694b8bd5c850e5ac1..c837fffdbd8e051d3275b5061a135455489a3a1c 100644 (file)
@@ -239,7 +239,7 @@ def main():
         root_dir = src_dir
 
     # Extract some information...
-    paths = common.manifest_paths(root_dir, None)
+    paths = common.manifest_paths(root_dir, [])
     if paths:
 
         version, vercode, package = common.parse_androidmanifests(paths)
index a3eded5b3819e7be225f9cc04642540c986f83d7..6f96a24b29843f3187a7a518382b107ed2f76416 100644 (file)
@@ -36,8 +36,11 @@ config = {}
 options = None
 
 
-def write_to_config(key, value):
+def write_to_config(thisconfig, key, value=None):
     '''write a key/value to the local config.py'''
+    if value is None:
+        origkey = key + '_orig'
+        value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
     with open('config.py', 'r') as f:
         data = f.read()
     pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
@@ -122,7 +125,8 @@ def main():
         examplesdir = prefix + '/examples'
 
     fdroiddir = os.getcwd()
-    test_config = common.get_default_config()
+    test_config = dict()
+    common.fill_config_defaults(test_config)
 
     # track down where the Android SDK is, the default is to use the path set
     # in ANDROID_HOME if that exists, otherwise None
@@ -154,7 +158,11 @@ def main():
         shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir)
         shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py')
         os.chmod('config.py', 0o0600)
-        write_to_config('sdk_path', test_config['sdk_path'])
+        # If android_home is None, test_config['sdk_path'] will be used and
+        # "$ANDROID_HOME" may be used if the env var is set up correctly.
+        # If android_home is not None, the path given from the command line
+        # will be directly written in the config.
+        write_to_config(test_config, 'sdk_path', options.android_home)
     else:
         logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
         logging.info('Try running `fdroid init` in an empty directory.')
@@ -179,7 +187,7 @@ def main():
             test_config['build_tools'] = ''
         else:
             test_config['build_tools'] = dirname
-        write_to_config('build_tools', test_config['build_tools'])
+        write_to_config(test_config, 'build_tools')
     if not common.test_build_tools_exists(test_config):
         sys.exit(3)
 
@@ -194,7 +202,7 @@ def main():
         logging.info('using ANDROID_NDK')
         ndk_path = os.environ['ANDROID_NDK']
     if os.path.isdir(ndk_path):
-        write_to_config('ndk_path', ndk_path)
+        write_to_config(test_config, 'ndk_path')
     # the NDK is optional so we don't prompt the user for it if its not found
 
     # find or generate the keystore for the repo signing key. First try the
@@ -213,18 +221,18 @@ def main():
             if not os.path.exists(keystore):
                 logging.info('"' + keystore
                              + '" does not exist, creating a new keystore there.')
-    write_to_config('keystore', keystore)
+    write_to_config(test_config, 'keystore', keystore)
     repo_keyalias = None
     if options.repo_keyalias:
         repo_keyalias = options.repo_keyalias
-        write_to_config('repo_keyalias', repo_keyalias)
+        write_to_config(test_config, 'repo_keyalias', repo_keyalias)
     if options.distinguished_name:
         keydname = options.distinguished_name
-        write_to_config('keydname', keydname)
+        write_to_config(test_config, 'keydname', keydname)
     if keystore == 'NONE':  # we're using a smartcard
-        write_to_config('repo_keyalias', '1')  # seems to be the default
+        write_to_config(test_config, 'repo_keyalias', '1')  # seems to be the default
         disable_in_config('keypass', 'never used with smartcard')
-        write_to_config('smartcardoptions',
+        write_to_config(test_config, 'smartcardoptions',
                         ('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
                          + '-providerClass sun.security.pkcs11.SunPKCS11 '
                          + '-providerArg opensc-fdroid.cfg'))
@@ -254,14 +262,14 @@ def main():
         if not os.path.exists(keystoredir):
             os.makedirs(keystoredir, mode=0o700)
         password = genpassword()
-        write_to_config('keystorepass', password)
-        write_to_config('keypass', password)
+        write_to_config(test_config, 'keystorepass', password)
+        write_to_config(test_config, 'keypass', password)
         if options.repo_keyalias is None:
             repo_keyalias = socket.getfqdn()
-            write_to_config('repo_keyalias', repo_keyalias)
+            write_to_config(test_config, 'repo_keyalias', repo_keyalias)
         if not options.distinguished_name:
             keydname = 'CN=' + repo_keyalias + ', OU=F-Droid'
-            write_to_config('keydname', keydname)
+            write_to_config(test_config, 'keydname', keydname)
         genkey(keystore, repo_keyalias, password, keydname)
 
     logging.info('Built repo based in "' + fdroiddir + '"')
index 98482be0cfb1f8988cc1880aa31d13c27f28b52f..bfacc2da93c50426ddc0b020de5bd518884bcbb4 100644 (file)
@@ -85,10 +85,12 @@ regex_warnings = {
     'Description': [
         (re.compile(r'^No description available$'),
          "Description yet to be filled"),
-        (re.compile(r'[ ]*[*#][^ .]'),
+        (re.compile(r'\s*[*#][^ .]'),
          "Invalid bulleted list"),
-        (re.compile(r'^ '),
+        (re.compile(r'^\s'),
          "Unnecessary leading space"),
+        (re.compile(r'.*\s$'),
+         "Unnecessary trailing space"),
         ],
 }
 
@@ -169,6 +171,8 @@ def main():
         if app['Disabled']:
             continue
 
+        count['app_total'] += 1
+
         for build in app['builds']:
             if build['commit'] and not build['disable']:
                 lastcommit = build['commit']
@@ -196,7 +200,7 @@ def main():
             if app['Summary'].lower() == name.lower():
                 warn("Summary '%s' is just the app's name" % app['Summary'])
 
-        if app['Summary'] and app['Description']:
+        if app['Summary'] and app['Description'] and len(app['Description']) == 1:
             if app['Summary'].lower() == app['Description'][0].lower():
                 warn("Description '%s' is just the app's summary" % app['Summary'])
 
@@ -249,7 +253,8 @@ def main():
         if not curid:
             print
 
-    logging.info("Found a total of %i warnings in %i apps." % (count['warn'], count['app']))
+    logging.info("Found a total of %i warnings in %i apps out of %i total." % (
+        count['warn'], count['app'], count['app_total']))
 
 if __name__ == "__main__":
     main()
index aa7151767b925177654927a8e845b51fc8999e79..979a01591cd6b37d7d480f31bd12cb24039d1a39 100644 (file)
@@ -57,6 +57,7 @@ app_defaults = OrderedDict([
     ('Requires Root', False),
     ('Repo Type', ''),
     ('Repo', ''),
+    ('Binaries', None),
     ('Maintainer Notes', []),
     ('Archive Policy', None),
     ('Auto Update Mode', 'None'),
@@ -99,7 +100,7 @@ flag_defaults = OrderedDict([
     ('build', ''),
     ('buildjni', []),
     ('preassemble', []),
-    ('antcommand', None),
+    ('antcommands', None),
     ('novcheck', False),
     ])
 
@@ -197,6 +198,11 @@ valuetypes = {
                    ["Repo Type"],
                    []),
 
+    FieldValidator("Binaries",
+                   r'^http[s]?://', None,
+                   ["Binaries"],
+                   []),
+
     FieldValidator("Archive Policy",
                    r'^[0-9]+ versions$', None,
                    ["Archive Policy"],
@@ -496,7 +502,7 @@ def read_metadata(xref=True):
         # errors are caught early rather than when they hit the build server.
         def linkres(appid):
             if appid in apps:
-                return ("fdroid:app" + appid, "Dummy name - don't know yet")
+                return ("fdroid.app:" + appid, "Dummy name - don't know yet")
             raise MetaDataException("Cannot resolve app id " + appid)
 
         for appid, app in apps.iteritems():
@@ -527,8 +533,8 @@ def metafieldtype(name):
 
 
 def flagtype(name):
-    if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
-                'update', 'scanignore', 'scandelete']:
+    if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
+                'update', 'scanignore', 'scandelete', 'gradle', 'antcommands']:
         return 'list'
     if name in ['init', 'prebuild', 'build']:
         return 'script'
@@ -601,7 +607,11 @@ def parse_metadata(metafile):
         t = flagtype(pk)
         if t == 'list':
             # Port legacy ';' separators
-            thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
+            pv = [v.strip() for v in pv.replace(';', ',').split(',')]
+            if pk == 'gradle':
+                if len(pv) == 1 and pv[0] in ['main', 'yes']:
+                    pv = ['yes']
+            thisbuild[pk] = pv
         elif t == 'string' or t == 'script':
             thisbuild[pk] = pv
         elif t == 'bool':
@@ -783,6 +793,8 @@ def parse_metadata(metafile):
     for build in thisinfo['builds']:
         fill_build_defaults(build)
 
+    thisinfo['builds'] = sorted(thisinfo['builds'], key=lambda build: int(build['vercode']))
+
     return (appid, thisinfo)
 
 
@@ -845,6 +857,8 @@ def write_metadata(dest, app):
     if app['Repo Type']:
         writefield('Repo Type')
         writefield('Repo')
+        if app['Binaries']:
+            writefield('Binaries')
         mf.write('\n')
     for build in app['builds']:
 
index efda4c414afed8bf9a9c965e65599fc2b91ab309..39f601c4f057ccc14fac25f5b4b51a8f996b5ee1 100644 (file)
@@ -111,60 +111,113 @@ def main():
                 continue
         logging.info("Processing " + apkfile)
 
-        # Figure out the key alias name we'll use. Only the first 8
-        # characters are significant, so we'll use the first 8 from
-        # the MD5 of the app's ID and hope there are no collisions.
-        # If a collision does occur later, we're going to have to
-        # come up with a new alogrithm, AND rename all existing keys
-        # in the keystore!
-        if appid in config['keyaliases']:
-            # For this particular app, the key alias is overridden...
-            keyalias = config['keyaliases'][appid]
-            if keyalias.startswith('@'):
+        # There ought to be valid metadata for this app, otherwise why are we
+        # trying to publish it?
+        if appid not in allapps:
+            logging.error("Unexpected {0} found in unsigned directory"
+                          .format(apkfilename))
+            sys.exit(1)
+        app = allapps[appid]
+
+        if app.get('Binaries', None):
+
+            # It's an app where we build from source, and verify the apk
+            # contents against a developer's binary, and then publish their
+            # version if everything checks out.
+
+            # Need the version name for the version code...
+            versionname = None
+            for build in app['builds']:
+                if build['vercode'] == vercode:
+                    versionname = build['version']
+                    break
+            if not versionname:
+                logging.error("...no defined build for version code {0}"
+                              .format(vercode))
+                continue
+
+            # Figure out where the developer's binary is supposed to come from...
+            url = app['Binaries']
+            url = url.replace('%v', versionname)
+            url = url.replace('%c', str(vercode))
+
+            # Grab the binary from where the developer publishes it...
+            logging.info("...retrieving " + url)
+            srcapk = os.path.join(tmp_dir, url.split('/')[-1])
+            p = FDroidPopen(['wget', '-nv', url], cwd=tmp_dir)
+            if p.returncode != 0 or not os.path.exists(srcapk):
+                logging.error("...failed to retrieve " + url +
+                              " - publish skipped")
+                continue
+
+            # Compare our unsigned one with the downloaded one...
+            compare_result = common.compare_apks(srcapk, apkfile, tmp_dir)
+            if compare_result:
+                logging.error("...verfication failed - publish skipped : "
+                              + compare_result)
+                continue
+
+            # Success! So move the downloaded file to the repo...
+            shutil.move(srcapk, os.path.join(output_dir, apkfilename))
+
+        else:
+
+            # It's a 'normal' app, i.e. we sign and publish it...
+
+            # Figure out the key alias name we'll use. Only the first 8
+            # characters are significant, so we'll use the first 8 from
+            # the MD5 of the app's ID and hope there are no collisions.
+            # If a collision does occur later, we're going to have to
+            # come up with a new alogrithm, AND rename all existing keys
+            # in the keystore!
+            if appid in config['keyaliases']:
+                # For this particular app, the key alias is overridden...
+                keyalias = config['keyaliases'][appid]
+                if keyalias.startswith('@'):
+                    m = md5.new()
+                    m.update(keyalias[1:])
+                    keyalias = m.hexdigest()[:8]
+            else:
                 m = md5.new()
-                m.update(keyalias[1:])
+                m.update(appid)
                 keyalias = m.hexdigest()[:8]
-        else:
-            m = md5.new()
-            m.update(appid)
-            keyalias = m.hexdigest()[:8]
-        logging.info("Key alias: " + keyalias)
-
-        # See if we already have a key for this application, and
-        # if not generate one...
-        p = FDroidPopen(['keytool', '-list',
-                         '-alias', keyalias, '-keystore', config['keystore'],
-                         '-storepass:file', config['keystorepassfile']])
-        if p.returncode != 0:
-            logging.info("Key does not exist - generating...")
-            p = FDroidPopen(['keytool', '-genkey',
-                             '-keystore', config['keystore'],
-                             '-alias', keyalias,
-                             '-keyalg', 'RSA', '-keysize', '2048',
-                             '-validity', '10000',
+            logging.info("Key alias: " + keyalias)
+
+            # See if we already have a key for this application, and
+            # if not generate one...
+            p = FDroidPopen(['keytool', '-list',
+                             '-alias', keyalias, '-keystore', config['keystore'],
+                             '-storepass:file', config['keystorepassfile']])
+            if p.returncode != 0:
+                logging.info("Key does not exist - generating...")
+                p = FDroidPopen(['keytool', '-genkey',
+                                 '-keystore', config['keystore'],
+                                 '-alias', keyalias,
+                                 '-keyalg', 'RSA', '-keysize', '2048',
+                                 '-validity', '10000',
+                                 '-storepass:file', config['keystorepassfile'],
+                                 '-keypass:file', config['keypassfile'],
+                                 '-dname', config['keydname']])
+                # TODO keypass should be sent via stdin
+                if p.returncode != 0:
+                    raise BuildException("Failed to generate key")
+
+            # Sign the application...
+            p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
                              '-storepass:file', config['keystorepassfile'],
-                             '-keypass:file', config['keypassfile'],
-                             '-dname', config['keydname']])
+                             '-keypass:file', config['keypassfile'], '-sigalg',
+                             'MD5withRSA', '-digestalg', 'SHA1',
+                             apkfile, keyalias])
             # TODO keypass should be sent via stdin
             if p.returncode != 0:
-                raise BuildException("Failed to generate key")
-
-        # Sign the application...
-        p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
-                         '-storepass:file', config['keystorepassfile'],
-                         '-keypass:file', config['keypassfile'], '-sigalg',
-                         'MD5withRSA', '-digestalg', 'SHA1',
-                         apkfile, keyalias])
-        # TODO keypass should be sent via stdin
-        if p.returncode != 0:
-            raise BuildException("Failed to sign application")
-
-        # Zipalign it...
-        p = FDroidPopen([config['zipalign'], '-v', '4', apkfile,
-                         os.path.join(output_dir, apkfilename)])
-        if p.returncode != 0:
-            raise BuildException("Failed to align application")
-        os.remove(apkfile)
+                raise BuildException("Failed to sign application")
+
+            # Zipalign it...
+            p = FDroidPopen([config['zipalign'], '-v', '4', apkfile,
+                             os.path.join(output_dir, apkfilename)])
+            if p.returncode != 0:
+                raise BuildException("Failed to align application")
+            os.remove(apkfile)
 
         # Move the source tarball into the output directory...
         tarfilename = apkfilename[:-4] + '_src.tar.gz'
index 4c2a5effb396741c7d5969edfe001dec74349a32..05c44d3fdad62ebbb486c9e49cbfe4d029882e8f 100644 (file)
@@ -91,8 +91,8 @@ def main():
             logging.info('Retrieving logs')
             ssh = paramiko.SSHClient()
             ssh.load_system_host_keys()
-            ssh.connect('f-droid.org', username='fdroid', timeout=10,
-                        key_filename=config['webserver_keyfile'])
+            ssh.connect(config['stats_server'], username=config['stats_user'],
+                        timeout=10, key_filename=config['webserver_keyfile'])
             ftp = ssh.open_sftp()
             ftp.get_channel().settimeout(60)
             logging.info("...connected")
index 859f4ae3de6b64058b40afb4bec056b3c53205c2..225f594d0ab3e69130a154e501cb1c28da1e3623 100644 (file)
@@ -29,6 +29,11 @@ import pickle
 from xml.dom.minidom import Document
 from optparse import OptionParser
 import time
+from pyasn1.error import PyAsn1Error
+from pyasn1.codec.der import decoder, encoder
+from pyasn1_modules import rfc2315
+from hashlib import md5
+
 from PIL import Image
 import logging
 
@@ -322,6 +327,59 @@ def resize_all_icons(repodirs):
                 resize_icon(iconpath, density)
 
 
+cert_path_regex = re.compile(r'^META-INF/.*\.RSA$')
+
+
+def getsig(apkpath):
+    """ Get the signing certificate of an apk. To get the same md5 has that
+    Android gets, we encode the .RSA certificate in a specific format and pass
+    it hex-encoded to the md5 digest algorithm.
+
+    :param apkpath: path to the apk
+    :returns: A string containing the md5 of the signature of the apk or None
+              if an error occurred.
+    """
+
+    cert = None
+
+    # verify the jar signature is correct
+    args = ['jarsigner', '-verify', apkpath]
+    p = FDroidPopen(args)
+    if p.returncode != 0:
+        logging.critical(apkpath + " has a bad signature!")
+        return None
+
+    with zipfile.ZipFile(apkpath, 'r') as apk:
+
+        certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
+
+        if len(certs) < 1:
+            logging.error("Found no signing certificates on %s" % apkpath)
+            return None
+        if len(certs) > 1:
+            logging.error("Found multiple signing certificates on %s" % apkpath)
+            return None
+
+        cert = apk.read(certs[0])
+
+    content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
+    if content.getComponentByName('contentType') != rfc2315.signedData:
+        logging.error("Unexpected format.")
+        return None
+
+    content = decoder.decode(content.getComponentByName('content'),
+                             asn1Spec=rfc2315.SignedData())[0]
+    try:
+        certificates = content.getComponentByName('certificates')
+    except PyAsn1Error:
+        logging.error("Certificates not found.")
+        return None
+
+    cert_encoded = encoder.encode(certificates)[4:]
+
+    return md5(cert_encoded.encode('hex')).hexdigest()
+
+
 def scan_apks(apps, apkcache, repodir, knownapks):
     """Scan the apks in the given repo directory.
 
@@ -466,18 +524,8 @@ def scan_apks(apps, apkcache, repodir, knownapks):
                 thisinfo['sha256'] = sha.hexdigest()
 
             # Get the signature (or md5 of, to be precise)...
-            getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
-            if not os.path.exists(getsig_dir + "/getsig.class"):
-                logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
-                sys.exit(1)
-            p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
-                             'getsig', os.path.join(os.getcwd(), apkfile)])
-            thisinfo['sig'] = None
-            for line in p.output.splitlines():
-                if line.startswith('Result:'):
-                    thisinfo['sig'] = line[7:].strip()
-                    break
-            if p.returncode != 0 or not thisinfo['sig']:
+            thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
+            if not thisinfo['sig']:
                 logging.critical("Failed to get apk signature")
                 sys.exit(1)
 
@@ -716,7 +764,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
 
         def linkres(appid):
             if appid in apps:
-                return ("fdroid:app" + appid, apps[appid]['Name'])
+                return ("fdroid.app:" + appid, apps[appid]['Name'])
             raise MetaDataException("Cannot resolve app id " + appid)
 
         addElement('desc',
@@ -1051,7 +1099,7 @@ def main():
 
         if bestver == 0:
             if app['Name'] is None:
-                app['Name'] = appid
+                app['Name'] = app['Auto Name'] or appid
             app['icon'] = None
             logging.warn("Application " + appid + " has no packages")
         else:
index 60983febf39c15182b39719d5e5ccda3ada82b9c..9139d37ac06e7e96a30f32a3638d9281a4de16dc 100644 (file)
@@ -19,8 +19,6 @@
 
 import sys
 import os
-import shutil
-import subprocess
 import glob
 from optparse import OptionParser
 import logging
@@ -80,30 +78,16 @@ def main():
                 os.remove(remoteapk)
             url = 'https://f-droid.org/repo/' + apkfilename
             logging.info("...retrieving " + url)
-            p = FDroidPopen(['wget', url], cwd=tmp_dir)
+            p = FDroidPopen(['wget', '-nv', url], cwd=tmp_dir)
             if p.returncode != 0:
                 raise FDroidException("Failed to get " + apkfilename)
 
-            thisdir = os.path.join(tmp_dir, 'this_apk')
-            thatdir = os.path.join(tmp_dir, 'that_apk')
-            for d in [thisdir, thatdir]:
-                if os.path.exists(d):
-                    shutil.rmtree(d)
-                os.mkdir(d)
-
-            if subprocess.call(['jar', 'xf',
-                                os.path.join("..", "..", unsigned_dir, apkfilename)],
-                               cwd=thisdir) != 0:
-                raise FDroidException("Failed to unpack local build of " + apkfilename)
-            if subprocess.call(['jar', 'xf',
-                                os.path.join("..", "..", remoteapk)],
-                               cwd=thatdir) != 0:
-                raise FDroidException("Failed to unpack remote build of " + apkfilename)
-
-            p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir)
-            lines = p.output.splitlines()
-            if len(lines) != 1 or 'META-INF' not in lines[0]:
-                raise FDroidException("Unexpected diff output - " + p.output)
+            compare_result = common.compare_apks(
+                os.path.join(unsigned_dir, apkfilename),
+                remoteapk,
+                tmp_dir)
+            if compare_result:
+                raise FDroidException(compare_result)
 
             logging.info("...successfully verified")
             verified += 1
index 4069d820d19df8c5251cf6ba3b1a66510f466e9e..6e50b663e420bd9e362035c40ce2e2ab077dc3a8 100755 (executable)
@@ -38,11 +38,6 @@ fi
 
 export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
 
-#------------------------------------------------------------------------------#
-# run local build
-cd $WORKSPACE/fdroidserver/getsig
-./make.sh
-
 
 #------------------------------------------------------------------------------#
 # run local tests
index 44efef29a5e2fd12de3ffc2d3e0c0be735276795..2f67f345942f2fff538bce55d558dc5c73ba2d0d 100755 (executable)
@@ -89,9 +89,12 @@ cachefiles = [
     ('gradle-1.12-bin.zip',
      'https://services.gradle.org/distributions/gradle-1.12-bin.zip',
      '8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
+    ('gradle-2.1-bin.zip',
+     'https://services.gradle.org/distributions/gradle-2.1-bin.zip',
+     '3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
     ('Kivy-1.7.2.tar.gz',
      'https://pypi.python.org/packages/source/K/Kivy/Kivy-1.7.2.tar.gz',
-     '0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1')
+     '0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1'),
     ]
 
 if config['arch64']:
index 82936c398a1c3c899369e295fa0257bb8c28aadd..669cfaee60174696d59d2485f29cb4ab9aa7810d 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -1,14 +1,8 @@
 #!/usr/bin/env python2
 
 from setuptools import setup
-import os
-import subprocess
 import sys
 
-if not os.path.exists('fdroidserver/getsig/getsig.class'):
-    subprocess.check_output('cd fdroidserver/getsig && javac getsig.java',
-                            shell=True)
-
 setup(name='fdroidserver',
       version='0.2.1',
       description='F-Droid Server Tools',
@@ -25,8 +19,6 @@ setup(name='fdroidserver',
                   'examples/makebs.config.py',
                   'examples/opensc-fdroid.cfg',
                   'examples/fdroid-icon.png']),
-          ('fdroidserver/getsig',
-              ['fdroidserver/getsig/getsig.class']),
           ],
       install_requires=[
           'mwclient',
@@ -34,6 +26,8 @@ setup(name='fdroidserver',
           'Pillow',
           'python-magic',
           'apache-libcloud >= 0.14.1',
+          'pyasn1',
+          'pyasn1-modules',
           ],
       classifiers=[
           'Development Status :: 3 - Alpha',
index b9a62d892b8e74b4dd9d56a403a19d9ab0b57799..c12c78a7b01bb51fbc159d85721080388de6f62d 100755 (executable)
@@ -92,6 +92,15 @@ cd $WORKSPACE
 ./hooks/pre-commit
 
 
+#------------------------------------------------------------------------------#
+echo_header "test python getsig replacement"
+
+cd $WORKSPACE/tests/getsig
+./make.sh
+cd $WORKSPACE/tests
+./update.TestCase
+
+
 #------------------------------------------------------------------------------#
 echo_header "create a source tarball and use that to build a repo"
 
@@ -101,8 +110,6 @@ $python setup.py sdist
 REPOROOT=`create_test_dir`
 cd $REPOROOT
 tar xzf `ls -1 $WORKSPACE/dist/fdroidserver-*.tar.gz | sort -n | tail -1`
-cd $REPOROOT/fdroidserver-*/fdroidserver/getsig
-./make.sh
 cd $REPOROOT
 ./fdroidserver-*/fdroid init
 copy_apks_into_repo $REPOROOT
@@ -313,5 +320,6 @@ $fdroid init --keystore NONE
 test -e opensc-fdroid.cfg
 test ! -e NONE
 
+rm -rf $WORKSPACE/fdroidserver.egg-info/
 
 echo SUCCESS
diff --git a/tests/update.TestCase b/tests/update.TestCase
new file mode 100755 (executable)
index 0000000..88b429f
--- /dev/null
@@ -0,0 +1,82 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
+
+import inspect
+import optparse
+import os
+import sys
+import unittest
+
+localmodule = os.path.realpath(os.path.join(
+        os.path.dirname(inspect.getfile(inspect.currentframe())),
+        '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0,localmodule)
+
+import fdroidserver.common
+import fdroidserver.update
+from fdroidserver.common import FDroidPopen, SilentPopen
+
+class UpdateTest(unittest.TestCase):
+    '''fdroid update'''
+
+    def javagetsig(self, apkfile):
+        getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
+        if not os.path.exists(getsig_dir + "/getsig.class"):
+            logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
+            sys.exit(1)
+        p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
+                         'getsig', os.path.join(os.getcwd(), apkfile)])
+        sig = None
+        for line in p.output.splitlines():
+            if line.startswith('Result:'):
+                sig = line[7:].strip()
+                break
+        if p.returncode == 0:
+            return sig
+        else:
+            return None
+        
+    def testGoodGetsig(self):
+        apkfile = os.path.join(os.path.dirname(__file__), 'urzip.apk')
+        sig = self.javagetsig(apkfile)
+        self.assertIsNotNone(sig, "sig is None")
+        pysig = fdroidserver.update.getsig(apkfile)
+        self.assertIsNotNone(pysig, "pysig is None")        
+        self.assertEquals(sig, fdroidserver.update.getsig(apkfile),
+                          "python sig not equal to java sig!")
+        self.assertEquals(len(sig), len(pysig),
+                          "the length of the two sigs are different!")
+        try:
+            self.assertEquals(sig.decode('hex'), pysig.decode('hex'),
+                              "the length of the two sigs are different!")
+        except TypeError as e:
+            print e
+            self.assertTrue(False, 'TypeError!')
+
+    def testBadGetsig(self):
+        apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badsig.apk')
+        sig = self.javagetsig(apkfile)
+        self.assertIsNone(sig, "sig should be None: " + str(sig))
+        pysig = fdroidserver.update.getsig(apkfile)
+        self.assertIsNone(pysig, "python sig should be None: " + str(sig))
+
+        apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badcert.apk')
+        sig = self.javagetsig(apkfile)
+        self.assertIsNone(sig, "sig should be None: " + str(sig))
+        pysig = fdroidserver.update.getsig(apkfile)
+        self.assertIsNone(pysig, "python sig should be None: " + str(sig))
+
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+
+    newSuite = unittest.TestSuite()
+    newSuite.addTest(unittest.makeSuite(UpdateTest))
+    unittest.main()
diff --git a/tests/urzip-badcert.apk b/tests/urzip-badcert.apk
new file mode 100644 (file)
index 0000000..cd7dd08
Binary files /dev/null and b/tests/urzip-badcert.apk differ
diff --git a/tests/urzip-badsig.apk b/tests/urzip-badsig.apk
new file mode 100644 (file)
index 0000000..89e106b
Binary files /dev/null and b/tests/urzip-badsig.apk differ
diff --git a/updateplugin b/updateplugin
deleted file mode 100755 (executable)
index fe798e9..0000000
+++ /dev/null
@@ -1 +0,0 @@
-scp -r wp-fdroid/ fdroid@f-droid.org:/home/fdroid/public_html/wp-content/plugins
index f70017f20cea6492fec1776109a79c38d7ec453d..f412eef96441be578de2cf5f8f611ac189001f72 100644 (file)
@@ -954,7 +954,7 @@ function linkify($vars) {
        $retvar = '';
        foreach($vars as $k => $v) {
                if($k!==null && $v!==null && $v!='')
-                       $retvar .= $k.'='.$v.'&';
+                       $retvar .= $k.'='.urlencode($v).'&';
        }
        return substr($retvar,0,-1);
 }