Support for publishing signed binaries from elsewhere

Done after verifying that they match ones built using a recipe.
Everything in the metadata should be the same as normal, with the
addition of the Binaries: directive to specify where (with pattern
substitution) to get the binaries from.

Publishing only takes place if there is a proper match. (Which seems
very unlikely to be the case unless the exact same toolchain is used, so
I would imagine that unless the person building and signing the incoming
binaries uses fdroidserver to build them, probably the exact same
buildserver id, they will not match. But at least we have the
functionality to support that.)

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 616da10f43f60ebbbbf697b65efa9b7a1f41de7a..9af1779a782e419b95412607479744158d8c30ab 100644 (file)
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -1785,3 +1785,40 @@ def place_srclib(root_dir, number, libpath):
                 o.write(line)
         if not placed:
             o.write('android.library.reference.%d=%s\n' % (number, relpath))
+
+
+def compare_apks(apk1, apk2, tmp_dir):
+    """Compare two apks
+
+    Returns None if the apk content is the same (apart from the signing key),
+    otherwise a string describing what's different, or what went wrong when
+    trying to do the comparison.
+    """
+
+    thisdir = os.path.join(tmp_dir, 'this_apk')
+    thatdir = os.path.join(tmp_dir, 'that_apk')
+    for d in [thisdir, thatdir]:
+        if os.path.exists(d):
+            shutil.rmtree(d)
+        os.mkdir(d)
+
+    if subprocess.call(['jar', 'xf',
+                        os.path.abspath(apk1)],
+                       cwd=thisdir) != 0:
+        return("Failed to unpack " + apk1)
+    if subprocess.call(['jar', 'xf',
+                        os.path.abspath(apk2)],
+                       cwd=thatdir) != 0:
+        return("Failed to unpack " + apk2)
+
+    p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir,
+            output=False)
+    lines = p.output.splitlines()
+    if len(lines) != 1 or 'META-INF' not in lines[0]:
+        return("Unexpected diff output - " + p.output)
+
+    # If we get here, it seems like they're the same!
+    return None
+
+
+

diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py
index 9499432627941d596c2520ecc2a9f63f53d6b3eb..8d7e57b24957be6fd094fa527b1c72bd7e741fe7 100644 (file)
--- a/fdroidserver/metadata.py
+++ b/fdroidserver/metadata.py
@@ -57,6 +57,7 @@ app_defaults = OrderedDict([
     ('Requires Root', False),
     ('Repo Type', ''),
     ('Repo', ''),
+    ('Binaries', ''),
     ('Maintainer Notes', []),
     ('Archive Policy', None),
     ('Auto Update Mode', 'None'),
@@ -197,6 +198,11 @@ valuetypes = {
                    ["Repo Type"],
                    []),
 
+    FieldValidator("Binaries",
+                   r'^http[s]?://', None,
+                   ["Binaries"],
+                   []),
+
     FieldValidator("Archive Policy",
                    r'^[0-9]+ versions$', None,
                    ["Archive Policy"],
@@ -851,6 +857,8 @@ def write_metadata(dest, app):
     if app['Repo Type']:
         writefield('Repo Type')
         writefield('Repo')
+        if app['Binaries']:
+            writefield('Binaries')
         mf.write('\n')
     for build in app['builds']:
 

diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py
index efda4c414afed8bf9a9c965e65599fc2b91ab309..520a72a0b9d871f66a7078901d68dd75a9c3b133 100644 (file)
--- a/fdroidserver/publish.py
+++ b/fdroidserver/publish.py
@@ -111,60 +111,113 @@ def main():
                 continue
         logging.info("Processing " + apkfile)
 
-        # Figure out the key alias name we'll use. Only the first 8
-        # characters are significant, so we'll use the first 8 from
-        # the MD5 of the app's ID and hope there are no collisions.
-        # If a collision does occur later, we're going to have to
-        # come up with a new alogrithm, AND rename all existing keys
-        # in the keystore!
-        if appid in config['keyaliases']:
-            # For this particular app, the key alias is overridden...
-            keyalias = config['keyaliases'][appid]
-            if keyalias.startswith('@'):
+        # There ought to be valid metadata for this app, otherwise why are we
+        # trying to publish it?
+        if not appid in allapps:
+            logging.error("Unexpected {0} found in unsigned directory"
+                    .format(apkfilename))
+            sys.exit(1)
+        app = allapps[appid]
+
+        if 'Binaries' in app:
+
+            # It's an app where we build from source, and verify the apk
+            # contents against a developer's binary, and then publish their
+            # version if everything checks out.
+
+            # Need the version name for the version code...
+            versionname = None
+            for build in app['builds']:
+                if build['vercode'] == vercode:
+                    versionname = build['version']
+                    break
+            if not versionname:
+                logging.error("...no defined build for version code {0}"
+                        .format(vercode))
+                continue
+
+            # Figure out where the developer's binary is supposed to come from...
+            url = app['Binaries']
+            url = url.replace('%v', versionname)
+            url = url.replace('%c', str(vercode))
+
+            # Grab the binary from where the developer publishes it...
+            logging.info("...retrieving " + url)
+            srcapk = os.path.join(tmp_dir, url.split('/')[-1])
+            p = FDroidPopen(['wget', '-nv', url], cwd=tmp_dir)
+            if p.returncode != 0 or not os.path.exists(srcapk):
+                logging.error("...failed to retrieve " + url +
+                        " - publish skipped")
+                continue
+
+            # Compare our unsigned one with the downloaded one...
+            compare_result = common.compare_apks(srcapk, apkfile, tmp_dir)
+            if compare_result:
+                logging.error("...verfication failed - publish skipped : "
+                        + compare_result)
+                continue
+
+            # Success! So move the downloaded file to the repo...
+            shutil.move(srcapk, os.path.join(output_dir, apkfilename))
+
+        else:
+
+            # It's a 'normal' app, i.e. we sign and publish it...
+
+            # Figure out the key alias name we'll use. Only the first 8
+            # characters are significant, so we'll use the first 8 from
+            # the MD5 of the app's ID and hope there are no collisions.
+            # If a collision does occur later, we're going to have to
+            # come up with a new alogrithm, AND rename all existing keys
+            # in the keystore!
+            if appid in config['keyaliases']:
+                # For this particular app, the key alias is overridden...
+                keyalias = config['keyaliases'][appid]
+                if keyalias.startswith('@'):
+                    m = md5.new()
+                    m.update(keyalias[1:])
+                    keyalias = m.hexdigest()[:8]
+            else:
                 m = md5.new()
-                m.update(keyalias[1:])
+                m.update(appid)
                 keyalias = m.hexdigest()[:8]
-        else:
-            m = md5.new()
-            m.update(appid)
-            keyalias = m.hexdigest()[:8]
-        logging.info("Key alias: " + keyalias)
-
-        # See if we already have a key for this application, and
-        # if not generate one...
-        p = FDroidPopen(['keytool', '-list',
-                         '-alias', keyalias, '-keystore', config['keystore'],
-                         '-storepass:file', config['keystorepassfile']])
-        if p.returncode != 0:
-            logging.info("Key does not exist - generating...")
-            p = FDroidPopen(['keytool', '-genkey',
-                             '-keystore', config['keystore'],
-                             '-alias', keyalias,
-                             '-keyalg', 'RSA', '-keysize', '2048',
-                             '-validity', '10000',
+            logging.info("Key alias: " + keyalias)
+
+            # See if we already have a key for this application, and
+            # if not generate one...
+            p = FDroidPopen(['keytool', '-list',
+                             '-alias', keyalias, '-keystore', config['keystore'],
+                             '-storepass:file', config['keystorepassfile']])
+            if p.returncode != 0:
+                logging.info("Key does not exist - generating...")
+                p = FDroidPopen(['keytool', '-genkey',
+                                 '-keystore', config['keystore'],
+                                 '-alias', keyalias,
+                                 '-keyalg', 'RSA', '-keysize', '2048',
+                                 '-validity', '10000',
+                                 '-storepass:file', config['keystorepassfile'],
+                                 '-keypass:file', config['keypassfile'],
+                                 '-dname', config['keydname']])
+                # TODO keypass should be sent via stdin
+                if p.returncode != 0:
+                    raise BuildException("Failed to generate key")
+
+            # Sign the application...
+            p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
                              '-storepass:file', config['keystorepassfile'],
-                             '-keypass:file', config['keypassfile'],
-                             '-dname', config['keydname']])
+                             '-keypass:file', config['keypassfile'], '-sigalg',
+                             'MD5withRSA', '-digestalg', 'SHA1',
+                             apkfile, keyalias])
             # TODO keypass should be sent via stdin
             if p.returncode != 0:
-                raise BuildException("Failed to generate key")
-
-        # Sign the application...
-        p = FDroidPopen(['jarsigner', '-keystore', config['keystore'],
-                         '-storepass:file', config['keystorepassfile'],
-                         '-keypass:file', config['keypassfile'], '-sigalg',
-                         'MD5withRSA', '-digestalg', 'SHA1',
-                         apkfile, keyalias])
-        # TODO keypass should be sent via stdin
-        if p.returncode != 0:
-            raise BuildException("Failed to sign application")
-
-        # Zipalign it...
-        p = FDroidPopen([config['zipalign'], '-v', '4', apkfile,
-                         os.path.join(output_dir, apkfilename)])
-        if p.returncode != 0:
-            raise BuildException("Failed to align application")
-        os.remove(apkfile)
+                raise BuildException("Failed to sign application")
+
+            # Zipalign it...
+            p = FDroidPopen([config['zipalign'], '-v', '4', apkfile,
+                             os.path.join(output_dir, apkfilename)])
+            if p.returncode != 0:
+                raise BuildException("Failed to align application")
+            os.remove(apkfile)
 
         # Move the source tarball into the output directory...
         tarfilename = apkfilename[:-4] + '_src.tar.gz'

diff --git a/fdroidserver/verify.py b/fdroidserver/verify.py
index 60983febf39c15182b39719d5e5ccda3ada82b9c..041b5cc7e4bbcb82810403619dcc17874bc64ec1 100644 (file)
--- a/fdroidserver/verify.py
+++ b/fdroidserver/verify.py
@@ -19,8 +19,6 @@
 
 import sys
 import os
-import shutil
-import subprocess
 import glob
 from optparse import OptionParser
 import logging
@@ -80,30 +78,16 @@ def main():
                 os.remove(remoteapk)
             url = 'https://f-droid.org/repo/' + apkfilename
             logging.info("...retrieving " + url)
-            p = FDroidPopen(['wget', url], cwd=tmp_dir)
+            p = FDroidPopen(['wget', '-nv', url], cwd=tmp_dir)
             if p.returncode != 0:
                 raise FDroidException("Failed to get " + apkfilename)
 
-            thisdir = os.path.join(tmp_dir, 'this_apk')
-            thatdir = os.path.join(tmp_dir, 'that_apk')
-            for d in [thisdir, thatdir]:
-                if os.path.exists(d):
-                    shutil.rmtree(d)
-                os.mkdir(d)
-
-            if subprocess.call(['jar', 'xf',
-                                os.path.join("..", "..", unsigned_dir, apkfilename)],
-                               cwd=thisdir) != 0:
-                raise FDroidException("Failed to unpack local build of " + apkfilename)
-            if subprocess.call(['jar', 'xf',
-                                os.path.join("..", "..", remoteapk)],
-                               cwd=thatdir) != 0:
-                raise FDroidException("Failed to unpack remote build of " + apkfilename)
-
-            p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir)
-            lines = p.output.splitlines()
-            if len(lines) != 1 or 'META-INF' not in lines[0]:
-                raise FDroidException("Unexpected diff output - " + p.output)
+            compare_result = common.compare_apks(
+                    os.path.join(unsigned_dir, apkfilename),
+                    remoteapk,
+                    tmp_dir)
+            if compare_result:
+                raise FDroidException(compare_result)
 
             logging.info("...successfully verified")
             verified += 1