chiark / gitweb /
update: handle APKs with a blank versionName
[fdroidserver.git] / fdroidserver / index.py
index 44b1a1b9b4140b5db6d7813a1523328a2a240bd6..61552138fcab4e5e54d2b146a53a2006378c699b 100644 (file)
@@ -27,19 +27,21 @@ import logging
 import os
 import re
 import shutil
-import sys
 import tempfile
 import urllib.parse
 import zipfile
+import calendar
 from binascii import hexlify, unhexlify
 from datetime import datetime
 from xml.dom.minidom import Document
 
-import requests
-
-from fdroidserver import metadata, signindex, common
-from fdroidserver.common import FDroidPopen, FDroidPopenBytes
-from fdroidserver.metadata import MetaDataException
+from . import _
+from . import common
+from . import metadata
+from . import net
+from . import signindex
+from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
+from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
 
 
 def make(apps, sortedids, apks, repodir, archive):
@@ -61,27 +63,8 @@ def make(apps, sortedids, apks, repodir, archive):
             return "fdroid.app:" + appid, apps[appid].Name
         raise MetaDataException("Cannot resolve app id " + appid)
 
-    nosigningkey = False
     if not common.options.nosign:
-        if 'repo_keyalias' not in common.config:
-            nosigningkey = True
-            logging.critical("'repo_keyalias' not found in config.py!")
-        if 'keystore' not in common.config:
-            nosigningkey = True
-            logging.critical("'keystore' not found in config.py!")
-        if 'keystorepass' not in common.config:
-            nosigningkey = True
-            logging.critical("'keystorepass' not found in config.py!")
-        if 'keypass' not in common.config:
-            nosigningkey = True
-            logging.critical("'keypass' not found in config.py!")
-        if not os.path.exists(common.config['keystore']):
-            nosigningkey = True
-            logging.critical("'" + common.config['keystore'] + "' does not exist!")
-        if nosigningkey:
-            logging.warning("`fdroid update` requires a signing key, you can create one using:")
-            logging.warning("\tfdroid update --create-key")
-            sys.exit(1)
+        common.assert_config_keystore(common.config)
 
     repodict = collections.OrderedDict()
     repodict['timestamp'] = datetime.utcnow()
@@ -108,7 +91,7 @@ def make(apps, sortedids, apks, repodir, archive):
     for mirror in sorted(common.config.get('mirrors', [])):
         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
-            logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
+            logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
             mirrorcheckfailed = True
         # must end with / or urljoin strips a whole path segment
         if mirror.endswith('/'):
@@ -116,11 +99,10 @@ def make(apps, sortedids, apks, repodir, archive):
         else:
             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
     for mirror in common.config.get('servergitmirrors', []):
-        mirror = get_raw_mirror(mirror)
-        if mirror is not None:
-            mirrors.append(mirror + '/')
+        for url in get_mirror_service_urls(mirror):
+            mirrors.append(url + '/' + repodir)
     if mirrorcheckfailed:
-        sys.exit(1)
+        raise FDroidException(_("Malformed repository mirrors."))
     if mirrors:
         repodict['mirrors'] = mirrors
 
@@ -149,29 +131,43 @@ def make(apps, sortedids, apks, repodir, archive):
             elif all(isinstance(item, str) for item in common.config[key]):
                 packageNames = common.config[key]
             else:
-                raise TypeError('only accepts strings, lists, and tuples')
+                raise TypeError(_('only accepts strings, lists, and tuples'))
         requestsdict[command] = packageNames
 
-    make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
-    make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
+    fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
+
+    make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
+            fdroid_signing_key_fingerprints)
+    make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
+            fdroid_signing_key_fingerprints)
 
 
-def make_v1(apps, packages, repodir, repodict, requestsdict):
+def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 
     def _index_encoder_default(obj):
         if isinstance(obj, set):
-            return list(obj)
+            return sorted(list(obj))
         if isinstance(obj, datetime):
-            return int(obj.timestamp() * 1000)  # Java expects milliseconds
+            # Java prefers milliseconds
+            # we also need to accound for time zone/daylight saving time
+            return int(calendar.timegm(obj.timetuple()) * 1000)
+        if isinstance(obj, dict):
+            d = collections.OrderedDict()
+            for key in sorted(obj.keys()):
+                d[key] = obj[key]
+            return d
         raise TypeError(repr(obj) + " is not JSON serializable")
 
     output = collections.OrderedDict()
     output['repo'] = repodict
     output['requests'] = requestsdict
 
+    # establish sort order of the index
+    v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
+
     appslist = []
     output['apps'] = appslist
-    for appid, appdict in apps.items():
+    for packageName, appdict in apps.items():
         d = collections.OrderedDict()
         appslist.append(d)
         for k, v in sorted(appdict.items()):
@@ -192,17 +188,41 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
                 k = 'suggestedVersionName'
             elif k == 'AutoName':
-                if 'Name' not in apps[appid]:
+                if 'Name' not in apps[packageName]:
                     d['name'] = v
                 continue
             else:
                 k = k[:1].lower() + k[1:]
             d[k] = v
 
+    # establish sort order in localized dicts
+    for app in output['apps']:
+        localized = app.get('localized')
+        if localized:
+            lordered = collections.OrderedDict()
+            for lkey, lvalue in sorted(localized.items()):
+                lordered[lkey] = collections.OrderedDict()
+                for ikey, iname in sorted(lvalue.items()):
+                    lordered[lkey][ikey] = iname
+            app['localized'] = lordered
+
     output_packages = collections.OrderedDict()
     output['packages'] = output_packages
     for package in packages:
         packageName = package['packageName']
+        if packageName not in apps:
+            logging.info(_('Ignoring package without metadata: ') + package['apkName'])
+            continue
+        if not package.get('versionName'):
+            app = apps[packageName]
+            versionCodeStr = str(package['versionCode'])  # TODO build.versionCode should be int!
+            for build in app['builds']:
+                if build['versionCode'] == versionCodeStr:
+                    versionName = build.get('versionName')
+                    logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}')
+                                 .format(apkfilename=package['apkName'], version=versionName))
+                    package['versionName'] = versionName
+                    break
         if packageName in output_packages:
             packagelist = output_packages[packageName]
         else:
@@ -226,13 +246,49 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
             json.dump(output, fp, default=_index_encoder_default)
 
     if common.options.nosign:
-        logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
+        logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
     else:
         signindex.config = common.config
         signindex.sign_index_v1(repodir, json_name)
 
 
-def make_v0(apps, apks, repodir, repodict, requestsdict):
+def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
+    """Sorts the supplied list to ensure a deterministic sort order for
+    package entries in the index file. This sort-order also expresses
+    installation preference to the clients.
+    (First in this list = first to install)
+
+    :param packages: list of packages which need to be sorted before but into index file.
+    """
+
+    GROUP_DEV_SIGNED = 1
+    GROUP_FDROID_SIGNED = 2
+    GROUP_OTHER_SIGNED = 3
+
+    def v1_sort_keys(package):
+        packageName = package.get('packageName', None)
+
+        sig = package.get('signer', None)
+
+        dev_sig = common.metadata_find_developer_signature(packageName)
+        group = GROUP_OTHER_SIGNED
+        if dev_sig and dev_sig == sig:
+            group = GROUP_DEV_SIGNED
+        else:
+            fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
+            if fdroidsig and fdroidsig == sig:
+                group = GROUP_FDROID_SIGNED
+
+        versionCode = None
+        if package.get('versionCode', None):
+            versionCode = -int(package['versionCode'])
+
+        return(packageName, group, sig, versionCode)
+
+    packages.sort(key=v1_sort_keys)
+
+
+def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
     """
     aka index.jar aka index.xml
     """
@@ -260,6 +316,35 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
         el.appendChild(doc.createCDATASection(value))
         parent.appendChild(el)
 
+    def addElementCheckLocalized(name, app, key, doc, parent, default=''):
+        '''Fill in field from metadata or localized block
+
+        For name/summary/description, they can come only from the app source,
+        or from a dir in fdroiddata.  They can be entirely missing from the
+        metadata file if there is localized versions.  This will fetch those
+        from the localized version if its not available in the metadata file.
+        '''
+
+        el = doc.createElement(name)
+        value = app.get(key)
+        lkey = key[:1].lower() + key[1:]
+        localized = app.get('localized')
+        if not value and localized:
+            for lang in ['en-US'] + [x for x in localized.keys()]:
+                if not lang.startswith('en'):
+                    continue
+                if lang in localized:
+                    value = localized[lang].get(lkey)
+                    if value:
+                        break
+        if not value and localized and len(localized) > 1:
+            lang = list(localized.keys())[0]
+            value = localized[lang].get(lkey)
+        if not value:
+            value = default
+        el.appendChild(doc.createTextNode(value))
+        parent.appendChild(el)
+
     root = doc.createElement("fdroid")
     doc.appendChild(root)
 
@@ -295,9 +380,27 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
 
         # Get a list of the apks for this app...
         apklist = []
+        apksbyversion = collections.defaultdict(lambda: [])
         for apk in apks:
-            if apk['packageName'] == appid:
-                apklist.append(apk)
+            if apk.get('versionCode') and apk.get('packageName') == appid:
+                apksbyversion[apk['versionCode']].append(apk)
+        for versionCode, apksforver in apksbyversion.items():
+            fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
+            fdroid_signed_apk = None
+            name_match_apk = None
+            for x in apksforver:
+                if fdroidsig and x.get('signer', None) == fdroidsig:
+                    fdroid_signed_apk = x
+                if common.apk_release_filename.match(x.get('apkName', '')):
+                    name_match_apk = x
+            # choose which of the available versions is most
+            # suiteable for index v0
+            if fdroid_signed_apk:
+                apklist.append(fdroid_signed_apk)
+            elif name_match_apk:
+                apklist.append(name_match_apk)
+            else:
+                apklist.append(apksforver[0])
 
         if len(apklist) == 0:
             continue
@@ -311,16 +414,16 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
         if app.lastUpdated:
             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
-        addElement('name', app.Name, doc, apel)
-        addElement('summary', app.Summary, doc, apel)
+
+        addElementCheckLocalized('name', app, 'Name', doc, apel)
+        addElementCheckLocalized('summary', app, 'Summary', doc, apel)
+
         if app.icon:
             addElement('icon', app.icon, doc, apel)
 
-        if app.get('Description'):
-            description = app.Description
-        else:
-            description = '<p>No description available</p>'
-        addElement('desc', description, doc, apel)
+        addElementCheckLocalized('desc', app, 'Description', doc, apel,
+                                 '<p>No description available</p>')
+
         addElement('license', app.License, doc, apel)
         if app.Categories:
             addElement('categories', ','.join(app.Categories), doc, apel)
@@ -338,6 +441,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
+        addElementNonEmpty('liberapay', app.LiberapayID, doc, apel)
 
         # These elements actually refer to the current version (i.e. which
         # one is recommended. They are historically mis-named, and need
@@ -362,10 +466,16 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
 
         # Check for duplicates - they will make the client unhappy...
         for i in range(len(apklist) - 1):
-            if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
-                logging.critical("duplicate versions: '%s' - '%s'" % (
-                    apklist[i]['apkName'], apklist[i + 1]['apkName']))
-                sys.exit(1)
+            first = apklist[i]
+            second = apklist[i + 1]
+            if first['versionCode'] == second['versionCode'] \
+               and first['sig'] == second['sig']:
+                if first['hash'] == second['hash']:
+                    raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
+                        repodir, first['apkName'], second['apkName']))
+                else:
+                    raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
+                        repodir, first['apkName'], second['apkName']))
 
         current_version_code = 0
         current_version_file = None
@@ -379,7 +489,17 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
 
             apkel = doc.createElement("package")
             apel.appendChild(apkel)
-            addElement('version', apk['versionName'], doc, apkel)
+
+            versionName = apk.get('versionName')
+            if not versionName:
+                versionCodeStr = str(apk['versionCode'])  # TODO build.versionCode should be int!
+                for build in app.builds:
+                    if build['versionCode'] == versionCodeStr and 'versionName' in build:
+                        versionName = build['versionName']
+                        break
+            if versionName:
+                addElement('version', versionName, doc, apkel)
+
             addElement('versioncode', str(apk['versionCode']), doc, apkel)
             addElement('apkname', apk['apkName'], doc, apkel)
             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
@@ -440,7 +560,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
                 and repodir == 'repo':  # only create these
             namefield = common.config['current_version_name_source']
             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
-            apklinkname = sanitized_name + b'.apk'
+            apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
             if os.path.islink(apklinkname):
                 os.remove(apklinkname)
@@ -465,17 +585,16 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
     if 'repo_keyalias' in common.config:
 
         if common.options.nosign:
-            logging.info("Creating unsigned index in preparation for signing")
+            logging.info(_("Creating unsigned index in preparation for signing"))
         else:
-            logging.info("Creating signed index with this key (SHA256):")
+            logging.info(_("Creating signed index with this key (SHA256):"))
             logging.info("%s" % repo_pubkey_fingerprint)
 
         # Create a jar of the index...
         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
         if p.returncode != 0:
-            logging.critical("Failed to create {0}".format(jar_output))
-            sys.exit(1)
+            raise FDroidException("Failed to create {0}".format(jar_output))
 
         # Sign the index...
         signed = os.path.join(repodir, 'index.jar')
@@ -512,51 +631,70 @@ def extract_pubkey():
             msg = "Failed to get repo pubkey!"
             if common.config['keystore'] == 'NONE':
                 msg += ' Is your crypto smartcard plugged in?'
-            logging.critical(msg)
-            sys.exit(1)
+            raise FDroidException(msg)
         pubkey = p.output
     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
     return hexlify(pubkey), repo_pubkey_fingerprint
 
 
-# Get raw URL from git service for mirroring
-def get_raw_mirror(url):
-    # Divide urls in parts
-    url = url.split("/")
+def get_mirror_service_urls(url):
+    '''Get direct URLs from git service for use by fdroidclient
+
+    Via 'servergitmirrors', fdroidserver can create and push a mirror
+    to certain well known git services like gitlab or github.  This
+    will always use the 'master' branch since that is the default
+    branch in git. The files are then accessible via alternate URLs,
+    where they are served in their raw format via a CDN rather than
+    from git.
+    '''
 
-    # Get the hostname
-    hostname = url[2]
+    if url.startswith('git@'):
+        url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 
-    # fdroidserver will use always 'master' branch for git-mirroring
+    segments = url.split("/")
+
+    if segments[4].endswith('.git'):
+        segments[4] = segments[4][:-4]
+
+    hostname = segments[2]
+    user = segments[3]
+    repo = segments[4]
     branch = "master"
     folder = "fdroid"
 
+    urls = []
     if hostname == "github.com":
-        # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
-        url[2] = "raw.githubusercontent.com"
-        url.extend([branch, folder])
+        # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
+        segments[2] = "raw.githubusercontent.com"
+        segments.extend([branch, folder])
+        urls.append('/'.join(segments))
     elif hostname == "gitlab.com":
-        # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
-        url.extend(["raw", branch, folder])
-    else:
-        return None
+        # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser
+        # This is because the `raw` URLs are not served with the correct mime types, so any
+        # index.html which is put in the repo will not be rendered. Putting an index.html file in
+        # the repo root is a common way for to make information about the repo available to end user.
 
-    url = "/".join(url)
-    return url
+        # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
+        gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
+        urls.append('/'.join(gitlab_pages))
+        # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
+        gitlab_raw = segments + ['raw', branch, folder]
+        urls.append('/'.join(gitlab_raw))
+        return urls
 
+    return urls
 
-class VerificationException(Exception):
-    pass
 
-
-def download_repo_index(url_str, verify_fingerprint=True):
+def download_repo_index(url_str, etag=None, verify_fingerprint=True):
     """
     Downloads the repository index from the given :param url_str
     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 
     :raises: VerificationException() if the repository could not be verified
 
-    :return: The index in JSON format.
+    :return: A tuple consisting of:
+        - The index in JSON format or None if the index did not change
+        - The new eTag as returned by the HTTP request
     """
     url = urllib.parse.urlsplit(url_str)
 
@@ -564,26 +702,30 @@ def download_repo_index(url_str, verify_fingerprint=True):
     if verify_fingerprint:
         query = urllib.parse.parse_qs(url.query)
         if 'fingerprint' not in query:
-            raise VerificationException("No fingerprint in URL.")
+            raise VerificationException(_("No fingerprint in URL."))
         fingerprint = query['fingerprint'][0]
 
     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
-    r = requests.get(url.geturl())
+    download, new_etag = net.http_get(url.geturl(), etag)
+
+    if download is None:
+        return None, new_etag
 
     with tempfile.NamedTemporaryFile() as fp:
         # write and open JAR file
-        fp.write(r.content)
+        fp.write(download)
         jar = zipfile.ZipFile(fp)
 
         # verify that the JAR signature is valid
-        verify_jar_signature(fp.name)
+        logging.debug(_('Verifying index signature:'))
+        common.verify_jar_signature(fp.name)
 
         # get public key and its fingerprint from JAR
         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 
         # compare the fingerprint if verify_fingerprint is True
         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
-            raise VerificationException("The repository's fingerprint does not match.")
+            raise VerificationException(_("The repository's fingerprint does not match."))
 
         # load repository index from JSON
         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
@@ -593,17 +735,7 @@ def download_repo_index(url_str, verify_fingerprint=True):
         # turn the apps into App objects
         index["apps"] = [metadata.App(app) for app in index["apps"]]
 
-        return index
-
-
-def verify_jar_signature(file):
-    """
-    Verifies the signature of a given JAR file.
-
-    :raises: VerificationException() if the JAR's signature could not be verified
-    """
-    if not common.verify_apk_signature(file, jar=True):
-        raise VerificationException("The repository's index could not be verified.")
+        return index, new_etag
 
 
 def get_public_key_from_jar(jar):
@@ -618,9 +750,9 @@ def get_public_key_from_jar(jar):
     # extract certificate from jar
     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
     if len(certs) < 1:
-        raise VerificationException("Found no signing certificates for repository.")
+        raise VerificationException(_("Found no signing certificates for repository."))
     if len(certs) > 1:
-        raise VerificationException("Found multiple signing certificates for repository.")
+        raise VerificationException(_("Found multiple signing certificates for repository."))
 
     # extract public key from certificate
     public_key = common.get_certificate(jar.read(certs[0]))