chiark / gitweb /
update: handle APKs with a blank versionName
[fdroidserver.git] / fdroidserver / index.py
index 9263b84f84e62a04db19577f9da0fc173e2a8a45..61552138fcab4e5e54d2b146a53a2006378c699b 100644 (file)
@@ -27,18 +27,21 @@ import logging
 import os
 import re
 import shutil
-import sys
+import tempfile
 import urllib.parse
+import zipfile
+import calendar
 from binascii import hexlify, unhexlify
 from datetime import datetime
 from xml.dom.minidom import Document
 
-from fdroidserver import metadata, signindex, common
-from fdroidserver.common import FDroidPopen, FDroidPopenBytes
-from fdroidserver.metadata import MetaDataException
-
-options = None
-config = None
+from . import _
+from . import common
+from . import metadata
+from . import net
+from . import signindex
+from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
+from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
 
 
 def make(apps, sortedids, apks, repodir, archive):
@@ -60,66 +63,46 @@ def make(apps, sortedids, apks, repodir, archive):
             return "fdroid.app:" + appid, apps[appid].Name
         raise MetaDataException("Cannot resolve app id " + appid)
 
-    nosigningkey = False
-    if not options.nosign:
-        if 'repo_keyalias' not in config:
-            nosigningkey = True
-            logging.critical("'repo_keyalias' not found in config.py!")
-        if 'keystore' not in config:
-            nosigningkey = True
-            logging.critical("'keystore' not found in config.py!")
-        if 'keystorepass' not in config and 'keystorepassfile' not in config:
-            nosigningkey = True
-            logging.critical("'keystorepass' not found in config.py!")
-        if 'keypass' not in config and 'keypassfile' not in config:
-            nosigningkey = True
-            logging.critical("'keypass' not found in config.py!")
-        if not os.path.exists(config['keystore']):
-            nosigningkey = True
-            logging.critical("'" + config['keystore'] + "' does not exist!")
-        if nosigningkey:
-            logging.warning("`fdroid update` requires a signing key, you can create one using:")
-            logging.warning("\tfdroid update --create-key")
-            sys.exit(1)
+    if not common.options.nosign:
+        common.assert_config_keystore(common.config)
 
     repodict = collections.OrderedDict()
     repodict['timestamp'] = datetime.utcnow()
     repodict['version'] = METADATA_VERSION
 
-    if config['repo_maxage'] != 0:
-        repodict['maxage'] = config['repo_maxage']
+    if common.config['repo_maxage'] != 0:
+        repodict['maxage'] = common.config['repo_maxage']
 
     if archive:
-        repodict['name'] = config['archive_name']
-        repodict['icon'] = os.path.basename(config['archive_icon'])
-        repodict['address'] = config['archive_url']
-        repodict['description'] = config['archive_description']
-        urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
+        repodict['name'] = common.config['archive_name']
+        repodict['icon'] = os.path.basename(common.config['archive_icon'])
+        repodict['address'] = common.config['archive_url']
+        repodict['description'] = common.config['archive_description']
+        urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
     else:
-        repodict['name'] = config['repo_name']
-        repodict['icon'] = os.path.basename(config['repo_icon'])
-        repodict['address'] = config['repo_url']
-        repodict['description'] = config['repo_description']
-        urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
+        repodict['name'] = common.config['repo_name']
+        repodict['icon'] = os.path.basename(common.config['repo_icon'])
+        repodict['address'] = common.config['repo_url']
+        repodict['description'] = common.config['repo_description']
+        urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
 
     mirrorcheckfailed = False
     mirrors = []
-    for mirror in sorted(config.get('mirrors', [])):
+    for mirror in sorted(common.config.get('mirrors', [])):
         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
-        if config.get('nonstandardwebroot') is not True and base != 'fdroid':
-            logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
+        if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
+            logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
             mirrorcheckfailed = True
         # must end with / or urljoin strips a whole path segment
         if mirror.endswith('/'):
             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
         else:
             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
-    for mirror in config.get('servergitmirrors', []):
-        mirror = get_raw_mirror(mirror)
-        if mirror is not None:
-            mirrors.append(mirror + '/')
+    for mirror in common.config.get('servergitmirrors', []):
+        for url in get_mirror_service_urls(mirror):
+            mirrors.append(url + '/' + repodir)
     if mirrorcheckfailed:
-        sys.exit(1)
+        raise FDroidException(_("Malformed repository mirrors."))
     if mirrors:
         repodict['mirrors'] = mirrors
 
@@ -138,39 +121,53 @@ def make(apps, sortedids, apks, repodir, archive):
                 appsWithPackages[packageName] = newapp
                 break
 
-    requestsdict = dict()
+    requestsdict = collections.OrderedDict()
     for command in ('install', 'uninstall'):
         packageNames = []
         key = command + '_list'
-        if key in config:
-            if isinstance(config[key], str):
-                packageNames = [config[key]]
-            elif all(isinstance(item, str) for item in config[key]):
-                packageNames = config[key]
+        if key in common.config:
+            if isinstance(common.config[key], str):
+                packageNames = [common.config[key]]
+            elif all(isinstance(item, str) for item in common.config[key]):
+                packageNames = common.config[key]
             else:
-                raise TypeError('only accepts strings, lists, and tuples')
+                raise TypeError(_('only accepts strings, lists, and tuples'))
         requestsdict[command] = packageNames
 
-    make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
-    make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
+    fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
+
+    make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
+            fdroid_signing_key_fingerprints)
+    make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
+            fdroid_signing_key_fingerprints)
 
 
-def make_v1(apps, packages, repodir, repodict, requestsdict):
+def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 
     def _index_encoder_default(obj):
         if isinstance(obj, set):
-            return list(obj)
+            return sorted(list(obj))
         if isinstance(obj, datetime):
-            return int(obj.timestamp() * 1000)  # Java expects milliseconds
+            # Java prefers milliseconds
+            # we also need to accound for time zone/daylight saving time
+            return int(calendar.timegm(obj.timetuple()) * 1000)
+        if isinstance(obj, dict):
+            d = collections.OrderedDict()
+            for key in sorted(obj.keys()):
+                d[key] = obj[key]
+            return d
         raise TypeError(repr(obj) + " is not JSON serializable")
 
     output = collections.OrderedDict()
     output['repo'] = repodict
     output['requests'] = requestsdict
 
+    # establish sort order of the index
+    v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
+
     appslist = []
     output['apps'] = appslist
-    for appid, appdict in apps.items():
+    for packageName, appdict in apps.items():
         d = collections.OrderedDict()
         appslist.append(d)
         for k, v in sorted(appdict.items()):
@@ -191,17 +188,41 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
                 k = 'suggestedVersionName'
             elif k == 'AutoName':
-                if 'Name' not in apps[appid]:
+                if 'Name' not in apps[packageName]:
                     d['name'] = v
                 continue
             else:
                 k = k[:1].lower() + k[1:]
             d[k] = v
 
-    output_packages = dict()
+    # establish sort order in localized dicts
+    for app in output['apps']:
+        localized = app.get('localized')
+        if localized:
+            lordered = collections.OrderedDict()
+            for lkey, lvalue in sorted(localized.items()):
+                lordered[lkey] = collections.OrderedDict()
+                for ikey, iname in sorted(lvalue.items()):
+                    lordered[lkey][ikey] = iname
+            app['localized'] = lordered
+
+    output_packages = collections.OrderedDict()
     output['packages'] = output_packages
     for package in packages:
         packageName = package['packageName']
+        if packageName not in apps:
+            logging.info(_('Ignoring package without metadata: ') + package['apkName'])
+            continue
+        if not package.get('versionName'):
+            app = apps[packageName]
+            versionCodeStr = str(package['versionCode'])  # TODO build.versionCode should be int!
+            for build in app['builds']:
+                if build['versionCode'] == versionCodeStr:
+                    versionName = build.get('versionName')
+                    logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}')
+                                 .format(apkfilename=package['apkName'], version=versionName))
+                    package['versionName'] = versionName
+                    break
         if packageName in output_packages:
             packagelist = output_packages[packageName]
         else:
@@ -219,16 +240,55 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
     json_name = 'index-v1.json'
     index_file = os.path.join(repodir, json_name)
     with open(index_file, 'w') as fp:
-        json.dump(output, fp, default=_index_encoder_default)
+        if common.options.pretty:
+            json.dump(output, fp, default=_index_encoder_default, indent=2)
+        else:
+            json.dump(output, fp, default=_index_encoder_default)
 
-    if options.nosign:
-        logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
+    if common.options.nosign:
+        logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
     else:
-        signindex.config = config
+        signindex.config = common.config
         signindex.sign_index_v1(repodir, json_name)
 
 
-def make_v0(apps, apks, repodir, repodict, requestsdict):
+def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
+    """Sorts the supplied list to ensure a deterministic sort order for
+    package entries in the index file. This sort-order also expresses
+    installation preference to the clients.
+    (First in this list = first to install)
+
+    :param packages: list of packages which need to be sorted before but into index file.
+    """
+
+    GROUP_DEV_SIGNED = 1
+    GROUP_FDROID_SIGNED = 2
+    GROUP_OTHER_SIGNED = 3
+
+    def v1_sort_keys(package):
+        packageName = package.get('packageName', None)
+
+        sig = package.get('signer', None)
+
+        dev_sig = common.metadata_find_developer_signature(packageName)
+        group = GROUP_OTHER_SIGNED
+        if dev_sig and dev_sig == sig:
+            group = GROUP_DEV_SIGNED
+        else:
+            fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
+            if fdroidsig and fdroidsig == sig:
+                group = GROUP_FDROID_SIGNED
+
+        versionCode = None
+        if package.get('versionCode', None):
+            versionCode = -int(package['versionCode'])
+
+        return(packageName, group, sig, versionCode)
+
+    packages.sort(key=v1_sort_keys)
+
+
+def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
     """
     aka index.jar aka index.xml
     """
@@ -256,6 +316,35 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
         el.appendChild(doc.createCDATASection(value))
         parent.appendChild(el)
 
+    def addElementCheckLocalized(name, app, key, doc, parent, default=''):
+        '''Fill in field from metadata or localized block
+
+        For name/summary/description, they can come only from the app source,
+        or from a dir in fdroiddata.  They can be entirely missing from the
+        metadata file if there is localized versions.  This will fetch those
+        from the localized version if its not available in the metadata file.
+        '''
+
+        el = doc.createElement(name)
+        value = app.get(key)
+        lkey = key[:1].lower() + key[1:]
+        localized = app.get('localized')
+        if not value and localized:
+            for lang in ['en-US'] + [x for x in localized.keys()]:
+                if not lang.startswith('en'):
+                    continue
+                if lang in localized:
+                    value = localized[lang].get(lkey)
+                    if value:
+                        break
+        if not value and localized and len(localized) > 1:
+            lang = list(localized.keys())[0]
+            value = localized[lang].get(lkey)
+        if not value:
+            value = default
+        el.appendChild(doc.createTextNode(value))
+        parent.appendChild(el)
+
     root = doc.createElement("fdroid")
     doc.appendChild(root)
 
@@ -291,9 +380,27 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
 
         # Get a list of the apks for this app...
         apklist = []
+        apksbyversion = collections.defaultdict(lambda: [])
         for apk in apks:
-            if apk['packageName'] == appid:
-                apklist.append(apk)
+            if apk.get('versionCode') and apk.get('packageName') == appid:
+                apksbyversion[apk['versionCode']].append(apk)
+        for versionCode, apksforver in apksbyversion.items():
+            fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
+            fdroid_signed_apk = None
+            name_match_apk = None
+            for x in apksforver:
+                if fdroidsig and x.get('signer', None) == fdroidsig:
+                    fdroid_signed_apk = x
+                if common.apk_release_filename.match(x.get('apkName', '')):
+                    name_match_apk = x
+            # choose which of the available versions is most
+            # suiteable for index v0
+            if fdroid_signed_apk:
+                apklist.append(fdroid_signed_apk)
+            elif name_match_apk:
+                apklist.append(name_match_apk)
+            else:
+                apklist.append(apksforver[0])
 
         if len(apklist) == 0:
             continue
@@ -307,16 +414,16 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
         if app.lastUpdated:
             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
-        addElement('name', app.Name, doc, apel)
-        addElement('summary', app.Summary, doc, apel)
+
+        addElementCheckLocalized('name', app, 'Name', doc, apel)
+        addElementCheckLocalized('summary', app, 'Summary', doc, apel)
+
         if app.icon:
             addElement('icon', app.icon, doc, apel)
 
-        if app.get('Description'):
-            description = app.Description
-        else:
-            description = '<p>No description available</p>'
-        addElement('desc', description, doc, apel)
+        addElementCheckLocalized('desc', app, 'Description', doc, apel,
+                                 '<p>No description available</p>')
+
         addElement('license', app.License, doc, apel)
         if app.Categories:
             addElement('categories', ','.join(app.Categories), doc, apel)
@@ -334,6 +441,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
+        addElementNonEmpty('liberapay', app.LiberapayID, doc, apel)
 
         # These elements actually refer to the current version (i.e. which
         # one is recommended. They are historically mis-named, and need
@@ -358,10 +466,16 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
 
         # Check for duplicates - they will make the client unhappy...
         for i in range(len(apklist) - 1):
-            if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
-                logging.critical("duplicate versions: '%s' - '%s'" % (
-                    apklist[i]['apkName'], apklist[i + 1]['apkName']))
-                sys.exit(1)
+            first = apklist[i]
+            second = apklist[i + 1]
+            if first['versionCode'] == second['versionCode'] \
+               and first['sig'] == second['sig']:
+                if first['hash'] == second['hash']:
+                    raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
+                        repodir, first['apkName'], second['apkName']))
+                else:
+                    raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
+                        repodir, first['apkName'], second['apkName']))
 
         current_version_code = 0
         current_version_file = None
@@ -375,7 +489,17 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
 
             apkel = doc.createElement("package")
             apel.appendChild(apkel)
-            addElement('version', apk['versionName'], doc, apkel)
+
+            versionName = apk.get('versionName')
+            if not versionName:
+                versionCodeStr = str(apk['versionCode'])  # TODO build.versionCode should be int!
+                for build in app.builds:
+                    if build['versionCode'] == versionCodeStr and 'versionName' in build:
+                        versionName = build['versionName']
+                        break
+            if versionName:
+                addElement('version', versionName, doc, apkel)
+
             addElement('versioncode', str(apk['versionCode']), doc, apkel)
             addElement('apkname', apk['apkName'], doc, apkel)
             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
@@ -413,7 +537,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
                     if perm_name.startswith("android.permission."):
                         perm_name = perm_name[19:]
                     old_permissions.add(perm_name)
-                addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
+                addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 
                 for permission in sorted_permissions:
                     permel = doc.createElement('uses-permission')
@@ -432,17 +556,17 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 
         if current_version_file is not None \
-                and config['make_current_version_link'] \
+                and common.config['make_current_version_link'] \
                 and repodir == 'repo':  # only create these
-            namefield = config['current_version_name_source']
-            sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
-            apklinkname = sanitized_name + '.apk'
-            current_version_path = os.path.join(repodir, current_version_file)
+            namefield = common.config['current_version_name_source']
+            sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
+            apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
+            current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
             if os.path.islink(apklinkname):
                 os.remove(apklinkname)
             os.symlink(current_version_path, apklinkname)
             # also symlink gpg signature, if it exists
-            for extension in ('.asc', '.sig'):
+            for extension in (b'.asc', b'.sig'):
                 sigfile_path = current_version_path + extension
                 if os.path.exists(sigfile_path):
                     siglinkname = apklinkname + extension
@@ -450,7 +574,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
                         os.remove(siglinkname)
                     os.symlink(sigfile_path, siglinkname)
 
-    if options.pretty:
+    if common.options.pretty:
         output = doc.toprettyxml(encoding='utf-8')
     else:
         output = doc.toxml(encoding='utf-8')
@@ -458,35 +582,34 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
         f.write(output)
 
-    if 'repo_keyalias' in config:
+    if 'repo_keyalias' in common.config:
 
-        if options.nosign:
-            logging.info("Creating unsigned index in preparation for signing")
+        if common.options.nosign:
+            logging.info(_("Creating unsigned index in preparation for signing"))
         else:
-            logging.info("Creating signed index with this key (SHA256):")
+            logging.info(_("Creating signed index with this key (SHA256):"))
             logging.info("%s" % repo_pubkey_fingerprint)
 
         # Create a jar of the index...
-        jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
+        jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
         if p.returncode != 0:
-            logging.critical("Failed to create {0}".format(jar_output))
-            sys.exit(1)
+            raise FDroidException("Failed to create {0}".format(jar_output))
 
         # Sign the index...
         signed = os.path.join(repodir, 'index.jar')
-        if options.nosign:
+        if common.options.nosign:
             # Remove old signed index if not signing
             if os.path.exists(signed):
                 os.remove(signed)
         else:
-            signindex.config = config
+            signindex.config = common.config
             signindex.sign_jar(signed)
 
     # Copy the repo icon into the repo directory...
     icon_dir = os.path.join(repodir, 'icons')
-    iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
-    shutil.copyfile(config['repo_icon'], iconfilename)
+    iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
+    shutil.copyfile(common.config['repo_icon'], iconfilename)
 
 
 def extract_pubkey():
@@ -494,47 +617,145 @@ def extract_pubkey():
     Extracts and returns the repository's public key from the keystore.
     :return: public key in hex, repository fingerprint
     """
-    if 'repo_pubkey' in config:
-        pubkey = unhexlify(config['repo_pubkey'])
+    if 'repo_pubkey' in common.config:
+        pubkey = unhexlify(common.config['repo_pubkey'])
     else:
-        p = FDroidPopenBytes([config['keytool'], '-exportcert',
-                              '-alias', config['repo_keyalias'],
-                              '-keystore', config['keystore'],
-                              '-storepass:file', config['keystorepassfile']]
-                             + config['smartcardoptions'],
-                             output=False, stderr_to_stdout=False)
+        env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
+        p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
+                              '-alias', common.config['repo_keyalias'],
+                              '-keystore', common.config['keystore'],
+                              '-storepass:env', 'FDROID_KEY_STORE_PASS']
+                             + common.config['smartcardoptions'],
+                             envs=env_vars, output=False, stderr_to_stdout=False)
         if p.returncode != 0 or len(p.output) < 20:
             msg = "Failed to get repo pubkey!"
-            if config['keystore'] == 'NONE':
+            if common.config['keystore'] == 'NONE':
                 msg += ' Is your crypto smartcard plugged in?'
-            logging.critical(msg)
-            sys.exit(1)
+            raise FDroidException(msg)
         pubkey = p.output
     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
     return hexlify(pubkey), repo_pubkey_fingerprint
 
 
-# Get raw URL from git service for mirroring
-def get_raw_mirror(url):
-    # Divide urls in parts
-    url = url.split("/")
+def get_mirror_service_urls(url):
+    '''Get direct URLs from git service for use by fdroidclient
+
+    Via 'servergitmirrors', fdroidserver can create and push a mirror
+    to certain well known git services like gitlab or github.  This
+    will always use the 'master' branch since that is the default
+    branch in git. The files are then accessible via alternate URLs,
+    where they are served in their raw format via a CDN rather than
+    from git.
+    '''
+
+    if url.startswith('git@'):
+        url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 
-    # Get the hostname
-    hostname = url[2]
+    segments = url.split("/")
 
-    # fdroidserver will use always 'master' branch for git-mirroring
+    if segments[4].endswith('.git'):
+        segments[4] = segments[4][:-4]
+
+    hostname = segments[2]
+    user = segments[3]
+    repo = segments[4]
     branch = "master"
     folder = "fdroid"
 
+    urls = []
     if hostname == "github.com":
-        # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
-        url[2] = "raw.githubusercontent.com"
-        url.extend([branch, folder])
+        # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
+        segments[2] = "raw.githubusercontent.com"
+        segments.extend([branch, folder])
+        urls.append('/'.join(segments))
     elif hostname == "gitlab.com":
-        # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
-        url.extend(["raw", branch, folder])
-    else:
-        return None
+        # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser
+        # This is because the `raw` URLs are not served with the correct mime types, so any
+        # index.html which is put in the repo will not be rendered. Putting an index.html file in
+        # the repo root is a common way for to make information about the repo available to end user.
+
+        # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
+        gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
+        urls.append('/'.join(gitlab_pages))
+        # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
+        gitlab_raw = segments + ['raw', branch, folder]
+        urls.append('/'.join(gitlab_raw))
+        return urls
+
+    return urls
+
+
+def download_repo_index(url_str, etag=None, verify_fingerprint=True):
+    """
+    Downloads the repository index from the given :param url_str
+    and verifies the repository's fingerprint if :param verify_fingerprint is not False.
+
+    :raises: VerificationException() if the repository could not be verified
+
+    :return: A tuple consisting of:
+        - The index in JSON format or None if the index did not change
+        - The new eTag as returned by the HTTP request
+    """
+    url = urllib.parse.urlsplit(url_str)
+
+    fingerprint = None
+    if verify_fingerprint:
+        query = urllib.parse.parse_qs(url.query)
+        if 'fingerprint' not in query:
+            raise VerificationException(_("No fingerprint in URL."))
+        fingerprint = query['fingerprint'][0]
 
-    url = "/".join(url)
-    return url
+    url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
+    download, new_etag = net.http_get(url.geturl(), etag)
+
+    if download is None:
+        return None, new_etag
+
+    with tempfile.NamedTemporaryFile() as fp:
+        # write and open JAR file
+        fp.write(download)
+        jar = zipfile.ZipFile(fp)
+
+        # verify that the JAR signature is valid
+        logging.debug(_('Verifying index signature:'))
+        common.verify_jar_signature(fp.name)
+
+        # get public key and its fingerprint from JAR
+        public_key, public_key_fingerprint = get_public_key_from_jar(jar)
+
+        # compare the fingerprint if verify_fingerprint is True
+        if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
+            raise VerificationException(_("The repository's fingerprint does not match."))
+
+        # load repository index from JSON
+        index = json.loads(jar.read('index-v1.json').decode("utf-8"))
+        index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
+        index["repo"]["fingerprint"] = public_key_fingerprint
+
+        # turn the apps into App objects
+        index["apps"] = [metadata.App(app) for app in index["apps"]]
+
+        return index, new_etag
+
+
+def get_public_key_from_jar(jar):
+    """
+    Get the public key and its fingerprint from a JAR file.
+
+    :raises: VerificationException() if the JAR was not signed exactly once
+
+    :param jar: a zipfile.ZipFile object
+    :return: the public key from the jar and its fingerprint
+    """
+    # extract certificate from jar
+    certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
+    if len(certs) < 1:
+        raise VerificationException(_("Found no signing certificates for repository."))
+    if len(certs) > 1:
+        raise VerificationException(_("Found multiple signing certificates for repository."))
+
+    # extract public key from certificate
+    public_key = common.get_certificate(jar.read(certs[0]))
+    public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
+
+    return public_key, public_key_fingerprint