X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=fdroidserver%2Findex.py;h=56caaad12dbbccae8652d28774fb93e0ccd777ff;hb=74c6555c719e685d225baa00670ee12779bdeff1;hp=6843f5208a7d5a6e9af66ca41c8455256f408f11;hpb=25f96e191138de08e6632b5bf37f2346b458ec11;p=fdroidserver.git diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 6843f520..56caaad1 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -30,12 +30,17 @@ import shutil import tempfile import urllib.parse import zipfile +import calendar from binascii import hexlify, unhexlify -from datetime import datetime +from datetime import datetime, timezone from xml.dom.minidom import Document -from fdroidserver import metadata, signindex, common, net -from fdroidserver.common import FDroidPopen, FDroidPopenBytes +from . import _ +from . import common +from . import metadata +from . import net +from . import signindex +from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints from fdroidserver.exception import FDroidException, VerificationException, MetaDataException @@ -58,29 +63,11 @@ def make(apps, sortedids, apks, repodir, archive): return "fdroid.app:" + appid, apps[appid].Name raise MetaDataException("Cannot resolve app id " + appid) - nosigningkey = False if not common.options.nosign: - if 'repo_keyalias' not in common.config: - nosigningkey = True - logging.critical("'repo_keyalias' not found in config.py!") - if 'keystore' not in common.config: - nosigningkey = True - logging.critical("'keystore' not found in config.py!") - if 'keystorepass' not in common.config: - nosigningkey = True - logging.critical("'keystorepass' not found in config.py!") - if 'keypass' not in common.config: - nosigningkey = True - logging.critical("'keypass' not found in config.py!") - if not os.path.exists(common.config['keystore']): - nosigningkey = True - logging.critical("'" + common.config['keystore'] + "' does not exist!") - if nosigningkey: - raise FDroidException("`fdroid update` requires a signing key, " + - "you can create one using: fdroid update --create-key") + common.assert_config_keystore(common.config) repodict = collections.OrderedDict() - repodict['timestamp'] = datetime.utcnow() + repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc) repodict['version'] = METADATA_VERSION if common.config['repo_maxage'] != 0: @@ -104,7 +91,7 @@ def make(apps, sortedids, apks, repodir, archive): for mirror in sorted(common.config.get('mirrors', [])): base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/')) if common.config.get('nonstandardwebroot') is not True and base != 'fdroid': - logging.error("mirror '" + mirror + "' does not end with 'fdroid'!") + logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror) mirrorcheckfailed = True # must end with / or urljoin strips a whole path segment if mirror.endswith('/'): @@ -112,11 +99,10 @@ def make(apps, sortedids, apks, repodir, archive): else: mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath)) for mirror in common.config.get('servergitmirrors', []): - mirror = get_mirror_service_url(mirror) - if mirror: - mirrors.append(mirror + '/' + repodir) + for url in get_mirror_service_urls(mirror): + mirrors.append(url + '/' + repodir) if mirrorcheckfailed: - raise FDroidException("Malformed repository mirrors.") + raise FDroidException(_("Malformed repository mirrors.")) if mirrors: repodict['mirrors'] = mirrors @@ -145,26 +131,40 @@ def make(apps, sortedids, apks, repodir, archive): elif all(isinstance(item, str) for item in common.config[key]): packageNames = common.config[key] else: - raise TypeError('only accepts strings, lists, and tuples') + raise TypeError(_('only accepts strings, lists, and tuples')) requestsdict[command] = packageNames - make_v0(appsWithPackages, apks, repodir, repodict, requestsdict) - make_v1(appsWithPackages, apks, repodir, repodict, requestsdict) + fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints() + make_v0(appsWithPackages, apks, repodir, repodict, requestsdict, + fdroid_signing_key_fingerprints) + make_v1(appsWithPackages, apks, repodir, repodict, requestsdict, + fdroid_signing_key_fingerprints) -def make_v1(apps, packages, repodir, repodict, requestsdict): + +def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints): def _index_encoder_default(obj): if isinstance(obj, set): - return list(obj) + return sorted(list(obj)) if isinstance(obj, datetime): - return int(obj.timestamp() * 1000) # Java expects milliseconds + # Java prefers milliseconds + # we also need to accound for time zone/daylight saving time + return int(calendar.timegm(obj.timetuple()) * 1000) + if isinstance(obj, dict): + d = collections.OrderedDict() + for key in sorted(obj.keys()): + d[key] = obj[key] + return d raise TypeError(repr(obj) + " is not JSON serializable") output = collections.OrderedDict() output['repo'] = repodict output['requests'] = requestsdict + # establish sort order of the index + v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints) + appslist = [] output['apps'] = appslist for packageName, appdict in apps.items(): @@ -195,13 +195,34 @@ def make_v1(apps, packages, repodir, repodict, requestsdict): k = k[:1].lower() + k[1:] d[k] = v + # establish sort order in localized dicts + for app in output['apps']: + localized = app.get('localized') + if localized: + lordered = collections.OrderedDict() + for lkey, lvalue in sorted(localized.items()): + lordered[lkey] = collections.OrderedDict() + for ikey, iname in sorted(lvalue.items()): + lordered[lkey][ikey] = iname + app['localized'] = lordered + output_packages = collections.OrderedDict() output['packages'] = output_packages for package in packages: packageName = package['packageName'] if packageName not in apps: - logging.info('Ignoring package without metadata: ' + package['apkName']) + logging.info(_('Ignoring package without metadata: ') + package['apkName']) continue + if not package.get('versionName'): + app = apps[packageName] + versionCodeStr = str(package['versionCode']) # TODO build.versionCode should be int! + for build in app['builds']: + if build['versionCode'] == versionCodeStr: + versionName = build.get('versionName') + logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}') + .format(apkfilename=package['apkName'], version=versionName)) + package['versionName'] = versionName + break if packageName in output_packages: packagelist = output_packages[packageName] else: @@ -225,13 +246,49 @@ def make_v1(apps, packages, repodir, repodict, requestsdict): json.dump(output, fp, default=_index_encoder_default) if common.options.nosign: - logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!') + logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!')) else: signindex.config = common.config signindex.sign_index_v1(repodir, json_name) -def make_v0(apps, apks, repodir, repodict, requestsdict): +def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints): + """Sorts the supplied list to ensure a deterministic sort order for + package entries in the index file. This sort-order also expresses + installation preference to the clients. + (First in this list = first to install) + + :param packages: list of packages which need to be sorted before but into index file. + """ + + GROUP_DEV_SIGNED = 1 + GROUP_FDROID_SIGNED = 2 + GROUP_OTHER_SIGNED = 3 + + def v1_sort_keys(package): + packageName = package.get('packageName', None) + + sig = package.get('signer', None) + + dev_sig = common.metadata_find_developer_signature(packageName) + group = GROUP_OTHER_SIGNED + if dev_sig and dev_sig == sig: + group = GROUP_DEV_SIGNED + else: + fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer') + if fdroidsig and fdroidsig == sig: + group = GROUP_FDROID_SIGNED + + versionCode = None + if package.get('versionCode', None): + versionCode = -int(package['versionCode']) + + return(packageName, group, sig, versionCode) + + packages.sort(key=v1_sort_keys) + + +def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints): """ aka index.jar aka index.xml """ @@ -259,6 +316,35 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): el.appendChild(doc.createCDATASection(value)) parent.appendChild(el) + def addElementCheckLocalized(name, app, key, doc, parent, default=''): + '''Fill in field from metadata or localized block + + For name/summary/description, they can come only from the app source, + or from a dir in fdroiddata. They can be entirely missing from the + metadata file if there is localized versions. This will fetch those + from the localized version if its not available in the metadata file. + ''' + + el = doc.createElement(name) + value = app.get(key) + lkey = key[:1].lower() + key[1:] + localized = app.get('localized') + if not value and localized: + for lang in ['en-US'] + [x for x in localized.keys()]: + if not lang.startswith('en'): + continue + if lang in localized: + value = localized[lang].get(lkey) + if value: + break + if not value and localized and len(localized) > 1: + lang = list(localized.keys())[0] + value = localized[lang].get(lkey) + if not value: + value = default + el.appendChild(doc.createTextNode(value)) + parent.appendChild(el) + root = doc.createElement("fdroid") doc.appendChild(root) @@ -294,12 +380,27 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): # Get a list of the apks for this app... apklist = [] - versionCodes = [] + apksbyversion = collections.defaultdict(lambda: []) for apk in apks: - if apk['packageName'] == appid: - if apk['versionCode'] not in versionCodes: - apklist.append(apk) - versionCodes.append(apk['versionCode']) + if apk.get('versionCode') and apk.get('packageName') == appid: + apksbyversion[apk['versionCode']].append(apk) + for versionCode, apksforver in apksbyversion.items(): + fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer') + fdroid_signed_apk = None + name_match_apk = None + for x in apksforver: + if fdroidsig and x.get('signer', None) == fdroidsig: + fdroid_signed_apk = x + if common.apk_release_filename.match(x.get('apkName', '')): + name_match_apk = x + # choose which of the available versions is most + # suiteable for index v0 + if fdroid_signed_apk: + apklist.append(fdroid_signed_apk) + elif name_match_apk: + apklist.append(name_match_apk) + else: + apklist.append(apksforver[0]) if len(apklist) == 0: continue @@ -313,16 +414,16 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel) if app.lastUpdated: addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel) - addElement('name', app.Name, doc, apel) - addElement('summary', app.Summary, doc, apel) + + addElementCheckLocalized('name', app, 'Name', doc, apel) + addElementCheckLocalized('summary', app, 'Summary', doc, apel) + if app.icon: addElement('icon', app.icon, doc, apel) - if app.get('Description'): - description = app.Description - else: - description = '

No description available

' - addElement('desc', description, doc, apel) + addElementCheckLocalized('desc', app, 'Description', doc, apel, + '

No description available

') + addElement('license', app.License, doc, apel) if app.Categories: addElement('categories', ','.join(app.Categories), doc, apel) @@ -340,6 +441,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel) addElementNonEmpty('litecoin', app.Litecoin, doc, apel) addElementNonEmpty('flattr', app.FlattrID, doc, apel) + addElementNonEmpty('liberapay', app.LiberapayID, doc, apel) # These elements actually refer to the current version (i.e. which # one is recommended. They are historically mis-named, and need @@ -387,7 +489,17 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): apkel = doc.createElement("package") apel.appendChild(apkel) - addElement('version', apk['versionName'], doc, apkel) + + versionName = apk.get('versionName') + if not versionName: + versionCodeStr = str(apk['versionCode']) # TODO build.versionCode should be int! + for build in app.builds: + if build['versionCode'] == versionCodeStr and 'versionName' in build: + versionName = build['versionName'] + break + if versionName: + addElement('version', versionName, doc, apkel) + addElement('versioncode', str(apk['versionCode']), doc, apkel) addElement('apkname', apk['apkName'], doc, apkel) addElementIfInApk('srcname', apk, 'srcname', doc, apkel) @@ -473,9 +585,9 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): if 'repo_keyalias' in common.config: if common.options.nosign: - logging.info("Creating unsigned index in preparation for signing") + logging.info(_("Creating unsigned index in preparation for signing")) else: - logging.info("Creating signed index with this key (SHA256):") + logging.info(_("Creating signed index with this key (SHA256):")) logging.info("%s" % repo_pubkey_fingerprint) # Create a jar of the index... @@ -525,14 +637,15 @@ def extract_pubkey(): return hexlify(pubkey), repo_pubkey_fingerprint -def get_mirror_service_url(url): - '''Get direct URL from git service for use by fdroidclient +def get_mirror_service_urls(url): + '''Get direct URLs from git service for use by fdroidclient Via 'servergitmirrors', fdroidserver can create and push a mirror to certain well known git services like gitlab or github. This will always use the 'master' branch since that is the default - branch in git. - + branch in git. The files are then accessible via alternate URLs, + where they are served in their raw format via a CDN rather than + from git. ''' if url.startswith('git@'): @@ -549,18 +662,27 @@ def get_mirror_service_url(url): branch = "master" folder = "fdroid" + urls = [] if hostname == "github.com": - # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/master/fdroid" + # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder" segments[2] = "raw.githubusercontent.com" segments.extend([branch, folder]) + urls.append('/'.join(segments)) elif hostname == "gitlab.com": - # Gitlab-like Pages segments "https://user.gitlab.com/repo/fdroid" - gitlab_url = ["https:", "", user + ".gitlab.io", repo, folder] - segments = gitlab_url - else: - return None + # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser + # This is because the `raw` URLs are not served with the correct mime types, so any + # index.html which is put in the repo will not be rendered. Putting an index.html file in + # the repo root is a common way for to make information about the repo available to end user. - return '/'.join(segments) + # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder" + gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder] + urls.append('/'.join(gitlab_pages)) + # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder" + gitlab_raw = segments + ['raw', branch, folder] + urls.append('/'.join(gitlab_raw)) + return urls + + return urls def download_repo_index(url_str, etag=None, verify_fingerprint=True): @@ -580,7 +702,7 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True): if verify_fingerprint: query = urllib.parse.parse_qs(url.query) if 'fingerprint' not in query: - raise VerificationException("No fingerprint in URL.") + raise VerificationException(_("No fingerprint in URL.")) fingerprint = query['fingerprint'][0] url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '') @@ -595,14 +717,15 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True): jar = zipfile.ZipFile(fp) # verify that the JAR signature is valid - verify_jar_signature(fp.name) + logging.debug(_('Verifying index signature:')) + common.verify_jar_signature(fp.name) # get public key and its fingerprint from JAR public_key, public_key_fingerprint = get_public_key_from_jar(jar) # compare the fingerprint if verify_fingerprint is True if verify_fingerprint and fingerprint.upper() != public_key_fingerprint: - raise VerificationException("The repository's fingerprint does not match.") + raise VerificationException(_("The repository's fingerprint does not match.")) # load repository index from JSON index = json.loads(jar.read('index-v1.json').decode("utf-8")) @@ -615,16 +738,6 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True): return index, new_etag -def verify_jar_signature(file): - """ - Verifies the signature of a given JAR file. - - :raises: VerificationException() if the JAR's signature could not be verified - """ - if not common.verify_apk_signature(file, jar=True): - raise VerificationException("The repository's index could not be verified.") - - def get_public_key_from_jar(jar): """ Get the public key and its fingerprint from a JAR file. @@ -637,9 +750,9 @@ def get_public_key_from_jar(jar): # extract certificate from jar certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)] if len(certs) < 1: - raise VerificationException("Found no signing certificates for repository.") + raise VerificationException(_("Found no signing certificates for repository.")) if len(certs) > 1: - raise VerificationException("Found multiple signing certificates for repository.") + raise VerificationException(_("Found multiple signing certificates for repository.")) # extract public key from certificate public_key = common.get_certificate(jar.read(certs[0]))