import tempfile
import urllib.parse
import zipfile
+import calendar
from binascii import hexlify, unhexlify
-from datetime import datetime
+from datetime import datetime, timezone
from xml.dom.minidom import Document
-from fdroidserver import metadata, signindex, common, net
-from fdroidserver.common import FDroidPopen, FDroidPopenBytes
+from . import _
+from . import common
+from . import metadata
+from . import net
+from . import signindex
+from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
return "fdroid.app:" + appid, apps[appid].Name
raise MetaDataException("Cannot resolve app id " + appid)
- nosigningkey = False
if not common.options.nosign:
- if 'repo_keyalias' not in common.config:
- nosigningkey = True
- logging.critical("'repo_keyalias' not found in config.py!")
- if 'keystore' not in common.config:
- nosigningkey = True
- logging.critical("'keystore' not found in config.py!")
- if 'keystorepass' not in common.config:
- nosigningkey = True
- logging.critical("'keystorepass' not found in config.py!")
- if 'keypass' not in common.config:
- nosigningkey = True
- logging.critical("'keypass' not found in config.py!")
- if not os.path.exists(common.config['keystore']):
- nosigningkey = True
- logging.critical("'" + common.config['keystore'] + "' does not exist!")
- if nosigningkey:
- raise FDroidException("`fdroid update` requires a signing key, " +
- "you can create one using: fdroid update --create-key")
+ common.assert_config_keystore(common.config)
repodict = collections.OrderedDict()
- repodict['timestamp'] = datetime.utcnow()
+ repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
repodict['version'] = METADATA_VERSION
if common.config['repo_maxage'] != 0:
for mirror in sorted(common.config.get('mirrors', [])):
base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
- logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
+ logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
mirrorcheckfailed = True
# must end with / or urljoin strips a whole path segment
if mirror.endswith('/'):
else:
mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
for mirror in common.config.get('servergitmirrors', []):
- mirror = get_mirror_service_url(mirror)
- if mirror:
- mirrors.append(mirror + '/' + repodir)
+ for url in get_mirror_service_urls(mirror):
+ mirrors.append(url + '/' + repodir)
if mirrorcheckfailed:
- raise FDroidException("Malformed repository mirrors.")
+ raise FDroidException(_("Malformed repository mirrors."))
if mirrors:
repodict['mirrors'] = mirrors
elif all(isinstance(item, str) for item in common.config[key]):
packageNames = common.config[key]
else:
- raise TypeError('only accepts strings, lists, and tuples')
+ raise TypeError(_('only accepts strings, lists, and tuples'))
requestsdict[command] = packageNames
- make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
- make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
+ fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
+ make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
+ fdroid_signing_key_fingerprints)
+ make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
+ fdroid_signing_key_fingerprints)
-def make_v1(apps, packages, repodir, repodict, requestsdict):
+
+def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
def _index_encoder_default(obj):
if isinstance(obj, set):
- return list(obj)
+ return sorted(list(obj))
if isinstance(obj, datetime):
- return int(obj.timestamp() * 1000) # Java expects milliseconds
+ # Java prefers milliseconds
+ # we also need to accound for time zone/daylight saving time
+ return int(calendar.timegm(obj.timetuple()) * 1000)
+ if isinstance(obj, dict):
+ d = collections.OrderedDict()
+ for key in sorted(obj.keys()):
+ d[key] = obj[key]
+ return d
raise TypeError(repr(obj) + " is not JSON serializable")
output = collections.OrderedDict()
output['repo'] = repodict
output['requests'] = requestsdict
+ # establish sort order of the index
+ v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
+
appslist = []
output['apps'] = appslist
for packageName, appdict in apps.items():
k = k[:1].lower() + k[1:]
d[k] = v
+ # establish sort order in localized dicts
+ for app in output['apps']:
+ localized = app.get('localized')
+ if localized:
+ lordered = collections.OrderedDict()
+ for lkey, lvalue in sorted(localized.items()):
+ lordered[lkey] = collections.OrderedDict()
+ for ikey, iname in sorted(lvalue.items()):
+ lordered[lkey][ikey] = iname
+ app['localized'] = lordered
+
output_packages = collections.OrderedDict()
output['packages'] = output_packages
for package in packages:
packageName = package['packageName']
if packageName not in apps:
- logging.info('Ignoring package without metadata: ' + package['apkName'])
+ logging.info(_('Ignoring package without metadata: ') + package['apkName'])
continue
+ if not package.get('versionName'):
+ app = apps[packageName]
+ versionCodeStr = str(package['versionCode']) # TODO build.versionCode should be int!
+ for build in app['builds']:
+ if build['versionCode'] == versionCodeStr:
+ versionName = build.get('versionName')
+ logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}')
+ .format(apkfilename=package['apkName'], version=versionName))
+ package['versionName'] = versionName
+ break
if packageName in output_packages:
packagelist = output_packages[packageName]
else:
json.dump(output, fp, default=_index_encoder_default)
if common.options.nosign:
- logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
+ logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
else:
signindex.config = common.config
signindex.sign_index_v1(repodir, json_name)
-def make_v0(apps, apks, repodir, repodict, requestsdict):
+def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
+ """Sorts the supplied list to ensure a deterministic sort order for
+ package entries in the index file. This sort-order also expresses
+ installation preference to the clients.
+ (First in this list = first to install)
+
+ :param packages: list of packages which need to be sorted before but into index file.
+ """
+
+ GROUP_DEV_SIGNED = 1
+ GROUP_FDROID_SIGNED = 2
+ GROUP_OTHER_SIGNED = 3
+
+ def v1_sort_keys(package):
+ packageName = package.get('packageName', None)
+
+ sig = package.get('signer', None)
+
+ dev_sig = common.metadata_find_developer_signature(packageName)
+ group = GROUP_OTHER_SIGNED
+ if dev_sig and dev_sig == sig:
+ group = GROUP_DEV_SIGNED
+ else:
+ fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
+ if fdroidsig and fdroidsig == sig:
+ group = GROUP_FDROID_SIGNED
+
+ versionCode = None
+ if package.get('versionCode', None):
+ versionCode = -int(package['versionCode'])
+
+ return(packageName, group, sig, versionCode)
+
+ packages.sort(key=v1_sort_keys)
+
+
+def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
"""
aka index.jar aka index.xml
"""
el.appendChild(doc.createCDATASection(value))
parent.appendChild(el)
+ def addElementCheckLocalized(name, app, key, doc, parent, default=''):
+ '''Fill in field from metadata or localized block
+
+ For name/summary/description, they can come only from the app source,
+ or from a dir in fdroiddata. They can be entirely missing from the
+ metadata file if there is localized versions. This will fetch those
+ from the localized version if its not available in the metadata file.
+ '''
+
+ el = doc.createElement(name)
+ value = app.get(key)
+ lkey = key[:1].lower() + key[1:]
+ localized = app.get('localized')
+ if not value and localized:
+ for lang in ['en-US'] + [x for x in localized.keys()]:
+ if not lang.startswith('en'):
+ continue
+ if lang in localized:
+ value = localized[lang].get(lkey)
+ if value:
+ break
+ if not value and localized and len(localized) > 1:
+ lang = list(localized.keys())[0]
+ value = localized[lang].get(lkey)
+ if not value:
+ value = default
+ el.appendChild(doc.createTextNode(value))
+ parent.appendChild(el)
+
root = doc.createElement("fdroid")
doc.appendChild(root)
# Get a list of the apks for this app...
apklist = []
- versionCodes = []
+ apksbyversion = collections.defaultdict(lambda: [])
for apk in apks:
- if apk['packageName'] == appid:
- if apk['versionCode'] not in versionCodes:
- apklist.append(apk)
- versionCodes.append(apk['versionCode'])
+ if apk.get('versionCode') and apk.get('packageName') == appid:
+ apksbyversion[apk['versionCode']].append(apk)
+ for versionCode, apksforver in apksbyversion.items():
+ fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
+ fdroid_signed_apk = None
+ name_match_apk = None
+ for x in apksforver:
+ if fdroidsig and x.get('signer', None) == fdroidsig:
+ fdroid_signed_apk = x
+ if common.apk_release_filename.match(x.get('apkName', '')):
+ name_match_apk = x
+ # choose which of the available versions is most
+ # suiteable for index v0
+ if fdroid_signed_apk:
+ apklist.append(fdroid_signed_apk)
+ elif name_match_apk:
+ apklist.append(name_match_apk)
+ else:
+ apklist.append(apksforver[0])
if len(apklist) == 0:
continue
addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
if app.lastUpdated:
addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
- addElement('name', app.Name, doc, apel)
- addElement('summary', app.Summary, doc, apel)
+
+ addElementCheckLocalized('name', app, 'Name', doc, apel)
+ addElementCheckLocalized('summary', app, 'Summary', doc, apel)
+
if app.icon:
addElement('icon', app.icon, doc, apel)
- if app.get('Description'):
- description = app.Description
- else:
- description = '<p>No description available</p>'
- addElement('desc', description, doc, apel)
+ addElementCheckLocalized('desc', app, 'Description', doc, apel,
+ '<p>No description available</p>')
+
addElement('license', app.License, doc, apel)
if app.Categories:
addElement('categories', ','.join(app.Categories), doc, apel)
addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
addElementNonEmpty('flattr', app.FlattrID, doc, apel)
+ addElementNonEmpty('liberapay', app.LiberapayID, doc, apel)
# These elements actually refer to the current version (i.e. which
# one is recommended. They are historically mis-named, and need
apkel = doc.createElement("package")
apel.appendChild(apkel)
- addElement('version', apk['versionName'], doc, apkel)
+
+ versionName = apk.get('versionName')
+ if not versionName:
+ versionCodeStr = str(apk['versionCode']) # TODO build.versionCode should be int!
+ for build in app.builds:
+ if build['versionCode'] == versionCodeStr and 'versionName' in build:
+ versionName = build['versionName']
+ break
+ if versionName:
+ addElement('version', versionName, doc, apkel)
+
addElement('versioncode', str(apk['versionCode']), doc, apkel)
addElement('apkname', apk['apkName'], doc, apkel)
addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
if 'repo_keyalias' in common.config:
if common.options.nosign:
- logging.info("Creating unsigned index in preparation for signing")
+ logging.info(_("Creating unsigned index in preparation for signing"))
else:
- logging.info("Creating signed index with this key (SHA256):")
+ logging.info(_("Creating signed index with this key (SHA256):"))
logging.info("%s" % repo_pubkey_fingerprint)
# Create a jar of the index...
return hexlify(pubkey), repo_pubkey_fingerprint
-def get_mirror_service_url(url):
- '''Get direct URL from git service for use by fdroidclient
+def get_mirror_service_urls(url):
+ '''Get direct URLs from git service for use by fdroidclient
Via 'servergitmirrors', fdroidserver can create and push a mirror
to certain well known git services like gitlab or github. This
will always use the 'master' branch since that is the default
- branch in git.
-
+ branch in git. The files are then accessible via alternate URLs,
+ where they are served in their raw format via a CDN rather than
+ from git.
'''
if url.startswith('git@'):
branch = "master"
folder = "fdroid"
+ urls = []
if hostname == "github.com":
- # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/master/fdroid"
+ # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
segments[2] = "raw.githubusercontent.com"
segments.extend([branch, folder])
+ urls.append('/'.join(segments))
elif hostname == "gitlab.com":
- # Gitlab-like Pages segments "https://user.gitlab.com/repo/fdroid"
- gitlab_url = ["https:", "", user + ".gitlab.io", repo, folder]
- segments = gitlab_url
- else:
- return None
+ # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser
+ # This is because the `raw` URLs are not served with the correct mime types, so any
+ # index.html which is put in the repo will not be rendered. Putting an index.html file in
+ # the repo root is a common way for to make information about the repo available to end user.
- return '/'.join(segments)
+ # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
+ gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
+ urls.append('/'.join(gitlab_pages))
+ # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
+ gitlab_raw = segments + ['raw', branch, folder]
+ urls.append('/'.join(gitlab_raw))
+ return urls
+
+ return urls
def download_repo_index(url_str, etag=None, verify_fingerprint=True):
if verify_fingerprint:
query = urllib.parse.parse_qs(url.query)
if 'fingerprint' not in query:
- raise VerificationException("No fingerprint in URL.")
+ raise VerificationException(_("No fingerprint in URL."))
fingerprint = query['fingerprint'][0]
url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
jar = zipfile.ZipFile(fp)
# verify that the JAR signature is valid
- verify_jar_signature(fp.name)
+ logging.debug(_('Verifying index signature:'))
+ common.verify_jar_signature(fp.name)
# get public key and its fingerprint from JAR
public_key, public_key_fingerprint = get_public_key_from_jar(jar)
# compare the fingerprint if verify_fingerprint is True
if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
- raise VerificationException("The repository's fingerprint does not match.")
+ raise VerificationException(_("The repository's fingerprint does not match."))
# load repository index from JSON
index = json.loads(jar.read('index-v1.json').decode("utf-8"))
return index, new_etag
-def verify_jar_signature(file):
- """
- Verifies the signature of a given JAR file.
-
- :raises: VerificationException() if the JAR's signature could not be verified
- """
- if not common.verify_apk_signature(file, jar=True):
- raise VerificationException("The repository's index could not be verified.")
-
-
def get_public_key_from_jar(jar):
"""
Get the public key and its fingerprint from a JAR file.
# extract certificate from jar
certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
if len(certs) < 1:
- raise VerificationException("Found no signing certificates for repository.")
+ raise VerificationException(_("Found no signing certificates for repository."))
if len(certs) > 1:
- raise VerificationException("Found multiple signing certificates for repository.")
+ raise VerificationException(_("Found multiple signing certificates for repository."))
# extract public key from certificate
public_key = common.get_certificate(jar.read(certs[0]))
~ianmdlvl / fdroidserver.git / blobdiff