import os
import re
import shutil
-import sys
import tempfile
import urllib.parse
import zipfile
+import calendar
from binascii import hexlify, unhexlify
from datetime import datetime
from xml.dom.minidom import Document
-import requests
-
-from fdroidserver import metadata, signindex, common
-from fdroidserver.common import FDroidPopen, FDroidPopenBytes
-from fdroidserver.metadata import MetaDataException
+from . import _
+from . import common
+from . import metadata
+from . import net
+from . import signindex
+from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
+from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
def make(apps, sortedids, apks, repodir, archive):
return "fdroid.app:" + appid, apps[appid].Name
raise MetaDataException("Cannot resolve app id " + appid)
- nosigningkey = False
if not common.options.nosign:
- if 'repo_keyalias' not in common.config:
- nosigningkey = True
- logging.critical("'repo_keyalias' not found in config.py!")
- if 'keystore' not in common.config:
- nosigningkey = True
- logging.critical("'keystore' not found in config.py!")
- if 'keystorepass' not in common.config:
- nosigningkey = True
- logging.critical("'keystorepass' not found in config.py!")
- if 'keypass' not in common.config:
- nosigningkey = True
- logging.critical("'keypass' not found in config.py!")
- if not os.path.exists(common.config['keystore']):
- nosigningkey = True
- logging.critical("'" + common.config['keystore'] + "' does not exist!")
- if nosigningkey:
- logging.warning("`fdroid update` requires a signing key, you can create one using:")
- logging.warning("\tfdroid update --create-key")
- sys.exit(1)
+ common.assert_config_keystore(common.config)
repodict = collections.OrderedDict()
repodict['timestamp'] = datetime.utcnow()
for mirror in sorted(common.config.get('mirrors', [])):
base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
- logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
+ logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
mirrorcheckfailed = True
# must end with / or urljoin strips a whole path segment
if mirror.endswith('/'):
else:
mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
for mirror in common.config.get('servergitmirrors', []):
- mirror = get_raw_mirror(mirror)
- if mirror is not None:
- mirrors.append(mirror + '/')
+ for url in get_mirror_service_urls(mirror):
+ mirrors.append(url + '/' + repodir)
if mirrorcheckfailed:
- sys.exit(1)
+ raise FDroidException(_("Malformed repository mirrors."))
if mirrors:
repodict['mirrors'] = mirrors
elif all(isinstance(item, str) for item in common.config[key]):
packageNames = common.config[key]
else:
- raise TypeError('only accepts strings, lists, and tuples')
+ raise TypeError(_('only accepts strings, lists, and tuples'))
requestsdict[command] = packageNames
- make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
- make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
+ fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
+
+ make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
+ fdroid_signing_key_fingerprints)
+ make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
+ fdroid_signing_key_fingerprints)
-def make_v1(apps, packages, repodir, repodict, requestsdict):
+def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
def _index_encoder_default(obj):
if isinstance(obj, set):
- return list(obj)
+ return sorted(list(obj))
if isinstance(obj, datetime):
- return int(obj.timestamp() * 1000) # Java expects milliseconds
+ # Java prefers milliseconds
+ # we also need to accound for time zone/daylight saving time
+ return int(calendar.timegm(obj.timetuple()) * 1000)
+ if isinstance(obj, dict):
+ d = collections.OrderedDict()
+ for key in sorted(obj.keys()):
+ d[key] = obj[key]
+ return d
raise TypeError(repr(obj) + " is not JSON serializable")
output = collections.OrderedDict()
output['repo'] = repodict
output['requests'] = requestsdict
+ # establish sort order of the index
+ v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
+
appslist = []
output['apps'] = appslist
- for appid, appdict in apps.items():
+ for packageName, appdict in apps.items():
d = collections.OrderedDict()
appslist.append(d)
for k, v in sorted(appdict.items()):
elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name
k = 'suggestedVersionName'
elif k == 'AutoName':
- if 'Name' not in apps[appid]:
+ if 'Name' not in apps[packageName]:
d['name'] = v
continue
else:
k = k[:1].lower() + k[1:]
d[k] = v
+ # establish sort order in localized dicts
+ for app in output['apps']:
+ localized = app.get('localized')
+ if localized:
+ lordered = collections.OrderedDict()
+ for lkey, lvalue in sorted(localized.items()):
+ lordered[lkey] = collections.OrderedDict()
+ for ikey, iname in sorted(lvalue.items()):
+ lordered[lkey][ikey] = iname
+ app['localized'] = lordered
+
output_packages = collections.OrderedDict()
output['packages'] = output_packages
for package in packages:
packageName = package['packageName']
+ if packageName not in apps:
+ logging.info(_('Ignoring package without metadata: ') + package['apkName'])
+ continue
+ if not package.get('versionName'):
+ app = apps[packageName]
+ versionCodeStr = str(package['versionCode']) # TODO build.versionCode should be int!
+ for build in app['builds']:
+ if build['versionCode'] == versionCodeStr:
+ versionName = build.get('versionName')
+ logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}')
+ .format(apkfilename=package['apkName'], version=versionName))
+ package['versionName'] = versionName
+ break
if packageName in output_packages:
packagelist = output_packages[packageName]
else:
json.dump(output, fp, default=_index_encoder_default)
if common.options.nosign:
- logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
+ logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
else:
signindex.config = common.config
signindex.sign_index_v1(repodir, json_name)
-def make_v0(apps, apks, repodir, repodict, requestsdict):
+def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
+ """Sorts the supplied list to ensure a deterministic sort order for
+ package entries in the index file. This sort-order also expresses
+ installation preference to the clients.
+ (First in this list = first to install)
+
+ :param packages: list of packages which need to be sorted before but into index file.
+ """
+
+ GROUP_DEV_SIGNED = 1
+ GROUP_FDROID_SIGNED = 2
+ GROUP_OTHER_SIGNED = 3
+
+ def v1_sort_keys(package):
+ packageName = package.get('packageName', None)
+
+ sig = package.get('signer', None)
+
+ dev_sig = common.metadata_find_developer_signature(packageName)
+ group = GROUP_OTHER_SIGNED
+ if dev_sig and dev_sig == sig:
+ group = GROUP_DEV_SIGNED
+ else:
+ fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
+ if fdroidsig and fdroidsig == sig:
+ group = GROUP_FDROID_SIGNED
+
+ versionCode = None
+ if package.get('versionCode', None):
+ versionCode = -int(package['versionCode'])
+
+ return(packageName, group, sig, versionCode)
+
+ packages.sort(key=v1_sort_keys)
+
+
+def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
"""
aka index.jar aka index.xml
"""
el.appendChild(doc.createCDATASection(value))
parent.appendChild(el)
+ def addElementCheckLocalized(name, app, key, doc, parent, default=''):
+ '''Fill in field from metadata or localized block
+
+ For name/summary/description, they can come only from the app source,
+ or from a dir in fdroiddata. They can be entirely missing from the
+ metadata file if there is localized versions. This will fetch those
+ from the localized version if its not available in the metadata file.
+ '''
+
+ el = doc.createElement(name)
+ value = app.get(key)
+ lkey = key[:1].lower() + key[1:]
+ localized = app.get('localized')
+ if not value and localized:
+ for lang in ['en-US'] + [x for x in localized.keys()]:
+ if not lang.startswith('en'):
+ continue
+ if lang in localized:
+ value = localized[lang].get(lkey)
+ if value:
+ break
+ if not value and localized and len(localized) > 1:
+ lang = list(localized.keys())[0]
+ value = localized[lang].get(lkey)
+ if not value:
+ value = default
+ el.appendChild(doc.createTextNode(value))
+ parent.appendChild(el)
+
root = doc.createElement("fdroid")
doc.appendChild(root)
# Get a list of the apks for this app...
apklist = []
+ apksbyversion = collections.defaultdict(lambda: [])
for apk in apks:
- if apk['packageName'] == appid:
- apklist.append(apk)
+ if apk.get('versionCode') and apk.get('packageName') == appid:
+ apksbyversion[apk['versionCode']].append(apk)
+ for versionCode, apksforver in apksbyversion.items():
+ fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
+ fdroid_signed_apk = None
+ name_match_apk = None
+ for x in apksforver:
+ if fdroidsig and x.get('signer', None) == fdroidsig:
+ fdroid_signed_apk = x
+ if common.apk_release_filename.match(x.get('apkName', '')):
+ name_match_apk = x
+ # choose which of the available versions is most
+ # suiteable for index v0
+ if fdroid_signed_apk:
+ apklist.append(fdroid_signed_apk)
+ elif name_match_apk:
+ apklist.append(name_match_apk)
+ else:
+ apklist.append(apksforver[0])
if len(apklist) == 0:
continue
addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
if app.lastUpdated:
addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
- addElement('name', app.Name, doc, apel)
- addElement('summary', app.Summary, doc, apel)
+
+ addElementCheckLocalized('name', app, 'Name', doc, apel)
+ addElementCheckLocalized('summary', app, 'Summary', doc, apel)
+
if app.icon:
addElement('icon', app.icon, doc, apel)
- if app.get('Description'):
- description = app.Description
- else:
- description = '<p>No description available</p>'
- addElement('desc', description, doc, apel)
+ addElementCheckLocalized('desc', app, 'Description', doc, apel,
+ '<p>No description available</p>')
+
addElement('license', app.License, doc, apel)
if app.Categories:
addElement('categories', ','.join(app.Categories), doc, apel)
addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
addElementNonEmpty('flattr', app.FlattrID, doc, apel)
+ addElementNonEmpty('liberapay', app.LiberapayID, doc, apel)
# These elements actually refer to the current version (i.e. which
# one is recommended. They are historically mis-named, and need
# Check for duplicates - they will make the client unhappy...
for i in range(len(apklist) - 1):
- if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
- logging.critical("duplicate versions: '%s' - '%s'" % (
- apklist[i]['apkName'], apklist[i + 1]['apkName']))
- sys.exit(1)
+ first = apklist[i]
+ second = apklist[i + 1]
+ if first['versionCode'] == second['versionCode'] \
+ and first['sig'] == second['sig']:
+ if first['hash'] == second['hash']:
+ raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
+ repodir, first['apkName'], second['apkName']))
+ else:
+ raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
+ repodir, first['apkName'], second['apkName']))
current_version_code = 0
current_version_file = None
apkel = doc.createElement("package")
apel.appendChild(apkel)
- addElement('version', apk['versionName'], doc, apkel)
+
+ versionName = apk.get('versionName')
+ if not versionName:
+ versionCodeStr = str(apk['versionCode']) # TODO build.versionCode should be int!
+ for build in app.builds:
+ if build['versionCode'] == versionCodeStr and 'versionName' in build:
+ versionName = build['versionName']
+ break
+ if versionName:
+ addElement('version', versionName, doc, apkel)
+
addElement('versioncode', str(apk['versionCode']), doc, apkel)
addElement('apkname', apk['apkName'], doc, apkel)
addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
and repodir == 'repo': # only create these
namefield = common.config['current_version_name_source']
sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
- apklinkname = sanitized_name + b'.apk'
+ apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
if os.path.islink(apklinkname):
os.remove(apklinkname)
if 'repo_keyalias' in common.config:
if common.options.nosign:
- logging.info("Creating unsigned index in preparation for signing")
+ logging.info(_("Creating unsigned index in preparation for signing"))
else:
- logging.info("Creating signed index with this key (SHA256):")
+ logging.info(_("Creating signed index with this key (SHA256):"))
logging.info("%s" % repo_pubkey_fingerprint)
# Create a jar of the index...
jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
if p.returncode != 0:
- logging.critical("Failed to create {0}".format(jar_output))
- sys.exit(1)
+ raise FDroidException("Failed to create {0}".format(jar_output))
# Sign the index...
signed = os.path.join(repodir, 'index.jar')
msg = "Failed to get repo pubkey!"
if common.config['keystore'] == 'NONE':
msg += ' Is your crypto smartcard plugged in?'
- logging.critical(msg)
- sys.exit(1)
+ raise FDroidException(msg)
pubkey = p.output
repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
return hexlify(pubkey), repo_pubkey_fingerprint
-# Get raw URL from git service for mirroring
-def get_raw_mirror(url):
- # Divide urls in parts
- url = url.split("/")
+def get_mirror_service_urls(url):
+ '''Get direct URLs from git service for use by fdroidclient
+
+ Via 'servergitmirrors', fdroidserver can create and push a mirror
+ to certain well known git services like gitlab or github. This
+ will always use the 'master' branch since that is the default
+ branch in git. The files are then accessible via alternate URLs,
+ where they are served in their raw format via a CDN rather than
+ from git.
+ '''
- # Get the hostname
- hostname = url[2]
+ if url.startswith('git@'):
+ url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
- # fdroidserver will use always 'master' branch for git-mirroring
+ segments = url.split("/")
+
+ if segments[4].endswith('.git'):
+ segments[4] = segments[4][:-4]
+
+ hostname = segments[2]
+ user = segments[3]
+ repo = segments[4]
branch = "master"
folder = "fdroid"
+ urls = []
if hostname == "github.com":
- # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
- url[2] = "raw.githubusercontent.com"
- url.extend([branch, folder])
+ # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
+ segments[2] = "raw.githubusercontent.com"
+ segments.extend([branch, folder])
+ urls.append('/'.join(segments))
elif hostname == "gitlab.com":
- # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
- url.extend(["raw", branch, folder])
- else:
- return None
+ # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser
+ # This is because the `raw` URLs are not served with the correct mime types, so any
+ # index.html which is put in the repo will not be rendered. Putting an index.html file in
+ # the repo root is a common way for to make information about the repo available to end user.
- url = "/".join(url)
- return url
+ # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
+ gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
+ urls.append('/'.join(gitlab_pages))
+ # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
+ gitlab_raw = segments + ['raw', branch, folder]
+ urls.append('/'.join(gitlab_raw))
+ return urls
+ return urls
-class VerificationException(Exception):
- pass
-
-def download_repo_index(url_str, verify_fingerprint=True):
+def download_repo_index(url_str, etag=None, verify_fingerprint=True):
"""
Downloads the repository index from the given :param url_str
and verifies the repository's fingerprint if :param verify_fingerprint is not False.
:raises: VerificationException() if the repository could not be verified
- :return: The index in JSON format.
+ :return: A tuple consisting of:
+ - The index in JSON format or None if the index did not change
+ - The new eTag as returned by the HTTP request
"""
url = urllib.parse.urlsplit(url_str)
if verify_fingerprint:
query = urllib.parse.parse_qs(url.query)
if 'fingerprint' not in query:
- raise VerificationException("No fingerprint in URL.")
+ raise VerificationException(_("No fingerprint in URL."))
fingerprint = query['fingerprint'][0]
url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
- r = requests.get(url.geturl())
+ download, new_etag = net.http_get(url.geturl(), etag)
+
+ if download is None:
+ return None, new_etag
with tempfile.NamedTemporaryFile() as fp:
# write and open JAR file
- fp.write(r.content)
+ fp.write(download)
jar = zipfile.ZipFile(fp)
# verify that the JAR signature is valid
- verify_jar_signature(fp.name)
+ logging.debug(_('Verifying index signature:'))
+ common.verify_jar_signature(fp.name)
# get public key and its fingerprint from JAR
public_key, public_key_fingerprint = get_public_key_from_jar(jar)
# compare the fingerprint if verify_fingerprint is True
if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
- raise VerificationException("The repository's fingerprint does not match.")
+ raise VerificationException(_("The repository's fingerprint does not match."))
# load repository index from JSON
index = json.loads(jar.read('index-v1.json').decode("utf-8"))
# turn the apps into App objects
index["apps"] = [metadata.App(app) for app in index["apps"]]
- return index
-
-
-def verify_jar_signature(file):
- """
- Verifies the signature of a given JAR file.
-
- :raises: VerificationException() if the JAR's signature could not be verified
- """
- if not common.verify_apk_signature(file, jar=True):
- raise VerificationException("The repository's index could not be verified.")
+ return index, new_etag
def get_public_key_from_jar(jar):
# extract certificate from jar
certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
if len(certs) < 1:
- raise VerificationException("Found no signing certificates for repository.")
+ raise VerificationException(_("Found no signing certificates for repository."))
if len(certs) > 1:
- raise VerificationException("Found multiple signing certificates for repository.")
+ raise VerificationException(_("Found multiple signing certificates for repository."))
# extract public key from certificate
public_key = common.get_certificate(jar.read(certs[0]))
~ianmdlvl / fdroidserver.git / blobdiff