3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
5 # Copyright (C) 2016, Blue Jay Wireless
6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
8 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU Affero General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Affero General Public License for more details.
20 # You should have received a copy of the GNU Affero General Public License
21 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from binascii import hexlify, unhexlify
34 from datetime import datetime
35 from xml.dom.minidom import Document
39 from . import metadata
41 from . import signindex
42 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
43 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
46 def make(apps, sortedids, apks, repodir, archive):
47 """Generate the repo index files.
49 This requires properly initialized options and config objects.
51 :param apps: fully populated apps list
52 :param sortedids: app package IDs, sorted
53 :param apks: full populated apks list
54 :param repodir: the repo directory
55 :param archive: True if this is the archive repo, False if it's the
58 from fdroidserver.update import METADATA_VERSION
60 def _resolve_description_link(appid):
62 return "fdroid.app:" + appid, apps[appid].Name
63 raise MetaDataException("Cannot resolve app id " + appid)
66 if not common.options.nosign:
67 if 'repo_keyalias' not in common.config:
69 logging.critical(_("'repo_keyalias' not found in config.py!"))
70 if 'keystore' not in common.config:
72 logging.critical(_("'keystore' not found in config.py!"))
73 if 'keystorepass' not in common.config:
75 logging.critical(_("'keystorepass' not found in config.py!"))
76 if 'keypass' not in common.config:
78 logging.critical(_("'keypass' not found in config.py!"))
79 if not os.path.exists(common.config['keystore']):
81 logging.critical("'" + common.config['keystore'] + "' does not exist!")
83 raise FDroidException("`fdroid update` requires a signing key, " +
84 "you can create one using: fdroid update --create-key")
86 repodict = collections.OrderedDict()
87 repodict['timestamp'] = datetime.utcnow()
88 repodict['version'] = METADATA_VERSION
90 if common.config['repo_maxage'] != 0:
91 repodict['maxage'] = common.config['repo_maxage']
94 repodict['name'] = common.config['archive_name']
95 repodict['icon'] = os.path.basename(common.config['archive_icon'])
96 repodict['address'] = common.config['archive_url']
97 repodict['description'] = common.config['archive_description']
98 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
100 repodict['name'] = common.config['repo_name']
101 repodict['icon'] = os.path.basename(common.config['repo_icon'])
102 repodict['address'] = common.config['repo_url']
103 repodict['description'] = common.config['repo_description']
104 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
106 mirrorcheckfailed = False
108 for mirror in sorted(common.config.get('mirrors', [])):
109 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
110 if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
111 logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
112 mirrorcheckfailed = True
113 # must end with / or urljoin strips a whole path segment
114 if mirror.endswith('/'):
115 mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
117 mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
118 for mirror in common.config.get('servergitmirrors', []):
119 for url in get_mirror_service_urls(mirror):
120 mirrors.append(url + '/' + repodir)
121 if mirrorcheckfailed:
122 raise FDroidException(_("Malformed repository mirrors."))
124 repodict['mirrors'] = mirrors
126 appsWithPackages = collections.OrderedDict()
127 for packageName in sortedids:
128 app = apps[packageName]
132 # only include apps with packages
134 if apk['packageName'] == packageName:
135 newapp = copy.copy(app) # update wiki needs unmodified description
136 newapp['Description'] = metadata.description_html(app['Description'],
137 _resolve_description_link)
138 appsWithPackages[packageName] = newapp
141 requestsdict = collections.OrderedDict()
142 for command in ('install', 'uninstall'):
144 key = command + '_list'
145 if key in common.config:
146 if isinstance(common.config[key], str):
147 packageNames = [common.config[key]]
148 elif all(isinstance(item, str) for item in common.config[key]):
149 packageNames = common.config[key]
151 raise TypeError(_('only accepts strings, lists, and tuples'))
152 requestsdict[command] = packageNames
154 make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
155 make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
158 def make_v1(apps, packages, repodir, repodict, requestsdict):
160 def _index_encoder_default(obj):
161 if isinstance(obj, set):
163 if isinstance(obj, datetime):
164 return int(obj.timestamp() * 1000) # Java expects milliseconds
165 raise TypeError(repr(obj) + " is not JSON serializable")
167 output = collections.OrderedDict()
168 output['repo'] = repodict
169 output['requests'] = requestsdict
172 output['apps'] = appslist
173 for packageName, appdict in apps.items():
174 d = collections.OrderedDict()
176 for k, v in sorted(appdict.items()):
179 if k in ('builds', 'comments', 'metadatapath',
180 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
181 'Provides', 'Repo', 'RepoType', 'RequiresRoot',
182 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
183 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
186 # name things after the App class fields in fdroidclient
189 elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name
190 k = 'suggestedVersionCode'
191 elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name
192 k = 'suggestedVersionName'
193 elif k == 'AutoName':
194 if 'Name' not in apps[packageName]:
198 k = k[:1].lower() + k[1:]
201 output_packages = collections.OrderedDict()
202 output['packages'] = output_packages
203 for package in packages:
204 packageName = package['packageName']
205 if packageName not in apps:
206 logging.info(_('Ignoring package without metadata: ') + package['apkName'])
208 if packageName in output_packages:
209 packagelist = output_packages[packageName]
212 output_packages[packageName] = packagelist
213 d = collections.OrderedDict()
214 packagelist.append(d)
215 for k, v in sorted(package.items()):
218 if k in ('icon', 'icons', 'icons_src', 'name', ):
222 json_name = 'index-v1.json'
223 index_file = os.path.join(repodir, json_name)
224 with open(index_file, 'w') as fp:
225 if common.options.pretty:
226 json.dump(output, fp, default=_index_encoder_default, indent=2)
228 json.dump(output, fp, default=_index_encoder_default)
230 if common.options.nosign:
231 logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
233 signindex.config = common.config
234 signindex.sign_index_v1(repodir, json_name)
237 def make_v0(apps, apks, repodir, repodict, requestsdict):
239 aka index.jar aka index.xml
244 def addElement(name, value, doc, parent):
245 el = doc.createElement(name)
246 el.appendChild(doc.createTextNode(value))
247 parent.appendChild(el)
249 def addElementNonEmpty(name, value, doc, parent):
252 addElement(name, value, doc, parent)
254 def addElementIfInApk(name, apk, key, doc, parent):
257 value = str(apk[key])
258 addElement(name, value, doc, parent)
260 def addElementCDATA(name, value, doc, parent):
261 el = doc.createElement(name)
262 el.appendChild(doc.createCDATASection(value))
263 parent.appendChild(el)
265 def addElementCheckLocalized(name, app, key, doc, parent, default=''):
266 '''Fill in field from metadata or localized block
268 For name/summary/description, they can come only from the app source,
269 or from a dir in fdroiddata. They can be entirely missing from the
270 metadata file if there is localized versions. This will fetch those
271 from the localized version if its not available in the metadata file.
274 el = doc.createElement(name)
276 lkey = key[:1].lower() + key[1:]
277 localized = app.get('localized')
278 if not value and localized:
279 for lang in ['en-US'] + [x for x in localized.keys()]:
280 if not lang.startswith('en'):
282 if lang in localized:
283 value = localized[lang].get(lkey)
286 if not value and localized and len(localized) > 1:
287 lang = list(localized.keys())[0]
288 value = localized[lang].get(lkey)
291 el.appendChild(doc.createTextNode(value))
292 parent.appendChild(el)
294 root = doc.createElement("fdroid")
295 doc.appendChild(root)
297 repoel = doc.createElement("repo")
299 repoel.setAttribute("name", repodict['name'])
300 if 'maxage' in repodict:
301 repoel.setAttribute("maxage", str(repodict['maxage']))
302 repoel.setAttribute("icon", os.path.basename(repodict['icon']))
303 repoel.setAttribute("url", repodict['address'])
304 addElement('description', repodict['description'], doc, repoel)
305 for mirror in repodict.get('mirrors', []):
306 addElement('mirror', mirror, doc, repoel)
308 repoel.setAttribute("version", str(repodict['version']))
309 repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
311 pubkey, repo_pubkey_fingerprint = extract_pubkey()
312 repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
313 root.appendChild(repoel)
315 for command in ('install', 'uninstall'):
316 for packageName in requestsdict[command]:
317 element = doc.createElement(command)
318 root.appendChild(element)
319 element.setAttribute('packageName', packageName)
321 for appid, appdict in apps.items():
322 app = metadata.App(appdict)
324 if app.Disabled is not None:
327 # Get a list of the apks for this app...
331 if apk['packageName'] == appid:
332 if apk['versionCode'] not in versionCodes:
334 versionCodes.append(apk['versionCode'])
336 if len(apklist) == 0:
339 apel = doc.createElement("application")
340 apel.setAttribute("id", app.id)
341 root.appendChild(apel)
343 addElement('id', app.id, doc, apel)
345 addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
347 addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
349 addElementCheckLocalized('name', app, 'Name', doc, apel)
350 addElementCheckLocalized('summary', app, 'Summary', doc, apel)
353 addElement('icon', app.icon, doc, apel)
355 addElementCheckLocalized('desc', app, 'Description', doc, apel,
356 '<p>No description available</p>')
358 addElement('license', app.License, doc, apel)
360 addElement('categories', ','.join(app.Categories), doc, apel)
361 # We put the first (primary) category in LAST, which will have
362 # the desired effect of making clients that only understand one
363 # category see that one.
364 addElement('category', app.Categories[0], doc, apel)
365 addElement('web', app.WebSite, doc, apel)
366 addElement('source', app.SourceCode, doc, apel)
367 addElement('tracker', app.IssueTracker, doc, apel)
368 addElementNonEmpty('changelog', app.Changelog, doc, apel)
369 addElementNonEmpty('author', app.AuthorName, doc, apel)
370 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
371 addElementNonEmpty('donate', app.Donate, doc, apel)
372 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
373 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
374 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
376 # These elements actually refer to the current version (i.e. which
377 # one is recommended. They are historically mis-named, and need
378 # changing, but stay like this for now to support existing clients.
379 addElement('marketversion', app.CurrentVersion, doc, apel)
380 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
383 pv = app.Provides.split(',')
384 addElementNonEmpty('provides', ','.join(pv), doc, apel)
386 addElement('requirements', 'root', doc, apel)
388 # Sort the apk list into version order, just so the web site
389 # doesn't have to do any work by default...
390 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
392 if 'antiFeatures' in apklist[0]:
393 app.AntiFeatures.extend(apklist[0]['antiFeatures'])
395 addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
397 # Check for duplicates - they will make the client unhappy...
398 for i in range(len(apklist) - 1):
400 second = apklist[i + 1]
401 if first['versionCode'] == second['versionCode'] \
402 and first['sig'] == second['sig']:
403 if first['hash'] == second['hash']:
404 raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
405 repodir, first['apkName'], second['apkName']))
407 raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
408 repodir, first['apkName'], second['apkName']))
410 current_version_code = 0
411 current_version_file = None
413 file_extension = common.get_file_extension(apk['apkName'])
414 # find the APK for the "Current Version"
415 if current_version_code < apk['versionCode']:
416 current_version_code = apk['versionCode']
417 if current_version_code < int(app.CurrentVersionCode):
418 current_version_file = apk['apkName']
420 apkel = doc.createElement("package")
421 apel.appendChild(apkel)
422 addElement('version', apk['versionName'], doc, apkel)
423 addElement('versioncode', str(apk['versionCode']), doc, apkel)
424 addElement('apkname', apk['apkName'], doc, apkel)
425 addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
427 hashel = doc.createElement("hash")
428 hashel.setAttribute('type', 'sha256')
429 hashel.appendChild(doc.createTextNode(apk['hash']))
430 apkel.appendChild(hashel)
432 addElement('size', str(apk['size']), doc, apkel)
433 addElementIfInApk('sdkver', apk,
434 'minSdkVersion', doc, apkel)
435 addElementIfInApk('targetSdkVersion', apk,
436 'targetSdkVersion', doc, apkel)
437 addElementIfInApk('maxsdkver', apk,
438 'maxSdkVersion', doc, apkel)
439 addElementIfInApk('obbMainFile', apk,
440 'obbMainFile', doc, apkel)
441 addElementIfInApk('obbMainFileSha256', apk,
442 'obbMainFileSha256', doc, apkel)
443 addElementIfInApk('obbPatchFile', apk,
444 'obbPatchFile', doc, apkel)
445 addElementIfInApk('obbPatchFileSha256', apk,
446 'obbPatchFileSha256', doc, apkel)
448 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
450 if file_extension == 'apk': # sig is required for APKs, but only APKs
451 addElement('sig', apk['sig'], doc, apkel)
453 old_permissions = set()
454 sorted_permissions = sorted(apk['uses-permission'])
455 for perm in sorted_permissions:
456 perm_name = perm.name
457 if perm_name.startswith("android.permission."):
458 perm_name = perm_name[19:]
459 old_permissions.add(perm_name)
460 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
462 for permission in sorted_permissions:
463 permel = doc.createElement('uses-permission')
464 permel.setAttribute('name', permission.name)
465 if permission.maxSdkVersion is not None:
466 permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
467 apkel.appendChild(permel)
468 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
469 permel = doc.createElement('uses-permission-sdk-23')
470 permel.setAttribute('name', permission_sdk_23.name)
471 if permission_sdk_23.maxSdkVersion is not None:
472 permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
473 apkel.appendChild(permel)
474 if 'nativecode' in apk:
475 addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
476 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
478 if current_version_file is not None \
479 and common.config['make_current_version_link'] \
480 and repodir == 'repo': # only create these
481 namefield = common.config['current_version_name_source']
482 sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
483 apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
484 current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
485 if os.path.islink(apklinkname):
486 os.remove(apklinkname)
487 os.symlink(current_version_path, apklinkname)
488 # also symlink gpg signature, if it exists
489 for extension in (b'.asc', b'.sig'):
490 sigfile_path = current_version_path + extension
491 if os.path.exists(sigfile_path):
492 siglinkname = apklinkname + extension
493 if os.path.islink(siglinkname):
494 os.remove(siglinkname)
495 os.symlink(sigfile_path, siglinkname)
497 if common.options.pretty:
498 output = doc.toprettyxml(encoding='utf-8')
500 output = doc.toxml(encoding='utf-8')
502 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
505 if 'repo_keyalias' in common.config:
507 if common.options.nosign:
508 logging.info(_("Creating unsigned index in preparation for signing"))
510 logging.info(_("Creating signed index with this key (SHA256):"))
511 logging.info("%s" % repo_pubkey_fingerprint)
513 # Create a jar of the index...
514 jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
515 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
516 if p.returncode != 0:
517 raise FDroidException("Failed to create {0}".format(jar_output))
520 signed = os.path.join(repodir, 'index.jar')
521 if common.options.nosign:
522 # Remove old signed index if not signing
523 if os.path.exists(signed):
526 signindex.config = common.config
527 signindex.sign_jar(signed)
529 # Copy the repo icon into the repo directory...
530 icon_dir = os.path.join(repodir, 'icons')
531 iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
532 shutil.copyfile(common.config['repo_icon'], iconfilename)
535 def extract_pubkey():
537 Extracts and returns the repository's public key from the keystore.
538 :return: public key in hex, repository fingerprint
540 if 'repo_pubkey' in common.config:
541 pubkey = unhexlify(common.config['repo_pubkey'])
543 env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
544 p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
545 '-alias', common.config['repo_keyalias'],
546 '-keystore', common.config['keystore'],
547 '-storepass:env', 'FDROID_KEY_STORE_PASS']
548 + common.config['smartcardoptions'],
549 envs=env_vars, output=False, stderr_to_stdout=False)
550 if p.returncode != 0 or len(p.output) < 20:
551 msg = "Failed to get repo pubkey!"
552 if common.config['keystore'] == 'NONE':
553 msg += ' Is your crypto smartcard plugged in?'
554 raise FDroidException(msg)
556 repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
557 return hexlify(pubkey), repo_pubkey_fingerprint
560 def get_mirror_service_urls(url):
561 '''Get direct URLs from git service for use by fdroidclient
563 Via 'servergitmirrors', fdroidserver can create and push a mirror
564 to certain well known git services like gitlab or github. This
565 will always use the 'master' branch since that is the default
566 branch in git. The files are then accessible via alternate URLs,
567 where they are served in their raw format via a CDN rather than
571 if url.startswith('git@'):
572 url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
574 segments = url.split("/")
576 if segments[4].endswith('.git'):
577 segments[4] = segments[4][:-4]
579 hostname = segments[2]
586 if hostname == "github.com":
587 # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
588 segments[2] = "raw.githubusercontent.com"
589 segments.extend([branch, folder])
590 urls.append('/'.join(segments))
591 elif hostname == "gitlab.com":
592 # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
593 gitlab_raw = segments + ['raw', branch, folder]
594 urls.append('/'.join(gitlab_raw))
595 # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
596 gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
597 urls.append('/'.join(gitlab_pages))
603 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
605 Downloads the repository index from the given :param url_str
606 and verifies the repository's fingerprint if :param verify_fingerprint is not False.
608 :raises: VerificationException() if the repository could not be verified
610 :return: A tuple consisting of:
611 - The index in JSON format or None if the index did not change
612 - The new eTag as returned by the HTTP request
614 url = urllib.parse.urlsplit(url_str)
617 if verify_fingerprint:
618 query = urllib.parse.parse_qs(url.query)
619 if 'fingerprint' not in query:
620 raise VerificationException(_("No fingerprint in URL."))
621 fingerprint = query['fingerprint'][0]
623 url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
624 download, new_etag = net.http_get(url.geturl(), etag)
627 return None, new_etag
629 with tempfile.NamedTemporaryFile() as fp:
630 # write and open JAR file
632 jar = zipfile.ZipFile(fp)
634 # verify that the JAR signature is valid
635 common.verify_jar_signature(fp.name)
637 # get public key and its fingerprint from JAR
638 public_key, public_key_fingerprint = get_public_key_from_jar(jar)
640 # compare the fingerprint if verify_fingerprint is True
641 if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
642 raise VerificationException(_("The repository's fingerprint does not match."))
644 # load repository index from JSON
645 index = json.loads(jar.read('index-v1.json').decode("utf-8"))
646 index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
647 index["repo"]["fingerprint"] = public_key_fingerprint
649 # turn the apps into App objects
650 index["apps"] = [metadata.App(app) for app in index["apps"]]
652 return index, new_etag
655 def get_public_key_from_jar(jar):
657 Get the public key and its fingerprint from a JAR file.
659 :raises: VerificationException() if the JAR was not signed exactly once
661 :param jar: a zipfile.ZipFile object
662 :return: the public key from the jar and its fingerprint
664 # extract certificate from jar
665 certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
667 raise VerificationException(_("Found no signing certificates for repository."))
669 raise VerificationException(_("Found multiple signing certificates for repository."))
671 # extract public key from certificate
672 public_key = common.get_certificate(jar.read(certs[0]))
673 public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
675 return public_key, public_key_fingerprint