3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
5 # Copyright (C) 2016, Blue Jay Wireless
6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
8 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU Affero General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Affero General Public License for more details.
20 # You should have received a copy of the GNU Affero General Public License
21 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from binascii import hexlify, unhexlify
34 from datetime import datetime
35 from xml.dom.minidom import Document
37 from fdroidserver import metadata, signindex, common, net
38 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
39 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
42 def make(apps, sortedids, apks, repodir, archive):
43 """Generate the repo index files.
45 This requires properly initialized options and config objects.
47 :param apps: fully populated apps list
48 :param sortedids: app package IDs, sorted
49 :param apks: full populated apks list
50 :param repodir: the repo directory
51 :param archive: True if this is the archive repo, False if it's the
54 from fdroidserver.update import METADATA_VERSION
56 def _resolve_description_link(appid):
58 return "fdroid.app:" + appid, apps[appid].Name
59 raise MetaDataException("Cannot resolve app id " + appid)
62 if not common.options.nosign:
63 if 'repo_keyalias' not in common.config:
65 logging.critical("'repo_keyalias' not found in config.py!")
66 if 'keystore' not in common.config:
68 logging.critical("'keystore' not found in config.py!")
69 if 'keystorepass' not in common.config:
71 logging.critical("'keystorepass' not found in config.py!")
72 if 'keypass' not in common.config:
74 logging.critical("'keypass' not found in config.py!")
75 if not os.path.exists(common.config['keystore']):
77 logging.critical("'" + common.config['keystore'] + "' does not exist!")
79 raise FDroidException("`fdroid update` requires a signing key, " +
80 "you can create one using: fdroid update --create-key")
82 repodict = collections.OrderedDict()
83 repodict['timestamp'] = datetime.utcnow()
84 repodict['version'] = METADATA_VERSION
86 if common.config['repo_maxage'] != 0:
87 repodict['maxage'] = common.config['repo_maxage']
90 repodict['name'] = common.config['archive_name']
91 repodict['icon'] = os.path.basename(common.config['archive_icon'])
92 repodict['address'] = common.config['archive_url']
93 repodict['description'] = common.config['archive_description']
94 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
96 repodict['name'] = common.config['repo_name']
97 repodict['icon'] = os.path.basename(common.config['repo_icon'])
98 repodict['address'] = common.config['repo_url']
99 repodict['description'] = common.config['repo_description']
100 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
102 mirrorcheckfailed = False
104 for mirror in sorted(common.config.get('mirrors', [])):
105 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
106 if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
107 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
108 mirrorcheckfailed = True
109 # must end with / or urljoin strips a whole path segment
110 if mirror.endswith('/'):
111 mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
113 mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
114 for mirror in common.config.get('servergitmirrors', []):
115 for url in get_mirror_service_urls(mirror):
116 mirrors.append(url + '/' + repodir)
117 if mirrorcheckfailed:
118 raise FDroidException("Malformed repository mirrors.")
120 repodict['mirrors'] = mirrors
122 appsWithPackages = collections.OrderedDict()
123 for packageName in sortedids:
124 app = apps[packageName]
128 # only include apps with packages
130 if apk['packageName'] == packageName:
131 newapp = copy.copy(app) # update wiki needs unmodified description
132 newapp['Description'] = metadata.description_html(app['Description'],
133 _resolve_description_link)
134 appsWithPackages[packageName] = newapp
137 requestsdict = collections.OrderedDict()
138 for command in ('install', 'uninstall'):
140 key = command + '_list'
141 if key in common.config:
142 if isinstance(common.config[key], str):
143 packageNames = [common.config[key]]
144 elif all(isinstance(item, str) for item in common.config[key]):
145 packageNames = common.config[key]
147 raise TypeError('only accepts strings, lists, and tuples')
148 requestsdict[command] = packageNames
150 make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
151 make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
154 def make_v1(apps, packages, repodir, repodict, requestsdict):
156 def _index_encoder_default(obj):
157 if isinstance(obj, set):
159 if isinstance(obj, datetime):
160 return int(obj.timestamp() * 1000) # Java expects milliseconds
161 raise TypeError(repr(obj) + " is not JSON serializable")
163 output = collections.OrderedDict()
164 output['repo'] = repodict
165 output['requests'] = requestsdict
168 output['apps'] = appslist
169 for packageName, appdict in apps.items():
170 d = collections.OrderedDict()
172 for k, v in sorted(appdict.items()):
175 if k in ('builds', 'comments', 'metadatapath',
176 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
177 'Provides', 'Repo', 'RepoType', 'RequiresRoot',
178 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
179 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
182 # name things after the App class fields in fdroidclient
185 elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name
186 k = 'suggestedVersionCode'
187 elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name
188 k = 'suggestedVersionName'
189 elif k == 'AutoName':
190 if 'Name' not in apps[packageName]:
194 k = k[:1].lower() + k[1:]
197 output_packages = collections.OrderedDict()
198 output['packages'] = output_packages
199 for package in packages:
200 packageName = package['packageName']
201 if packageName not in apps:
202 logging.info('Ignoring package without metadata: ' + package['apkName'])
204 if packageName in output_packages:
205 packagelist = output_packages[packageName]
208 output_packages[packageName] = packagelist
209 d = collections.OrderedDict()
210 packagelist.append(d)
211 for k, v in sorted(package.items()):
214 if k in ('icon', 'icons', 'icons_src', 'name', ):
218 json_name = 'index-v1.json'
219 index_file = os.path.join(repodir, json_name)
220 with open(index_file, 'w') as fp:
221 if common.options.pretty:
222 json.dump(output, fp, default=_index_encoder_default, indent=2)
224 json.dump(output, fp, default=_index_encoder_default)
226 if common.options.nosign:
227 logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
229 signindex.config = common.config
230 signindex.sign_index_v1(repodir, json_name)
233 def make_v0(apps, apks, repodir, repodict, requestsdict):
235 aka index.jar aka index.xml
240 def addElement(name, value, doc, parent):
241 el = doc.createElement(name)
242 el.appendChild(doc.createTextNode(value))
243 parent.appendChild(el)
245 def addElementNonEmpty(name, value, doc, parent):
248 addElement(name, value, doc, parent)
250 def addElementIfInApk(name, apk, key, doc, parent):
253 value = str(apk[key])
254 addElement(name, value, doc, parent)
256 def addElementCDATA(name, value, doc, parent):
257 el = doc.createElement(name)
258 el.appendChild(doc.createCDATASection(value))
259 parent.appendChild(el)
261 def addElementCheckLocalized(name, app, key, doc, parent, default=''):
262 '''Fill in field from metadata or localized block
264 For name/summary/description, they can come only from the app source,
265 or from a dir in fdroiddata. They can be entirely missing from the
266 metadata file if there is localized versions. This will fetch those
267 from the localized version if its not available in the metadata file.
270 el = doc.createElement(name)
272 lkey = key[:1].lower() + key[1:]
273 localized = app.get('localized')
274 if not value and localized:
275 for lang in ['en-US'] + [x for x in localized.keys()]:
276 if not lang.startswith('en'):
278 if lang in localized:
279 value = localized[lang].get(lkey)
282 if not value and localized and len(localized) > 1:
283 lang = list(localized.keys())[0]
284 value = localized[lang].get(lkey)
287 el.appendChild(doc.createTextNode(value))
288 parent.appendChild(el)
290 root = doc.createElement("fdroid")
291 doc.appendChild(root)
293 repoel = doc.createElement("repo")
295 repoel.setAttribute("name", repodict['name'])
296 if 'maxage' in repodict:
297 repoel.setAttribute("maxage", str(repodict['maxage']))
298 repoel.setAttribute("icon", os.path.basename(repodict['icon']))
299 repoel.setAttribute("url", repodict['address'])
300 addElement('description', repodict['description'], doc, repoel)
301 for mirror in repodict.get('mirrors', []):
302 addElement('mirror', mirror, doc, repoel)
304 repoel.setAttribute("version", str(repodict['version']))
305 repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
307 pubkey, repo_pubkey_fingerprint = extract_pubkey()
308 repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
309 root.appendChild(repoel)
311 for command in ('install', 'uninstall'):
312 for packageName in requestsdict[command]:
313 element = doc.createElement(command)
314 root.appendChild(element)
315 element.setAttribute('packageName', packageName)
317 for appid, appdict in apps.items():
318 app = metadata.App(appdict)
320 if app.Disabled is not None:
323 # Get a list of the apks for this app...
327 if apk['packageName'] == appid:
328 if apk['versionCode'] not in versionCodes:
330 versionCodes.append(apk['versionCode'])
332 if len(apklist) == 0:
335 apel = doc.createElement("application")
336 apel.setAttribute("id", app.id)
337 root.appendChild(apel)
339 addElement('id', app.id, doc, apel)
341 addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
343 addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
345 addElementCheckLocalized('name', app, 'Name', doc, apel)
346 addElementCheckLocalized('summary', app, 'Summary', doc, apel)
349 addElement('icon', app.icon, doc, apel)
351 addElementCheckLocalized('desc', app, 'Description', doc, apel,
352 '<p>No description available</p>')
354 addElement('license', app.License, doc, apel)
356 addElement('categories', ','.join(app.Categories), doc, apel)
357 # We put the first (primary) category in LAST, which will have
358 # the desired effect of making clients that only understand one
359 # category see that one.
360 addElement('category', app.Categories[0], doc, apel)
361 addElement('web', app.WebSite, doc, apel)
362 addElement('source', app.SourceCode, doc, apel)
363 addElement('tracker', app.IssueTracker, doc, apel)
364 addElementNonEmpty('changelog', app.Changelog, doc, apel)
365 addElementNonEmpty('author', app.AuthorName, doc, apel)
366 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
367 addElementNonEmpty('donate', app.Donate, doc, apel)
368 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
369 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
370 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
372 # These elements actually refer to the current version (i.e. which
373 # one is recommended. They are historically mis-named, and need
374 # changing, but stay like this for now to support existing clients.
375 addElement('marketversion', app.CurrentVersion, doc, apel)
376 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
379 pv = app.Provides.split(',')
380 addElementNonEmpty('provides', ','.join(pv), doc, apel)
382 addElement('requirements', 'root', doc, apel)
384 # Sort the apk list into version order, just so the web site
385 # doesn't have to do any work by default...
386 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
388 if 'antiFeatures' in apklist[0]:
389 app.AntiFeatures.extend(apklist[0]['antiFeatures'])
391 addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
393 # Check for duplicates - they will make the client unhappy...
394 for i in range(len(apklist) - 1):
396 second = apklist[i + 1]
397 if first['versionCode'] == second['versionCode'] \
398 and first['sig'] == second['sig']:
399 if first['hash'] == second['hash']:
400 raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
401 repodir, first['apkName'], second['apkName']))
403 raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
404 repodir, first['apkName'], second['apkName']))
406 current_version_code = 0
407 current_version_file = None
409 file_extension = common.get_file_extension(apk['apkName'])
410 # find the APK for the "Current Version"
411 if current_version_code < apk['versionCode']:
412 current_version_code = apk['versionCode']
413 if current_version_code < int(app.CurrentVersionCode):
414 current_version_file = apk['apkName']
416 apkel = doc.createElement("package")
417 apel.appendChild(apkel)
418 addElement('version', apk['versionName'], doc, apkel)
419 addElement('versioncode', str(apk['versionCode']), doc, apkel)
420 addElement('apkname', apk['apkName'], doc, apkel)
421 addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
423 hashel = doc.createElement("hash")
424 hashel.setAttribute('type', 'sha256')
425 hashel.appendChild(doc.createTextNode(apk['hash']))
426 apkel.appendChild(hashel)
428 addElement('size', str(apk['size']), doc, apkel)
429 addElementIfInApk('sdkver', apk,
430 'minSdkVersion', doc, apkel)
431 addElementIfInApk('targetSdkVersion', apk,
432 'targetSdkVersion', doc, apkel)
433 addElementIfInApk('maxsdkver', apk,
434 'maxSdkVersion', doc, apkel)
435 addElementIfInApk('obbMainFile', apk,
436 'obbMainFile', doc, apkel)
437 addElementIfInApk('obbMainFileSha256', apk,
438 'obbMainFileSha256', doc, apkel)
439 addElementIfInApk('obbPatchFile', apk,
440 'obbPatchFile', doc, apkel)
441 addElementIfInApk('obbPatchFileSha256', apk,
442 'obbPatchFileSha256', doc, apkel)
444 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
446 if file_extension == 'apk': # sig is required for APKs, but only APKs
447 addElement('sig', apk['sig'], doc, apkel)
449 old_permissions = set()
450 sorted_permissions = sorted(apk['uses-permission'])
451 for perm in sorted_permissions:
452 perm_name = perm.name
453 if perm_name.startswith("android.permission."):
454 perm_name = perm_name[19:]
455 old_permissions.add(perm_name)
456 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
458 for permission in sorted_permissions:
459 permel = doc.createElement('uses-permission')
460 permel.setAttribute('name', permission.name)
461 if permission.maxSdkVersion is not None:
462 permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
463 apkel.appendChild(permel)
464 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
465 permel = doc.createElement('uses-permission-sdk-23')
466 permel.setAttribute('name', permission_sdk_23.name)
467 if permission_sdk_23.maxSdkVersion is not None:
468 permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
469 apkel.appendChild(permel)
470 if 'nativecode' in apk:
471 addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
472 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
474 if current_version_file is not None \
475 and common.config['make_current_version_link'] \
476 and repodir == 'repo': # only create these
477 namefield = common.config['current_version_name_source']
478 sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
479 apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
480 current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
481 if os.path.islink(apklinkname):
482 os.remove(apklinkname)
483 os.symlink(current_version_path, apklinkname)
484 # also symlink gpg signature, if it exists
485 for extension in (b'.asc', b'.sig'):
486 sigfile_path = current_version_path + extension
487 if os.path.exists(sigfile_path):
488 siglinkname = apklinkname + extension
489 if os.path.islink(siglinkname):
490 os.remove(siglinkname)
491 os.symlink(sigfile_path, siglinkname)
493 if common.options.pretty:
494 output = doc.toprettyxml(encoding='utf-8')
496 output = doc.toxml(encoding='utf-8')
498 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
501 if 'repo_keyalias' in common.config:
503 if common.options.nosign:
504 logging.info("Creating unsigned index in preparation for signing")
506 logging.info("Creating signed index with this key (SHA256):")
507 logging.info("%s" % repo_pubkey_fingerprint)
509 # Create a jar of the index...
510 jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
511 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
512 if p.returncode != 0:
513 raise FDroidException("Failed to create {0}".format(jar_output))
516 signed = os.path.join(repodir, 'index.jar')
517 if common.options.nosign:
518 # Remove old signed index if not signing
519 if os.path.exists(signed):
522 signindex.config = common.config
523 signindex.sign_jar(signed)
525 # Copy the repo icon into the repo directory...
526 icon_dir = os.path.join(repodir, 'icons')
527 iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
528 shutil.copyfile(common.config['repo_icon'], iconfilename)
531 def extract_pubkey():
533 Extracts and returns the repository's public key from the keystore.
534 :return: public key in hex, repository fingerprint
536 if 'repo_pubkey' in common.config:
537 pubkey = unhexlify(common.config['repo_pubkey'])
539 env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
540 p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
541 '-alias', common.config['repo_keyalias'],
542 '-keystore', common.config['keystore'],
543 '-storepass:env', 'FDROID_KEY_STORE_PASS']
544 + common.config['smartcardoptions'],
545 envs=env_vars, output=False, stderr_to_stdout=False)
546 if p.returncode != 0 or len(p.output) < 20:
547 msg = "Failed to get repo pubkey!"
548 if common.config['keystore'] == 'NONE':
549 msg += ' Is your crypto smartcard plugged in?'
550 raise FDroidException(msg)
552 repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
553 return hexlify(pubkey), repo_pubkey_fingerprint
556 def get_mirror_service_urls(url):
557 '''Get direct URLs from git service for use by fdroidclient
559 Via 'servergitmirrors', fdroidserver can create and push a mirror
560 to certain well known git services like gitlab or github. This
561 will always use the 'master' branch since that is the default
562 branch in git. The files are then accessible via alternate URLs,
563 where they are served in their raw format via a CDN rather than
567 if url.startswith('git@'):
568 url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
570 segments = url.split("/")
572 if segments[4].endswith('.git'):
573 segments[4] = segments[4][:-4]
575 hostname = segments[2]
582 if hostname == "github.com":
583 # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
584 segments[2] = "raw.githubusercontent.com"
585 segments.extend([branch, folder])
586 urls.append('/'.join(segments))
587 elif hostname == "gitlab.com":
588 # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
589 gitlab_raw = segments + ['raw', branch, folder]
590 urls.append('/'.join(gitlab_raw))
591 # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
592 gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
593 urls.append('/'.join(gitlab_pages))
599 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
601 Downloads the repository index from the given :param url_str
602 and verifies the repository's fingerprint if :param verify_fingerprint is not False.
604 :raises: VerificationException() if the repository could not be verified
606 :return: A tuple consisting of:
607 - The index in JSON format or None if the index did not change
608 - The new eTag as returned by the HTTP request
610 url = urllib.parse.urlsplit(url_str)
613 if verify_fingerprint:
614 query = urllib.parse.parse_qs(url.query)
615 if 'fingerprint' not in query:
616 raise VerificationException("No fingerprint in URL.")
617 fingerprint = query['fingerprint'][0]
619 url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
620 download, new_etag = net.http_get(url.geturl(), etag)
623 return None, new_etag
625 with tempfile.NamedTemporaryFile() as fp:
626 # write and open JAR file
628 jar = zipfile.ZipFile(fp)
630 # verify that the JAR signature is valid
631 verify_jar_signature(fp.name)
633 # get public key and its fingerprint from JAR
634 public_key, public_key_fingerprint = get_public_key_from_jar(jar)
636 # compare the fingerprint if verify_fingerprint is True
637 if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
638 raise VerificationException("The repository's fingerprint does not match.")
640 # load repository index from JSON
641 index = json.loads(jar.read('index-v1.json').decode("utf-8"))
642 index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
643 index["repo"]["fingerprint"] = public_key_fingerprint
645 # turn the apps into App objects
646 index["apps"] = [metadata.App(app) for app in index["apps"]]
648 return index, new_etag
651 def verify_jar_signature(file):
653 Verifies the signature of a given JAR file.
655 :raises: VerificationException() if the JAR's signature could not be verified
657 if not common.verify_apk_signature(file, jar=True):
658 raise VerificationException("The repository's index could not be verified.")
661 def get_public_key_from_jar(jar):
663 Get the public key and its fingerprint from a JAR file.
665 :raises: VerificationException() if the JAR was not signed exactly once
667 :param jar: a zipfile.ZipFile object
668 :return: the public key from the jar and its fingerprint
670 # extract certificate from jar
671 certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
673 raise VerificationException("Found no signing certificates for repository.")
675 raise VerificationException("Found multiple signing certificates for repository.")
677 # extract public key from certificate
678 public_key = common.get_certificate(jar.read(certs[0]))
679 public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
681 return public_key, public_key_fingerprint