3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
5 # Copyright (C) 2016, Blue Jay Wireless
6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
8 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU Affero General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Affero General Public License for more details.
20 # You should have received a copy of the GNU Affero General Public License
21 # along with this program. If not, see <http://www.gnu.org/licenses/>.
34 from binascii import hexlify, unhexlify
35 from datetime import datetime
36 from xml.dom.minidom import Document
38 from fdroidserver import metadata, signindex, common, net
39 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
40 from fdroidserver.metadata import MetaDataException
43 def make(apps, sortedids, apks, repodir, archive):
44 """Generate the repo index files.
46 This requires properly initialized options and config objects.
48 :param apps: fully populated apps list
49 :param sortedids: app package IDs, sorted
50 :param apks: full populated apks list
51 :param repodir: the repo directory
52 :param archive: True if this is the archive repo, False if it's the
55 from fdroidserver.update import METADATA_VERSION
57 def _resolve_description_link(appid):
59 return "fdroid.app:" + appid, apps[appid].Name
60 raise MetaDataException("Cannot resolve app id " + appid)
63 if not common.options.nosign:
64 if 'repo_keyalias' not in common.config:
66 logging.critical("'repo_keyalias' not found in config.py!")
67 if 'keystore' not in common.config:
69 logging.critical("'keystore' not found in config.py!")
70 if 'keystorepass' not in common.config:
72 logging.critical("'keystorepass' not found in config.py!")
73 if 'keypass' not in common.config:
75 logging.critical("'keypass' not found in config.py!")
76 if not os.path.exists(common.config['keystore']):
78 logging.critical("'" + common.config['keystore'] + "' does not exist!")
80 logging.warning("`fdroid update` requires a signing key, you can create one using:")
81 logging.warning("\tfdroid update --create-key")
84 repodict = collections.OrderedDict()
85 repodict['timestamp'] = datetime.utcnow()
86 repodict['version'] = METADATA_VERSION
88 if common.config['repo_maxage'] != 0:
89 repodict['maxage'] = common.config['repo_maxage']
92 repodict['name'] = common.config['archive_name']
93 repodict['icon'] = os.path.basename(common.config['archive_icon'])
94 repodict['address'] = common.config['archive_url']
95 repodict['description'] = common.config['archive_description']
96 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
98 repodict['name'] = common.config['repo_name']
99 repodict['icon'] = os.path.basename(common.config['repo_icon'])
100 repodict['address'] = common.config['repo_url']
101 repodict['description'] = common.config['repo_description']
102 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
104 mirrorcheckfailed = False
106 for mirror in sorted(common.config.get('mirrors', [])):
107 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
108 if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
109 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
110 mirrorcheckfailed = True
111 # must end with / or urljoin strips a whole path segment
112 if mirror.endswith('/'):
113 mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
115 mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
116 for mirror in common.config.get('servergitmirrors', []):
117 mirror = get_raw_mirror(mirror)
118 if mirror is not None:
119 mirrors.append(mirror + '/')
120 if mirrorcheckfailed:
123 repodict['mirrors'] = mirrors
125 appsWithPackages = collections.OrderedDict()
126 for packageName in sortedids:
127 app = apps[packageName]
131 # only include apps with packages
133 if apk['packageName'] == packageName:
134 newapp = copy.copy(app) # update wiki needs unmodified description
135 newapp['Description'] = metadata.description_html(app['Description'],
136 _resolve_description_link)
137 appsWithPackages[packageName] = newapp
140 requestsdict = collections.OrderedDict()
141 for command in ('install', 'uninstall'):
143 key = command + '_list'
144 if key in common.config:
145 if isinstance(common.config[key], str):
146 packageNames = [common.config[key]]
147 elif all(isinstance(item, str) for item in common.config[key]):
148 packageNames = common.config[key]
150 raise TypeError('only accepts strings, lists, and tuples')
151 requestsdict[command] = packageNames
153 make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
154 make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
157 def make_v1(apps, packages, repodir, repodict, requestsdict):
159 def _index_encoder_default(obj):
160 if isinstance(obj, set):
162 if isinstance(obj, datetime):
163 return int(obj.timestamp() * 1000) # Java expects milliseconds
164 raise TypeError(repr(obj) + " is not JSON serializable")
166 output = collections.OrderedDict()
167 output['repo'] = repodict
168 output['requests'] = requestsdict
171 output['apps'] = appslist
172 for appid, appdict in apps.items():
173 d = collections.OrderedDict()
175 for k, v in sorted(appdict.items()):
178 if k in ('builds', 'comments', 'metadatapath',
179 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
180 'Provides', 'Repo', 'RepoType', 'RequiresRoot',
181 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
182 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
185 # name things after the App class fields in fdroidclient
188 elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name
189 k = 'suggestedVersionCode'
190 elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name
191 k = 'suggestedVersionName'
192 elif k == 'AutoName':
193 if 'Name' not in apps[appid]:
197 k = k[:1].lower() + k[1:]
200 output_packages = collections.OrderedDict()
201 output['packages'] = output_packages
202 for package in packages:
203 packageName = package['packageName']
204 if packageName in output_packages:
205 packagelist = output_packages[packageName]
208 output_packages[packageName] = packagelist
209 d = collections.OrderedDict()
210 packagelist.append(d)
211 for k, v in sorted(package.items()):
214 if k in ('icon', 'icons', 'icons_src', 'name', ):
218 json_name = 'index-v1.json'
219 index_file = os.path.join(repodir, json_name)
220 with open(index_file, 'w') as fp:
221 if common.options.pretty:
222 json.dump(output, fp, default=_index_encoder_default, indent=2)
224 json.dump(output, fp, default=_index_encoder_default)
226 if common.options.nosign:
227 logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
229 signindex.config = common.config
230 signindex.sign_index_v1(repodir, json_name)
233 def make_v0(apps, apks, repodir, repodict, requestsdict):
235 aka index.jar aka index.xml
240 def addElement(name, value, doc, parent):
241 el = doc.createElement(name)
242 el.appendChild(doc.createTextNode(value))
243 parent.appendChild(el)
245 def addElementNonEmpty(name, value, doc, parent):
248 addElement(name, value, doc, parent)
250 def addElementIfInApk(name, apk, key, doc, parent):
253 value = str(apk[key])
254 addElement(name, value, doc, parent)
256 def addElementCDATA(name, value, doc, parent):
257 el = doc.createElement(name)
258 el.appendChild(doc.createCDATASection(value))
259 parent.appendChild(el)
261 root = doc.createElement("fdroid")
262 doc.appendChild(root)
264 repoel = doc.createElement("repo")
266 repoel.setAttribute("name", repodict['name'])
267 if 'maxage' in repodict:
268 repoel.setAttribute("maxage", str(repodict['maxage']))
269 repoel.setAttribute("icon", os.path.basename(repodict['icon']))
270 repoel.setAttribute("url", repodict['address'])
271 addElement('description', repodict['description'], doc, repoel)
272 for mirror in repodict.get('mirrors', []):
273 addElement('mirror', mirror, doc, repoel)
275 repoel.setAttribute("version", str(repodict['version']))
276 repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
278 pubkey, repo_pubkey_fingerprint = extract_pubkey()
279 repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
280 root.appendChild(repoel)
282 for command in ('install', 'uninstall'):
283 for packageName in requestsdict[command]:
284 element = doc.createElement(command)
285 root.appendChild(element)
286 element.setAttribute('packageName', packageName)
288 for appid, appdict in apps.items():
289 app = metadata.App(appdict)
291 if app.Disabled is not None:
294 # Get a list of the apks for this app...
297 if apk['packageName'] == appid:
300 if len(apklist) == 0:
303 apel = doc.createElement("application")
304 apel.setAttribute("id", app.id)
305 root.appendChild(apel)
307 addElement('id', app.id, doc, apel)
309 addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
311 addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
312 addElement('name', app.Name, doc, apel)
313 addElement('summary', app.Summary, doc, apel)
315 addElement('icon', app.icon, doc, apel)
317 if app.get('Description'):
318 description = app.Description
320 description = '<p>No description available</p>'
321 addElement('desc', description, doc, apel)
322 addElement('license', app.License, doc, apel)
324 addElement('categories', ','.join(app.Categories), doc, apel)
325 # We put the first (primary) category in LAST, which will have
326 # the desired effect of making clients that only understand one
327 # category see that one.
328 addElement('category', app.Categories[0], doc, apel)
329 addElement('web', app.WebSite, doc, apel)
330 addElement('source', app.SourceCode, doc, apel)
331 addElement('tracker', app.IssueTracker, doc, apel)
332 addElementNonEmpty('changelog', app.Changelog, doc, apel)
333 addElementNonEmpty('author', app.AuthorName, doc, apel)
334 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
335 addElementNonEmpty('donate', app.Donate, doc, apel)
336 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
337 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
338 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
340 # These elements actually refer to the current version (i.e. which
341 # one is recommended. They are historically mis-named, and need
342 # changing, but stay like this for now to support existing clients.
343 addElement('marketversion', app.CurrentVersion, doc, apel)
344 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
347 pv = app.Provides.split(',')
348 addElementNonEmpty('provides', ','.join(pv), doc, apel)
350 addElement('requirements', 'root', doc, apel)
352 # Sort the apk list into version order, just so the web site
353 # doesn't have to do any work by default...
354 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
356 if 'antiFeatures' in apklist[0]:
357 app.AntiFeatures.extend(apklist[0]['antiFeatures'])
359 addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
361 # Check for duplicates - they will make the client unhappy...
362 for i in range(len(apklist) - 1):
363 if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
364 logging.critical("duplicate versions: '%s' - '%s'" % (
365 apklist[i]['apkName'], apklist[i + 1]['apkName']))
368 current_version_code = 0
369 current_version_file = None
371 file_extension = common.get_file_extension(apk['apkName'])
372 # find the APK for the "Current Version"
373 if current_version_code < apk['versionCode']:
374 current_version_code = apk['versionCode']
375 if current_version_code < int(app.CurrentVersionCode):
376 current_version_file = apk['apkName']
378 apkel = doc.createElement("package")
379 apel.appendChild(apkel)
380 addElement('version', apk['versionName'], doc, apkel)
381 addElement('versioncode', str(apk['versionCode']), doc, apkel)
382 addElement('apkname', apk['apkName'], doc, apkel)
383 addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
385 hashel = doc.createElement("hash")
386 hashel.setAttribute('type', 'sha256')
387 hashel.appendChild(doc.createTextNode(apk['hash']))
388 apkel.appendChild(hashel)
390 addElement('size', str(apk['size']), doc, apkel)
391 addElementIfInApk('sdkver', apk,
392 'minSdkVersion', doc, apkel)
393 addElementIfInApk('targetSdkVersion', apk,
394 'targetSdkVersion', doc, apkel)
395 addElementIfInApk('maxsdkver', apk,
396 'maxSdkVersion', doc, apkel)
397 addElementIfInApk('obbMainFile', apk,
398 'obbMainFile', doc, apkel)
399 addElementIfInApk('obbMainFileSha256', apk,
400 'obbMainFileSha256', doc, apkel)
401 addElementIfInApk('obbPatchFile', apk,
402 'obbPatchFile', doc, apkel)
403 addElementIfInApk('obbPatchFileSha256', apk,
404 'obbPatchFileSha256', doc, apkel)
406 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
408 if file_extension == 'apk': # sig is required for APKs, but only APKs
409 addElement('sig', apk['sig'], doc, apkel)
411 old_permissions = set()
412 sorted_permissions = sorted(apk['uses-permission'])
413 for perm in sorted_permissions:
414 perm_name = perm.name
415 if perm_name.startswith("android.permission."):
416 perm_name = perm_name[19:]
417 old_permissions.add(perm_name)
418 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
420 for permission in sorted_permissions:
421 permel = doc.createElement('uses-permission')
422 permel.setAttribute('name', permission.name)
423 if permission.maxSdkVersion is not None:
424 permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
425 apkel.appendChild(permel)
426 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
427 permel = doc.createElement('uses-permission-sdk-23')
428 permel.setAttribute('name', permission_sdk_23.name)
429 if permission_sdk_23.maxSdkVersion is not None:
430 permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
431 apkel.appendChild(permel)
432 if 'nativecode' in apk:
433 addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
434 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
436 if current_version_file is not None \
437 and common.config['make_current_version_link'] \
438 and repodir == 'repo': # only create these
439 namefield = common.config['current_version_name_source']
440 sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
441 apklinkname = sanitized_name + b'.apk'
442 current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
443 if os.path.islink(apklinkname):
444 os.remove(apklinkname)
445 os.symlink(current_version_path, apklinkname)
446 # also symlink gpg signature, if it exists
447 for extension in (b'.asc', b'.sig'):
448 sigfile_path = current_version_path + extension
449 if os.path.exists(sigfile_path):
450 siglinkname = apklinkname + extension
451 if os.path.islink(siglinkname):
452 os.remove(siglinkname)
453 os.symlink(sigfile_path, siglinkname)
455 if common.options.pretty:
456 output = doc.toprettyxml(encoding='utf-8')
458 output = doc.toxml(encoding='utf-8')
460 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
463 if 'repo_keyalias' in common.config:
465 if common.options.nosign:
466 logging.info("Creating unsigned index in preparation for signing")
468 logging.info("Creating signed index with this key (SHA256):")
469 logging.info("%s" % repo_pubkey_fingerprint)
471 # Create a jar of the index...
472 jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
473 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
474 if p.returncode != 0:
475 logging.critical("Failed to create {0}".format(jar_output))
479 signed = os.path.join(repodir, 'index.jar')
480 if common.options.nosign:
481 # Remove old signed index if not signing
482 if os.path.exists(signed):
485 signindex.config = common.config
486 signindex.sign_jar(signed)
488 # Copy the repo icon into the repo directory...
489 icon_dir = os.path.join(repodir, 'icons')
490 iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
491 shutil.copyfile(common.config['repo_icon'], iconfilename)
494 def extract_pubkey():
496 Extracts and returns the repository's public key from the keystore.
497 :return: public key in hex, repository fingerprint
499 if 'repo_pubkey' in common.config:
500 pubkey = unhexlify(common.config['repo_pubkey'])
502 env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
503 p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
504 '-alias', common.config['repo_keyalias'],
505 '-keystore', common.config['keystore'],
506 '-storepass:env', 'FDROID_KEY_STORE_PASS']
507 + common.config['smartcardoptions'],
508 envs=env_vars, output=False, stderr_to_stdout=False)
509 if p.returncode != 0 or len(p.output) < 20:
510 msg = "Failed to get repo pubkey!"
511 if common.config['keystore'] == 'NONE':
512 msg += ' Is your crypto smartcard plugged in?'
513 logging.critical(msg)
516 repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
517 return hexlify(pubkey), repo_pubkey_fingerprint
520 def get_raw_mirror(url):
521 '''Get direct URL from git service for use by fdroidclient
523 Via 'servergitmirrors', fdroidserver can create and push a mirror
524 to certain well known git services like gitlab or github. This
525 will always use the 'master' branch since that is the default
530 if url.startswith('git@'):
531 url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
533 segments = url.split("/")
534 hostname = segments[2]
538 if hostname == "github.com":
539 # Github like RAW segments "https://raw.githubusercontent.com/user/repo/master/fdroid"
540 segments[2] = "raw.githubusercontent.com"
541 segments.extend([branch, folder])
542 elif hostname == "gitlab.com":
543 # Gitlab like RAW segments "https://gitlab.com/user/repo/raw/master/fdroid"
544 segments.extend(["raw", branch, folder])
548 if segments[4].endswith('.git'):
549 segments[4] = segments[4][:-4]
551 return '/'.join(segments)
554 class VerificationException(Exception):
558 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
560 Downloads the repository index from the given :param url_str
561 and verifies the repository's fingerprint if :param verify_fingerprint is not False.
563 :raises: VerificationException() if the repository could not be verified
565 :return: A tuple consisting of:
566 - The index in JSON format or None if the index did not change
567 - The new eTag as returned by the HTTP request
569 url = urllib.parse.urlsplit(url_str)
572 if verify_fingerprint:
573 query = urllib.parse.parse_qs(url.query)
574 if 'fingerprint' not in query:
575 raise VerificationException("No fingerprint in URL.")
576 fingerprint = query['fingerprint'][0]
578 url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
579 download, new_etag = net.http_get(url.geturl(), etag)
582 return None, new_etag
584 with tempfile.NamedTemporaryFile() as fp:
585 # write and open JAR file
587 jar = zipfile.ZipFile(fp)
589 # verify that the JAR signature is valid
590 verify_jar_signature(fp.name)
592 # get public key and its fingerprint from JAR
593 public_key, public_key_fingerprint = get_public_key_from_jar(jar)
595 # compare the fingerprint if verify_fingerprint is True
596 if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
597 raise VerificationException("The repository's fingerprint does not match.")
599 # load repository index from JSON
600 index = json.loads(jar.read('index-v1.json').decode("utf-8"))
601 index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
602 index["repo"]["fingerprint"] = public_key_fingerprint
604 # turn the apps into App objects
605 index["apps"] = [metadata.App(app) for app in index["apps"]]
607 return index, new_etag
610 def verify_jar_signature(file):
612 Verifies the signature of a given JAR file.
614 :raises: VerificationException() if the JAR's signature could not be verified
616 if not common.verify_apk_signature(file, jar=True):
617 raise VerificationException("The repository's index could not be verified.")
620 def get_public_key_from_jar(jar):
622 Get the public key and its fingerprint from a JAR file.
624 :raises: VerificationException() if the JAR was not signed exactly once
626 :param jar: a zipfile.ZipFile object
627 :return: the public key from the jar and its fingerprint
629 # extract certificate from jar
630 certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
632 raise VerificationException("Found no signing certificates for repository.")
634 raise VerificationException("Found multiple signing certificates for repository.")
636 # extract public key from certificate
637 public_key = common.get_certificate(jar.read(certs[0]))
638 public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
640 return public_key, public_key_fingerprint