3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
5 # Copyright (C) 2016, Blue Jay Wireless
6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
8 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU Affero General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Affero General Public License for more details.
20 # You should have received a copy of the GNU Affero General Public License
21 # along with this program. If not, see <http://www.gnu.org/licenses/>.
34 from binascii import hexlify, unhexlify
35 from datetime import datetime
36 from xml.dom.minidom import Document
40 from fdroidserver import metadata, signindex, common
41 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
42 from fdroidserver.metadata import MetaDataException
45 def make(apps, sortedids, apks, repodir, archive):
46 """Generate the repo index files.
48 This requires properly initialized options and config objects.
50 :param apps: fully populated apps list
51 :param sortedids: app package IDs, sorted
52 :param apks: full populated apks list
53 :param repodir: the repo directory
54 :param archive: True if this is the archive repo, False if it's the
57 from fdroidserver.update import METADATA_VERSION
59 def _resolve_description_link(appid):
61 return "fdroid.app:" + appid, apps[appid].Name
62 raise MetaDataException("Cannot resolve app id " + appid)
65 if not common.options.nosign:
66 if 'repo_keyalias' not in common.config:
68 logging.critical("'repo_keyalias' not found in config.py!")
69 if 'keystore' not in common.config:
71 logging.critical("'keystore' not found in config.py!")
72 if 'keystorepass' not in common.config and 'keystorepassfile' not in common.config:
74 logging.critical("'keystorepass' not found in config.py!")
75 if 'keypass' not in common.config and 'keypassfile' not in common.config:
77 logging.critical("'keypass' not found in config.py!")
78 if not os.path.exists(common.config['keystore']):
80 logging.critical("'" + common.config['keystore'] + "' does not exist!")
82 logging.warning("`fdroid update` requires a signing key, you can create one using:")
83 logging.warning("\tfdroid update --create-key")
86 repodict = collections.OrderedDict()
87 repodict['timestamp'] = datetime.utcnow()
88 repodict['version'] = METADATA_VERSION
90 if common.config['repo_maxage'] != 0:
91 repodict['maxage'] = common.config['repo_maxage']
94 repodict['name'] = common.config['archive_name']
95 repodict['icon'] = os.path.basename(common.config['archive_icon'])
96 repodict['address'] = common.config['archive_url']
97 repodict['description'] = common.config['archive_description']
98 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
100 repodict['name'] = common.config['repo_name']
101 repodict['icon'] = os.path.basename(common.config['repo_icon'])
102 repodict['address'] = common.config['repo_url']
103 repodict['description'] = common.config['repo_description']
104 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
106 mirrorcheckfailed = False
108 for mirror in sorted(common.config.get('mirrors', [])):
109 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
110 if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
111 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
112 mirrorcheckfailed = True
113 # must end with / or urljoin strips a whole path segment
114 if mirror.endswith('/'):
115 mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
117 mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
118 for mirror in common.config.get('servergitmirrors', []):
119 mirror = get_raw_mirror(mirror)
120 if mirror is not None:
121 mirrors.append(mirror + '/')
122 if mirrorcheckfailed:
125 repodict['mirrors'] = mirrors
127 appsWithPackages = collections.OrderedDict()
128 for packageName in sortedids:
129 app = apps[packageName]
133 # only include apps with packages
135 if apk['packageName'] == packageName:
136 newapp = copy.copy(app) # update wiki needs unmodified description
137 newapp['Description'] = metadata.description_html(app['Description'],
138 _resolve_description_link)
139 appsWithPackages[packageName] = newapp
142 requestsdict = collections.OrderedDict()
143 for command in ('install', 'uninstall'):
145 key = command + '_list'
146 if key in common.config:
147 if isinstance(common.config[key], str):
148 packageNames = [common.config[key]]
149 elif all(isinstance(item, str) for item in common.config[key]):
150 packageNames = common.config[key]
152 raise TypeError('only accepts strings, lists, and tuples')
153 requestsdict[command] = packageNames
155 make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
156 make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
159 def make_v1(apps, packages, repodir, repodict, requestsdict):
161 def _index_encoder_default(obj):
162 if isinstance(obj, set):
164 if isinstance(obj, datetime):
165 return int(obj.timestamp() * 1000) # Java expects milliseconds
166 raise TypeError(repr(obj) + " is not JSON serializable")
168 output = collections.OrderedDict()
169 output['repo'] = repodict
170 output['requests'] = requestsdict
173 output['apps'] = appslist
174 for appid, appdict in apps.items():
175 d = collections.OrderedDict()
177 for k, v in sorted(appdict.items()):
180 if k in ('builds', 'comments', 'metadatapath',
181 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
182 'Provides', 'Repo', 'RepoType', 'RequiresRoot',
183 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
184 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
187 # name things after the App class fields in fdroidclient
190 elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name
191 k = 'suggestedVersionCode'
192 elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name
193 k = 'suggestedVersionName'
194 elif k == 'AutoName':
195 if 'Name' not in apps[appid]:
199 k = k[:1].lower() + k[1:]
202 output_packages = collections.OrderedDict()
203 output['packages'] = output_packages
204 for package in packages:
205 packageName = package['packageName']
206 if packageName in output_packages:
207 packagelist = output_packages[packageName]
210 output_packages[packageName] = packagelist
211 d = collections.OrderedDict()
212 packagelist.append(d)
213 for k, v in sorted(package.items()):
216 if k in ('icon', 'icons', 'icons_src', 'name', ):
220 json_name = 'index-v1.json'
221 index_file = os.path.join(repodir, json_name)
222 with open(index_file, 'w') as fp:
223 json.dump(output, fp, default=_index_encoder_default)
225 if common.options.nosign:
226 logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
228 signindex.config = common.config
229 signindex.sign_index_v1(repodir, json_name)
232 def make_v0(apps, apks, repodir, repodict, requestsdict):
234 aka index.jar aka index.xml
239 def addElement(name, value, doc, parent):
240 el = doc.createElement(name)
241 el.appendChild(doc.createTextNode(value))
242 parent.appendChild(el)
244 def addElementNonEmpty(name, value, doc, parent):
247 addElement(name, value, doc, parent)
249 def addElementIfInApk(name, apk, key, doc, parent):
252 value = str(apk[key])
253 addElement(name, value, doc, parent)
255 def addElementCDATA(name, value, doc, parent):
256 el = doc.createElement(name)
257 el.appendChild(doc.createCDATASection(value))
258 parent.appendChild(el)
260 root = doc.createElement("fdroid")
261 doc.appendChild(root)
263 repoel = doc.createElement("repo")
265 repoel.setAttribute("name", repodict['name'])
266 if 'maxage' in repodict:
267 repoel.setAttribute("maxage", str(repodict['maxage']))
268 repoel.setAttribute("icon", os.path.basename(repodict['icon']))
269 repoel.setAttribute("url", repodict['address'])
270 addElement('description', repodict['description'], doc, repoel)
271 for mirror in repodict.get('mirrors', []):
272 addElement('mirror', mirror, doc, repoel)
274 repoel.setAttribute("version", str(repodict['version']))
275 repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
277 pubkey, repo_pubkey_fingerprint = extract_pubkey()
278 repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
279 root.appendChild(repoel)
281 for command in ('install', 'uninstall'):
282 for packageName in requestsdict[command]:
283 element = doc.createElement(command)
284 root.appendChild(element)
285 element.setAttribute('packageName', packageName)
287 for appid, appdict in apps.items():
288 app = metadata.App(appdict)
290 if app.Disabled is not None:
293 # Get a list of the apks for this app...
296 if apk['packageName'] == appid:
299 if len(apklist) == 0:
302 apel = doc.createElement("application")
303 apel.setAttribute("id", app.id)
304 root.appendChild(apel)
306 addElement('id', app.id, doc, apel)
308 addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
310 addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
311 addElement('name', app.Name, doc, apel)
312 addElement('summary', app.Summary, doc, apel)
314 addElement('icon', app.icon, doc, apel)
316 if app.get('Description'):
317 description = app.Description
319 description = '<p>No description available</p>'
320 addElement('desc', description, doc, apel)
321 addElement('license', app.License, doc, apel)
323 addElement('categories', ','.join(app.Categories), doc, apel)
324 # We put the first (primary) category in LAST, which will have
325 # the desired effect of making clients that only understand one
326 # category see that one.
327 addElement('category', app.Categories[0], doc, apel)
328 addElement('web', app.WebSite, doc, apel)
329 addElement('source', app.SourceCode, doc, apel)
330 addElement('tracker', app.IssueTracker, doc, apel)
331 addElementNonEmpty('changelog', app.Changelog, doc, apel)
332 addElementNonEmpty('author', app.AuthorName, doc, apel)
333 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
334 addElementNonEmpty('donate', app.Donate, doc, apel)
335 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
336 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
337 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
339 # These elements actually refer to the current version (i.e. which
340 # one is recommended. They are historically mis-named, and need
341 # changing, but stay like this for now to support existing clients.
342 addElement('marketversion', app.CurrentVersion, doc, apel)
343 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
346 pv = app.Provides.split(',')
347 addElementNonEmpty('provides', ','.join(pv), doc, apel)
349 addElement('requirements', 'root', doc, apel)
351 # Sort the apk list into version order, just so the web site
352 # doesn't have to do any work by default...
353 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
355 if 'antiFeatures' in apklist[0]:
356 app.AntiFeatures.extend(apklist[0]['antiFeatures'])
358 addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
360 # Check for duplicates - they will make the client unhappy...
361 for i in range(len(apklist) - 1):
362 if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
363 logging.critical("duplicate versions: '%s' - '%s'" % (
364 apklist[i]['apkName'], apklist[i + 1]['apkName']))
367 current_version_code = 0
368 current_version_file = None
370 file_extension = common.get_file_extension(apk['apkName'])
371 # find the APK for the "Current Version"
372 if current_version_code < apk['versionCode']:
373 current_version_code = apk['versionCode']
374 if current_version_code < int(app.CurrentVersionCode):
375 current_version_file = apk['apkName']
377 apkel = doc.createElement("package")
378 apel.appendChild(apkel)
379 addElement('version', apk['versionName'], doc, apkel)
380 addElement('versioncode', str(apk['versionCode']), doc, apkel)
381 addElement('apkname', apk['apkName'], doc, apkel)
382 addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
384 hashel = doc.createElement("hash")
385 hashel.setAttribute('type', 'sha256')
386 hashel.appendChild(doc.createTextNode(apk['hash']))
387 apkel.appendChild(hashel)
389 addElement('size', str(apk['size']), doc, apkel)
390 addElementIfInApk('sdkver', apk,
391 'minSdkVersion', doc, apkel)
392 addElementIfInApk('targetSdkVersion', apk,
393 'targetSdkVersion', doc, apkel)
394 addElementIfInApk('maxsdkver', apk,
395 'maxSdkVersion', doc, apkel)
396 addElementIfInApk('obbMainFile', apk,
397 'obbMainFile', doc, apkel)
398 addElementIfInApk('obbMainFileSha256', apk,
399 'obbMainFileSha256', doc, apkel)
400 addElementIfInApk('obbPatchFile', apk,
401 'obbPatchFile', doc, apkel)
402 addElementIfInApk('obbPatchFileSha256', apk,
403 'obbPatchFileSha256', doc, apkel)
405 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
407 if file_extension == 'apk': # sig is required for APKs, but only APKs
408 addElement('sig', apk['sig'], doc, apkel)
410 old_permissions = set()
411 sorted_permissions = sorted(apk['uses-permission'])
412 for perm in sorted_permissions:
413 perm_name = perm.name
414 if perm_name.startswith("android.permission."):
415 perm_name = perm_name[19:]
416 old_permissions.add(perm_name)
417 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
419 for permission in sorted_permissions:
420 permel = doc.createElement('uses-permission')
421 permel.setAttribute('name', permission.name)
422 if permission.maxSdkVersion is not None:
423 permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
424 apkel.appendChild(permel)
425 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
426 permel = doc.createElement('uses-permission-sdk-23')
427 permel.setAttribute('name', permission_sdk_23.name)
428 if permission_sdk_23.maxSdkVersion is not None:
429 permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
430 apkel.appendChild(permel)
431 if 'nativecode' in apk:
432 addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
433 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
435 if current_version_file is not None \
436 and common.config['make_current_version_link'] \
437 and repodir == 'repo': # only create these
438 namefield = common.config['current_version_name_source']
439 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
440 apklinkname = sanitized_name + '.apk'
441 current_version_path = os.path.join(repodir, current_version_file)
442 if os.path.islink(apklinkname):
443 os.remove(apklinkname)
444 os.symlink(current_version_path, apklinkname)
445 # also symlink gpg signature, if it exists
446 for extension in ('.asc', '.sig'):
447 sigfile_path = current_version_path + extension
448 if os.path.exists(sigfile_path):
449 siglinkname = apklinkname + extension
450 if os.path.islink(siglinkname):
451 os.remove(siglinkname)
452 os.symlink(sigfile_path, siglinkname)
454 if common.options.pretty:
455 output = doc.toprettyxml(encoding='utf-8')
457 output = doc.toxml(encoding='utf-8')
459 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
462 if 'repo_keyalias' in common.config:
464 if common.options.nosign:
465 logging.info("Creating unsigned index in preparation for signing")
467 logging.info("Creating signed index with this key (SHA256):")
468 logging.info("%s" % repo_pubkey_fingerprint)
470 # Create a jar of the index...
471 jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
472 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
473 if p.returncode != 0:
474 logging.critical("Failed to create {0}".format(jar_output))
478 signed = os.path.join(repodir, 'index.jar')
479 if common.options.nosign:
480 # Remove old signed index if not signing
481 if os.path.exists(signed):
484 signindex.config = common.config
485 signindex.sign_jar(signed)
487 # Copy the repo icon into the repo directory...
488 icon_dir = os.path.join(repodir, 'icons')
489 iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
490 shutil.copyfile(common.config['repo_icon'], iconfilename)
493 def extract_pubkey():
495 Extracts and returns the repository's public key from the keystore.
496 :return: public key in hex, repository fingerprint
498 if 'repo_pubkey' in common.config:
499 pubkey = unhexlify(common.config['repo_pubkey'])
501 p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
502 '-alias', common.config['repo_keyalias'],
503 '-keystore', common.config['keystore'],
504 '-storepass:file', common.config['keystorepassfile']]
505 + common.config['smartcardoptions'],
506 output=False, stderr_to_stdout=False)
507 if p.returncode != 0 or len(p.output) < 20:
508 msg = "Failed to get repo pubkey!"
509 if common.config['keystore'] == 'NONE':
510 msg += ' Is your crypto smartcard plugged in?'
511 logging.critical(msg)
514 repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
515 return hexlify(pubkey), repo_pubkey_fingerprint
518 # Get raw URL from git service for mirroring
519 def get_raw_mirror(url):
520 # Divide urls in parts
526 # fdroidserver will use always 'master' branch for git-mirroring
530 if hostname == "github.com":
531 # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
532 url[2] = "raw.githubusercontent.com"
533 url.extend([branch, folder])
534 elif hostname == "gitlab.com":
535 # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
536 url.extend(["raw", branch, folder])
544 class VerificationException(Exception):
548 def download_repo_index(url_str, verify_fingerprint=True):
550 Downloads the repository index from the given :param url_str
551 and verifies the repository's fingerprint if :param verify_fingerprint is not False.
553 :raises: VerificationException() if the repository could not be verified
555 :return: The index in JSON format.
557 url = urllib.parse.urlsplit(url_str)
560 if verify_fingerprint:
561 query = urllib.parse.parse_qs(url.query)
562 if 'fingerprint' not in query:
563 raise VerificationException("No fingerprint in URL.")
564 fingerprint = query['fingerprint'][0]
566 url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
567 r = requests.get(url.geturl())
569 with tempfile.NamedTemporaryFile() as fp:
570 # write and open JAR file
572 jar = zipfile.ZipFile(fp)
574 # verify that the JAR signature is valid
575 verify_jar_signature(fp.name)
577 # get public key and its fingerprint from JAR
578 public_key, public_key_fingerprint = get_public_key_from_jar(jar)
580 # compare the fingerprint if verify_fingerprint is True
581 if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
582 raise VerificationException("The repository's fingerprint does not match.")
584 # load repository index from JSON
585 index = json.loads(jar.read('index-v1.json').decode("utf-8"))
586 index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
587 index["repo"]["fingerprint"] = public_key_fingerprint
589 # turn the apps into App objects
590 index["apps"] = [metadata.App(app) for app in index["apps"]]
595 def verify_jar_signature(file):
597 Verifies the signature of a given JAR file.
599 :raises: VerificationException() if the JAR's signature could not be verified
601 if not common.verify_apk_signature(file, jar=True):
602 raise VerificationException("The repository's index could not be verified.")
605 def get_public_key_from_jar(jar):
607 Get the public key and its fingerprint from a JAR file.
609 :raises: VerificationException() if the JAR was not signed exactly once
611 :param jar: a zipfile.ZipFile object
612 :return: the public key from the jar and its fingerprint
614 # extract certificate from jar
615 certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
617 raise VerificationException("Found no signing certificates for repository.")
619 raise VerificationException("Found multiple signing certificates for repository.")
621 # extract public key from certificate
622 public_key = common.get_certificate(jar.read(certs[0]))
623 public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
625 return public_key, public_key_fingerprint