3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
5 # Copyright (C) 2016, Blue Jay Wireless
6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
8 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU Affero General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Affero General Public License for more details.
20 # You should have received a copy of the GNU Affero General Public License
21 # along with this program. If not, see <http://www.gnu.org/licenses/>.
34 from binascii import hexlify, unhexlify
35 from datetime import datetime
36 from xml.dom.minidom import Document
40 from fdroidserver import metadata, signindex, common
41 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
42 from fdroidserver.metadata import MetaDataException
45 def make(apps, sortedids, apks, repodir, archive):
46 """Generate the repo index files.
48 This requires properly initialized options and config objects.
50 :param apps: fully populated apps list
51 :param sortedids: app package IDs, sorted
52 :param apks: full populated apks list
53 :param repodir: the repo directory
54 :param archive: True if this is the archive repo, False if it's the
57 from fdroidserver.update import METADATA_VERSION
59 def _resolve_description_link(appid):
61 return "fdroid.app:" + appid, apps[appid].Name
62 raise MetaDataException("Cannot resolve app id " + appid)
65 if not common.options.nosign:
66 if 'repo_keyalias' not in common.config:
68 logging.critical("'repo_keyalias' not found in config.py!")
69 if 'keystore' not in common.config:
71 logging.critical("'keystore' not found in config.py!")
72 if 'keystorepass' not in common.config and 'keystorepassfile' not in common.config:
74 logging.critical("'keystorepass' not found in config.py!")
75 if 'keypass' not in common.config and 'keypassfile' not in common.config:
77 logging.critical("'keypass' not found in config.py!")
78 if not os.path.exists(common.config['keystore']):
80 logging.critical("'" + common.config['keystore'] + "' does not exist!")
82 logging.warning("`fdroid update` requires a signing key, you can create one using:")
83 logging.warning("\tfdroid update --create-key")
86 repodict = collections.OrderedDict()
87 repodict['timestamp'] = datetime.utcnow()
88 repodict['version'] = METADATA_VERSION
90 if common.config['repo_maxage'] != 0:
91 repodict['maxage'] = common.config['repo_maxage']
94 repodict['name'] = common.config['archive_name']
95 repodict['icon'] = os.path.basename(common.config['archive_icon'])
96 repodict['address'] = common.config['archive_url']
97 repodict['description'] = common.config['archive_description']
98 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
100 repodict['name'] = common.config['repo_name']
101 repodict['icon'] = os.path.basename(common.config['repo_icon'])
102 repodict['address'] = common.config['repo_url']
103 repodict['description'] = common.config['repo_description']
104 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
106 mirrorcheckfailed = False
108 for mirror in sorted(common.config.get('mirrors', [])):
109 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
110 if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
111 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
112 mirrorcheckfailed = True
113 # must end with / or urljoin strips a whole path segment
114 if mirror.endswith('/'):
115 mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
117 mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
118 for mirror in common.config.get('servergitmirrors', []):
119 mirror = get_raw_mirror(mirror)
120 if mirror is not None:
121 mirrors.append(mirror + '/')
122 if mirrorcheckfailed:
125 repodict['mirrors'] = mirrors
127 appsWithPackages = collections.OrderedDict()
128 for packageName in sortedids:
129 app = apps[packageName]
133 # only include apps with packages
135 if apk['packageName'] == packageName:
136 newapp = copy.copy(app) # update wiki needs unmodified description
137 newapp['Description'] = metadata.description_html(app['Description'],
138 _resolve_description_link)
139 appsWithPackages[packageName] = newapp
142 requestsdict = collections.OrderedDict()
143 for command in ('install', 'uninstall'):
145 key = command + '_list'
146 if key in common.config:
147 if isinstance(common.config[key], str):
148 packageNames = [common.config[key]]
149 elif all(isinstance(item, str) for item in common.config[key]):
150 packageNames = common.config[key]
152 raise TypeError('only accepts strings, lists, and tuples')
153 requestsdict[command] = packageNames
155 make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
156 make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
159 def make_v1(apps, packages, repodir, repodict, requestsdict):
161 def _index_encoder_default(obj):
162 if isinstance(obj, set):
164 if isinstance(obj, datetime):
165 return int(obj.timestamp() * 1000) # Java expects milliseconds
166 raise TypeError(repr(obj) + " is not JSON serializable")
168 output = collections.OrderedDict()
169 output['repo'] = repodict
170 output['requests'] = requestsdict
173 output['apps'] = appslist
174 for appid, appdict in apps.items():
175 d = collections.OrderedDict()
177 for k, v in sorted(appdict.items()):
180 if k in ('builds', 'comments', 'metadatapath',
181 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
182 'Provides', 'Repo', 'RepoType', 'RequiresRoot',
183 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
184 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
187 # name things after the App class fields in fdroidclient
190 elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name
191 k = 'suggestedVersionCode'
192 elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name
193 k = 'suggestedVersionName'
194 elif k == 'AutoName':
195 if 'Name' not in apps[appid]:
199 k = k[:1].lower() + k[1:]
202 output_packages = collections.OrderedDict()
203 output['packages'] = output_packages
204 for package in packages:
205 packageName = package['packageName']
206 if packageName in output_packages:
207 packagelist = output_packages[packageName]
210 output_packages[packageName] = packagelist
211 d = collections.OrderedDict()
212 packagelist.append(d)
213 for k, v in sorted(package.items()):
216 if k in ('icon', 'icons', 'icons_src', 'name', ):
220 json_name = 'index-v1.json'
221 index_file = os.path.join(repodir, json_name)
222 with open(index_file, 'w') as fp:
223 if common.options.pretty:
224 json.dump(output, fp, default=_index_encoder_default, indent=2)
226 json.dump(output, fp, default=_index_encoder_default)
228 if common.options.nosign:
229 logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
231 signindex.config = common.config
232 signindex.sign_index_v1(repodir, json_name)
235 def make_v0(apps, apks, repodir, repodict, requestsdict):
237 aka index.jar aka index.xml
242 def addElement(name, value, doc, parent):
243 el = doc.createElement(name)
244 el.appendChild(doc.createTextNode(value))
245 parent.appendChild(el)
247 def addElementNonEmpty(name, value, doc, parent):
250 addElement(name, value, doc, parent)
252 def addElementIfInApk(name, apk, key, doc, parent):
255 value = str(apk[key])
256 addElement(name, value, doc, parent)
258 def addElementCDATA(name, value, doc, parent):
259 el = doc.createElement(name)
260 el.appendChild(doc.createCDATASection(value))
261 parent.appendChild(el)
263 root = doc.createElement("fdroid")
264 doc.appendChild(root)
266 repoel = doc.createElement("repo")
268 repoel.setAttribute("name", repodict['name'])
269 if 'maxage' in repodict:
270 repoel.setAttribute("maxage", str(repodict['maxage']))
271 repoel.setAttribute("icon", os.path.basename(repodict['icon']))
272 repoel.setAttribute("url", repodict['address'])
273 addElement('description', repodict['description'], doc, repoel)
274 for mirror in repodict.get('mirrors', []):
275 addElement('mirror', mirror, doc, repoel)
277 repoel.setAttribute("version", str(repodict['version']))
278 repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
280 pubkey, repo_pubkey_fingerprint = extract_pubkey()
281 repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
282 root.appendChild(repoel)
284 for command in ('install', 'uninstall'):
285 for packageName in requestsdict[command]:
286 element = doc.createElement(command)
287 root.appendChild(element)
288 element.setAttribute('packageName', packageName)
290 for appid, appdict in apps.items():
291 app = metadata.App(appdict)
293 if app.Disabled is not None:
296 # Get a list of the apks for this app...
299 if apk['packageName'] == appid:
302 if len(apklist) == 0:
305 apel = doc.createElement("application")
306 apel.setAttribute("id", app.id)
307 root.appendChild(apel)
309 addElement('id', app.id, doc, apel)
311 addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
313 addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
314 addElement('name', app.Name, doc, apel)
315 addElement('summary', app.Summary, doc, apel)
317 addElement('icon', app.icon, doc, apel)
319 if app.get('Description'):
320 description = app.Description
322 description = '<p>No description available</p>'
323 addElement('desc', description, doc, apel)
324 addElement('license', app.License, doc, apel)
326 addElement('categories', ','.join(app.Categories), doc, apel)
327 # We put the first (primary) category in LAST, which will have
328 # the desired effect of making clients that only understand one
329 # category see that one.
330 addElement('category', app.Categories[0], doc, apel)
331 addElement('web', app.WebSite, doc, apel)
332 addElement('source', app.SourceCode, doc, apel)
333 addElement('tracker', app.IssueTracker, doc, apel)
334 addElementNonEmpty('changelog', app.Changelog, doc, apel)
335 addElementNonEmpty('author', app.AuthorName, doc, apel)
336 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
337 addElementNonEmpty('donate', app.Donate, doc, apel)
338 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
339 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
340 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
342 # These elements actually refer to the current version (i.e. which
343 # one is recommended. They are historically mis-named, and need
344 # changing, but stay like this for now to support existing clients.
345 addElement('marketversion', app.CurrentVersion, doc, apel)
346 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
349 pv = app.Provides.split(',')
350 addElementNonEmpty('provides', ','.join(pv), doc, apel)
352 addElement('requirements', 'root', doc, apel)
354 # Sort the apk list into version order, just so the web site
355 # doesn't have to do any work by default...
356 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
358 if 'antiFeatures' in apklist[0]:
359 app.AntiFeatures.extend(apklist[0]['antiFeatures'])
361 addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
363 # Check for duplicates - they will make the client unhappy...
364 for i in range(len(apklist) - 1):
365 if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
366 logging.critical("duplicate versions: '%s' - '%s'" % (
367 apklist[i]['apkName'], apklist[i + 1]['apkName']))
370 current_version_code = 0
371 current_version_file = None
373 file_extension = common.get_file_extension(apk['apkName'])
374 # find the APK for the "Current Version"
375 if current_version_code < apk['versionCode']:
376 current_version_code = apk['versionCode']
377 if current_version_code < int(app.CurrentVersionCode):
378 current_version_file = apk['apkName']
380 apkel = doc.createElement("package")
381 apel.appendChild(apkel)
382 addElement('version', apk['versionName'], doc, apkel)
383 addElement('versioncode', str(apk['versionCode']), doc, apkel)
384 addElement('apkname', apk['apkName'], doc, apkel)
385 addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
387 hashel = doc.createElement("hash")
388 hashel.setAttribute('type', 'sha256')
389 hashel.appendChild(doc.createTextNode(apk['hash']))
390 apkel.appendChild(hashel)
392 addElement('size', str(apk['size']), doc, apkel)
393 addElementIfInApk('sdkver', apk,
394 'minSdkVersion', doc, apkel)
395 addElementIfInApk('targetSdkVersion', apk,
396 'targetSdkVersion', doc, apkel)
397 addElementIfInApk('maxsdkver', apk,
398 'maxSdkVersion', doc, apkel)
399 addElementIfInApk('obbMainFile', apk,
400 'obbMainFile', doc, apkel)
401 addElementIfInApk('obbMainFileSha256', apk,
402 'obbMainFileSha256', doc, apkel)
403 addElementIfInApk('obbPatchFile', apk,
404 'obbPatchFile', doc, apkel)
405 addElementIfInApk('obbPatchFileSha256', apk,
406 'obbPatchFileSha256', doc, apkel)
408 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
410 if file_extension == 'apk': # sig is required for APKs, but only APKs
411 addElement('sig', apk['sig'], doc, apkel)
413 old_permissions = set()
414 sorted_permissions = sorted(apk['uses-permission'])
415 for perm in sorted_permissions:
416 perm_name = perm.name
417 if perm_name.startswith("android.permission."):
418 perm_name = perm_name[19:]
419 old_permissions.add(perm_name)
420 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
422 for permission in sorted_permissions:
423 permel = doc.createElement('uses-permission')
424 permel.setAttribute('name', permission.name)
425 if permission.maxSdkVersion is not None:
426 permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
427 apkel.appendChild(permel)
428 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
429 permel = doc.createElement('uses-permission-sdk-23')
430 permel.setAttribute('name', permission_sdk_23.name)
431 if permission_sdk_23.maxSdkVersion is not None:
432 permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
433 apkel.appendChild(permel)
434 if 'nativecode' in apk:
435 addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
436 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
438 if current_version_file is not None \
439 and common.config['make_current_version_link'] \
440 and repodir == 'repo': # only create these
441 namefield = common.config['current_version_name_source']
442 sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
443 apklinkname = sanitized_name + b'.apk'
444 current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
445 if os.path.islink(apklinkname):
446 os.remove(apklinkname)
447 os.symlink(current_version_path, apklinkname)
448 # also symlink gpg signature, if it exists
449 for extension in (b'.asc', b'.sig'):
450 sigfile_path = current_version_path + extension
451 if os.path.exists(sigfile_path):
452 siglinkname = apklinkname + extension
453 if os.path.islink(siglinkname):
454 os.remove(siglinkname)
455 os.symlink(sigfile_path, siglinkname)
457 if common.options.pretty:
458 output = doc.toprettyxml(encoding='utf-8')
460 output = doc.toxml(encoding='utf-8')
462 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
465 if 'repo_keyalias' in common.config:
467 if common.options.nosign:
468 logging.info("Creating unsigned index in preparation for signing")
470 logging.info("Creating signed index with this key (SHA256):")
471 logging.info("%s" % repo_pubkey_fingerprint)
473 # Create a jar of the index...
474 jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
475 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
476 if p.returncode != 0:
477 logging.critical("Failed to create {0}".format(jar_output))
481 signed = os.path.join(repodir, 'index.jar')
482 if common.options.nosign:
483 # Remove old signed index if not signing
484 if os.path.exists(signed):
487 signindex.config = common.config
488 signindex.sign_jar(signed)
490 # Copy the repo icon into the repo directory...
491 icon_dir = os.path.join(repodir, 'icons')
492 iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
493 shutil.copyfile(common.config['repo_icon'], iconfilename)
496 def extract_pubkey():
498 Extracts and returns the repository's public key from the keystore.
499 :return: public key in hex, repository fingerprint
501 if 'repo_pubkey' in common.config:
502 pubkey = unhexlify(common.config['repo_pubkey'])
504 p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
505 '-alias', common.config['repo_keyalias'],
506 '-keystore', common.config['keystore'],
507 '-storepass:file', common.config['keystorepassfile']]
508 + common.config['smartcardoptions'],
509 output=False, stderr_to_stdout=False)
510 if p.returncode != 0 or len(p.output) < 20:
511 msg = "Failed to get repo pubkey!"
512 if common.config['keystore'] == 'NONE':
513 msg += ' Is your crypto smartcard plugged in?'
514 logging.critical(msg)
517 repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
518 return hexlify(pubkey), repo_pubkey_fingerprint
521 # Get raw URL from git service for mirroring
522 def get_raw_mirror(url):
523 # Divide urls in parts
529 # fdroidserver will use always 'master' branch for git-mirroring
533 if hostname == "github.com":
534 # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
535 url[2] = "raw.githubusercontent.com"
536 url.extend([branch, folder])
537 elif hostname == "gitlab.com":
538 # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
539 url.extend(["raw", branch, folder])
547 class VerificationException(Exception):
551 def download_repo_index(url_str, verify_fingerprint=True):
553 Downloads the repository index from the given :param url_str
554 and verifies the repository's fingerprint if :param verify_fingerprint is not False.
556 :raises: VerificationException() if the repository could not be verified
558 :return: The index in JSON format.
560 url = urllib.parse.urlsplit(url_str)
563 if verify_fingerprint:
564 query = urllib.parse.parse_qs(url.query)
565 if 'fingerprint' not in query:
566 raise VerificationException("No fingerprint in URL.")
567 fingerprint = query['fingerprint'][0]
569 url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
570 r = requests.get(url.geturl())
572 with tempfile.NamedTemporaryFile() as fp:
573 # write and open JAR file
575 jar = zipfile.ZipFile(fp)
577 # verify that the JAR signature is valid
578 verify_jar_signature(fp.name)
580 # get public key and its fingerprint from JAR
581 public_key, public_key_fingerprint = get_public_key_from_jar(jar)
583 # compare the fingerprint if verify_fingerprint is True
584 if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
585 raise VerificationException("The repository's fingerprint does not match.")
587 # load repository index from JSON
588 index = json.loads(jar.read('index-v1.json').decode("utf-8"))
589 index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
590 index["repo"]["fingerprint"] = public_key_fingerprint
592 # turn the apps into App objects
593 index["apps"] = [metadata.App(app) for app in index["apps"]]
598 def verify_jar_signature(file):
600 Verifies the signature of a given JAR file.
602 :raises: VerificationException() if the JAR's signature could not be verified
604 if not common.verify_apk_signature(file, jar=True):
605 raise VerificationException("The repository's index could not be verified.")
608 def get_public_key_from_jar(jar):
610 Get the public key and its fingerprint from a JAR file.
612 :raises: VerificationException() if the JAR was not signed exactly once
614 :param jar: a zipfile.ZipFile object
615 :return: the public key from the jar and its fingerprint
617 # extract certificate from jar
618 certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
620 raise VerificationException("Found no signing certificates for repository.")
622 raise VerificationException("Found multiple signing certificates for repository.")
624 # extract public key from certificate
625 public_key = common.get_certificate(jar.read(certs[0]))
626 public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
628 return public_key, public_key_fingerprint