3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
5 # Copyright (C) 2016, Blue Jay Wireless
6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
8 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU Affero General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Affero General Public License for more details.
20 # You should have received a copy of the GNU Affero General Public License
21 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from binascii import hexlify, unhexlify
33 from datetime import datetime
34 from xml.dom.minidom import Document
36 from fdroidserver import metadata, signindex, common
37 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
38 from fdroidserver.metadata import MetaDataException
41 def make(apps, sortedids, apks, repodir, archive):
42 """Generate the repo index files.
44 This requires properly initialized options and config objects.
46 :param apps: fully populated apps list
47 :param sortedids: app package IDs, sorted
48 :param apks: full populated apks list
49 :param repodir: the repo directory
50 :param archive: True if this is the archive repo, False if it's the
53 from fdroidserver.update import METADATA_VERSION
55 def _resolve_description_link(appid):
57 return "fdroid.app:" + appid, apps[appid].Name
58 raise MetaDataException("Cannot resolve app id " + appid)
61 if not common.options.nosign:
62 if 'repo_keyalias' not in common.config:
64 logging.critical("'repo_keyalias' not found in config.py!")
65 if 'keystore' not in common.config:
67 logging.critical("'keystore' not found in config.py!")
68 if 'keystorepass' not in common.config and 'keystorepassfile' not in common.config:
70 logging.critical("'keystorepass' not found in config.py!")
71 if 'keypass' not in common.config and 'keypassfile' not in common.config:
73 logging.critical("'keypass' not found in config.py!")
74 if not os.path.exists(common.config['keystore']):
76 logging.critical("'" + common.config['keystore'] + "' does not exist!")
78 logging.warning("`fdroid update` requires a signing key, you can create one using:")
79 logging.warning("\tfdroid update --create-key")
82 repodict = collections.OrderedDict()
83 repodict['timestamp'] = datetime.utcnow()
84 repodict['version'] = METADATA_VERSION
86 if common.config['repo_maxage'] != 0:
87 repodict['maxage'] = common.config['repo_maxage']
90 repodict['name'] = common.config['archive_name']
91 repodict['icon'] = os.path.basename(common.config['archive_icon'])
92 repodict['address'] = common.config['archive_url']
93 repodict['description'] = common.config['archive_description']
94 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
96 repodict['name'] = common.config['repo_name']
97 repodict['icon'] = os.path.basename(common.config['repo_icon'])
98 repodict['address'] = common.config['repo_url']
99 repodict['description'] = common.config['repo_description']
100 urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
102 mirrorcheckfailed = False
104 for mirror in sorted(common.config.get('mirrors', [])):
105 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
106 if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
107 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
108 mirrorcheckfailed = True
109 # must end with / or urljoin strips a whole path segment
110 if mirror.endswith('/'):
111 mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
113 mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
114 for mirror in common.config.get('servergitmirrors', []):
115 mirror = get_raw_mirror(mirror)
116 if mirror is not None:
117 mirrors.append(mirror + '/')
118 if mirrorcheckfailed:
121 repodict['mirrors'] = mirrors
123 appsWithPackages = collections.OrderedDict()
124 for packageName in sortedids:
125 app = apps[packageName]
129 # only include apps with packages
131 if apk['packageName'] == packageName:
132 newapp = copy.copy(app) # update wiki needs unmodified description
133 newapp['Description'] = metadata.description_html(app['Description'],
134 _resolve_description_link)
135 appsWithPackages[packageName] = newapp
138 requestsdict = dict()
139 for command in ('install', 'uninstall'):
141 key = command + '_list'
142 if key in common.config:
143 if isinstance(common.config[key], str):
144 packageNames = [common.config[key]]
145 elif all(isinstance(item, str) for item in common.config[key]):
146 packageNames = common.config[key]
148 raise TypeError('only accepts strings, lists, and tuples')
149 requestsdict[command] = packageNames
151 make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
152 make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
155 def make_v1(apps, packages, repodir, repodict, requestsdict):
157 def _index_encoder_default(obj):
158 if isinstance(obj, set):
160 if isinstance(obj, datetime):
161 return int(obj.timestamp() * 1000) # Java expects milliseconds
162 raise TypeError(repr(obj) + " is not JSON serializable")
164 output = collections.OrderedDict()
165 output['repo'] = repodict
166 output['requests'] = requestsdict
169 output['apps'] = appslist
170 for appid, appdict in apps.items():
171 d = collections.OrderedDict()
173 for k, v in sorted(appdict.items()):
176 if k in ('builds', 'comments', 'metadatapath',
177 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
178 'Provides', 'Repo', 'RepoType', 'RequiresRoot',
179 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
180 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
183 # name things after the App class fields in fdroidclient
186 elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name
187 k = 'suggestedVersionCode'
188 elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name
189 k = 'suggestedVersionName'
190 elif k == 'AutoName':
191 if 'Name' not in apps[appid]:
195 k = k[:1].lower() + k[1:]
198 output_packages = dict()
199 output['packages'] = output_packages
200 for package in packages:
201 packageName = package['packageName']
202 if packageName in output_packages:
203 packagelist = output_packages[packageName]
206 output_packages[packageName] = packagelist
207 d = collections.OrderedDict()
208 packagelist.append(d)
209 for k, v in sorted(package.items()):
212 if k in ('icon', 'icons', 'icons_src', 'name', ):
216 json_name = 'index-v1.json'
217 index_file = os.path.join(repodir, json_name)
218 with open(index_file, 'w') as fp:
219 json.dump(output, fp, default=_index_encoder_default)
221 if common.options.nosign:
222 logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
224 signindex.config = common.config
225 signindex.sign_index_v1(repodir, json_name)
228 def make_v0(apps, apks, repodir, repodict, requestsdict):
230 aka index.jar aka index.xml
235 def addElement(name, value, doc, parent):
236 el = doc.createElement(name)
237 el.appendChild(doc.createTextNode(value))
238 parent.appendChild(el)
240 def addElementNonEmpty(name, value, doc, parent):
243 addElement(name, value, doc, parent)
245 def addElementIfInApk(name, apk, key, doc, parent):
248 value = str(apk[key])
249 addElement(name, value, doc, parent)
251 def addElementCDATA(name, value, doc, parent):
252 el = doc.createElement(name)
253 el.appendChild(doc.createCDATASection(value))
254 parent.appendChild(el)
256 root = doc.createElement("fdroid")
257 doc.appendChild(root)
259 repoel = doc.createElement("repo")
261 repoel.setAttribute("name", repodict['name'])
262 if 'maxage' in repodict:
263 repoel.setAttribute("maxage", str(repodict['maxage']))
264 repoel.setAttribute("icon", os.path.basename(repodict['icon']))
265 repoel.setAttribute("url", repodict['address'])
266 addElement('description', repodict['description'], doc, repoel)
267 for mirror in repodict.get('mirrors', []):
268 addElement('mirror', mirror, doc, repoel)
270 repoel.setAttribute("version", str(repodict['version']))
271 repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
273 pubkey, repo_pubkey_fingerprint = extract_pubkey()
274 repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
275 root.appendChild(repoel)
277 for command in ('install', 'uninstall'):
278 for packageName in requestsdict[command]:
279 element = doc.createElement(command)
280 root.appendChild(element)
281 element.setAttribute('packageName', packageName)
283 for appid, appdict in apps.items():
284 app = metadata.App(appdict)
286 if app.Disabled is not None:
289 # Get a list of the apks for this app...
292 if apk['packageName'] == appid:
295 if len(apklist) == 0:
298 apel = doc.createElement("application")
299 apel.setAttribute("id", app.id)
300 root.appendChild(apel)
302 addElement('id', app.id, doc, apel)
304 addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
306 addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
307 addElement('name', app.Name, doc, apel)
308 addElement('summary', app.Summary, doc, apel)
310 addElement('icon', app.icon, doc, apel)
312 if app.get('Description'):
313 description = app.Description
315 description = '<p>No description available</p>'
316 addElement('desc', description, doc, apel)
317 addElement('license', app.License, doc, apel)
319 addElement('categories', ','.join(app.Categories), doc, apel)
320 # We put the first (primary) category in LAST, which will have
321 # the desired effect of making clients that only understand one
322 # category see that one.
323 addElement('category', app.Categories[0], doc, apel)
324 addElement('web', app.WebSite, doc, apel)
325 addElement('source', app.SourceCode, doc, apel)
326 addElement('tracker', app.IssueTracker, doc, apel)
327 addElementNonEmpty('changelog', app.Changelog, doc, apel)
328 addElementNonEmpty('author', app.AuthorName, doc, apel)
329 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
330 addElementNonEmpty('donate', app.Donate, doc, apel)
331 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
332 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
333 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
335 # These elements actually refer to the current version (i.e. which
336 # one is recommended. They are historically mis-named, and need
337 # changing, but stay like this for now to support existing clients.
338 addElement('marketversion', app.CurrentVersion, doc, apel)
339 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
342 pv = app.Provides.split(',')
343 addElementNonEmpty('provides', ','.join(pv), doc, apel)
345 addElement('requirements', 'root', doc, apel)
347 # Sort the apk list into version order, just so the web site
348 # doesn't have to do any work by default...
349 apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
351 if 'antiFeatures' in apklist[0]:
352 app.AntiFeatures.extend(apklist[0]['antiFeatures'])
354 addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
356 # Check for duplicates - they will make the client unhappy...
357 for i in range(len(apklist) - 1):
358 if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
359 logging.critical("duplicate versions: '%s' - '%s'" % (
360 apklist[i]['apkName'], apklist[i + 1]['apkName']))
363 current_version_code = 0
364 current_version_file = None
366 file_extension = common.get_file_extension(apk['apkName'])
367 # find the APK for the "Current Version"
368 if current_version_code < apk['versionCode']:
369 current_version_code = apk['versionCode']
370 if current_version_code < int(app.CurrentVersionCode):
371 current_version_file = apk['apkName']
373 apkel = doc.createElement("package")
374 apel.appendChild(apkel)
375 addElement('version', apk['versionName'], doc, apkel)
376 addElement('versioncode', str(apk['versionCode']), doc, apkel)
377 addElement('apkname', apk['apkName'], doc, apkel)
378 addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
380 hashel = doc.createElement("hash")
381 hashel.setAttribute('type', 'sha256')
382 hashel.appendChild(doc.createTextNode(apk['hash']))
383 apkel.appendChild(hashel)
385 addElement('size', str(apk['size']), doc, apkel)
386 addElementIfInApk('sdkver', apk,
387 'minSdkVersion', doc, apkel)
388 addElementIfInApk('targetSdkVersion', apk,
389 'targetSdkVersion', doc, apkel)
390 addElementIfInApk('maxsdkver', apk,
391 'maxSdkVersion', doc, apkel)
392 addElementIfInApk('obbMainFile', apk,
393 'obbMainFile', doc, apkel)
394 addElementIfInApk('obbMainFileSha256', apk,
395 'obbMainFileSha256', doc, apkel)
396 addElementIfInApk('obbPatchFile', apk,
397 'obbPatchFile', doc, apkel)
398 addElementIfInApk('obbPatchFileSha256', apk,
399 'obbPatchFileSha256', doc, apkel)
401 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
403 if file_extension == 'apk': # sig is required for APKs, but only APKs
404 addElement('sig', apk['sig'], doc, apkel)
406 old_permissions = set()
407 sorted_permissions = sorted(apk['uses-permission'])
408 for perm in sorted_permissions:
409 perm_name = perm.name
410 if perm_name.startswith("android.permission."):
411 perm_name = perm_name[19:]
412 old_permissions.add(perm_name)
413 addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
415 for permission in sorted_permissions:
416 permel = doc.createElement('uses-permission')
417 permel.setAttribute('name', permission.name)
418 if permission.maxSdkVersion is not None:
419 permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
420 apkel.appendChild(permel)
421 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
422 permel = doc.createElement('uses-permission-sdk-23')
423 permel.setAttribute('name', permission_sdk_23.name)
424 if permission_sdk_23.maxSdkVersion is not None:
425 permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
426 apkel.appendChild(permel)
427 if 'nativecode' in apk:
428 addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
429 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
431 if current_version_file is not None \
432 and common.config['make_current_version_link'] \
433 and repodir == 'repo': # only create these
434 namefield = common.config['current_version_name_source']
435 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
436 apklinkname = sanitized_name + '.apk'
437 current_version_path = os.path.join(repodir, current_version_file)
438 if os.path.islink(apklinkname):
439 os.remove(apklinkname)
440 os.symlink(current_version_path, apklinkname)
441 # also symlink gpg signature, if it exists
442 for extension in ('.asc', '.sig'):
443 sigfile_path = current_version_path + extension
444 if os.path.exists(sigfile_path):
445 siglinkname = apklinkname + extension
446 if os.path.islink(siglinkname):
447 os.remove(siglinkname)
448 os.symlink(sigfile_path, siglinkname)
450 if common.options.pretty:
451 output = doc.toprettyxml(encoding='utf-8')
453 output = doc.toxml(encoding='utf-8')
455 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
458 if 'repo_keyalias' in common.config:
460 if common.options.nosign:
461 logging.info("Creating unsigned index in preparation for signing")
463 logging.info("Creating signed index with this key (SHA256):")
464 logging.info("%s" % repo_pubkey_fingerprint)
466 # Create a jar of the index...
467 jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
468 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
469 if p.returncode != 0:
470 logging.critical("Failed to create {0}".format(jar_output))
474 signed = os.path.join(repodir, 'index.jar')
475 if common.options.nosign:
476 # Remove old signed index if not signing
477 if os.path.exists(signed):
480 signindex.config = common.config
481 signindex.sign_jar(signed)
483 # Copy the repo icon into the repo directory...
484 icon_dir = os.path.join(repodir, 'icons')
485 iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
486 shutil.copyfile(common.config['repo_icon'], iconfilename)
489 def extract_pubkey():
491 Extracts and returns the repository's public key from the keystore.
492 :return: public key in hex, repository fingerprint
494 if 'repo_pubkey' in common.config:
495 pubkey = unhexlify(common.config['repo_pubkey'])
497 p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
498 '-alias', common.config['repo_keyalias'],
499 '-keystore', common.config['keystore'],
500 '-storepass:file', common.config['keystorepassfile']]
501 + common.config['smartcardoptions'],
502 output=False, stderr_to_stdout=False)
503 if p.returncode != 0 or len(p.output) < 20:
504 msg = "Failed to get repo pubkey!"
505 if common.config['keystore'] == 'NONE':
506 msg += ' Is your crypto smartcard plugged in?'
507 logging.critical(msg)
510 repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
511 return hexlify(pubkey), repo_pubkey_fingerprint
514 # Get raw URL from git service for mirroring
515 def get_raw_mirror(url):
516 # Divide urls in parts
522 # fdroidserver will use always 'master' branch for git-mirroring
526 if hostname == "github.com":
527 # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
528 url[2] = "raw.githubusercontent.com"
529 url.extend([branch, folder])
530 elif hostname == "gitlab.com":
531 # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
532 url.extend(["raw", branch, folder])