fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import sys
  31 import urllib.parse
  32 from binascii import hexlify, unhexlify
  33 from datetime import datetime
  34 from xml.dom.minidom import Document
  35
  36 from fdroidserver import metadata, signindex, common
  37 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
  38 from fdroidserver.metadata import MetaDataException
  39
  40
  41 def make(apps, sortedids, apks, repodir, archive):
  42     """Generate the repo index files.
  43
  44     This requires properly initialized options and config objects.
  45
  46     :param apps: fully populated apps list
  47     :param sortedids: app package IDs, sorted
  48     :param apks: full populated apks list
  49     :param repodir: the repo directory
  50     :param archive: True if this is the archive repo, False if it's the
  51                     main one.
  52     """
  53     from fdroidserver.update import METADATA_VERSION
  54
  55     def _resolve_description_link(appid):
  56         if appid in apps:
  57             return "fdroid.app:" + appid, apps[appid].Name
  58         raise MetaDataException("Cannot resolve app id " + appid)
  59
  60     nosigningkey = False
  61     if not common.options.nosign:
  62         if 'repo_keyalias' not in common.config:
  63             nosigningkey = True
  64             logging.critical("'repo_keyalias' not found in config.py!")
  65         if 'keystore' not in common.config:
  66             nosigningkey = True
  67             logging.critical("'keystore' not found in config.py!")
  68         if 'keystorepass' not in common.config and 'keystorepassfile' not in common.config:
  69             nosigningkey = True
  70             logging.critical("'keystorepass' not found in config.py!")
  71         if 'keypass' not in common.config and 'keypassfile' not in common.config:
  72             nosigningkey = True
  73             logging.critical("'keypass' not found in config.py!")
  74         if not os.path.exists(common.config['keystore']):
  75             nosigningkey = True
  76             logging.critical("'" + common.config['keystore'] + "' does not exist!")
  77         if nosigningkey:
  78             logging.warning("`fdroid update` requires a signing key, you can create one using:")
  79             logging.warning("\tfdroid update --create-key")
  80             sys.exit(1)
  81
  82     repodict = collections.OrderedDict()
  83     repodict['timestamp'] = datetime.utcnow()
  84     repodict['version'] = METADATA_VERSION
  85
  86     if common.config['repo_maxage'] != 0:
  87         repodict['maxage'] = common.config['repo_maxage']
  88
  89     if archive:
  90         repodict['name'] = common.config['archive_name']
  91         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  92         repodict['address'] = common.config['archive_url']
  93         repodict['description'] = common.config['archive_description']
  94         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
  95     else:
  96         repodict['name'] = common.config['repo_name']
  97         repodict['icon'] = os.path.basename(common.config['repo_icon'])
  98         repodict['address'] = common.config['repo_url']
  99         repodict['description'] = common.config['repo_description']
 100         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
 101
 102     mirrorcheckfailed = False
 103     mirrors = []
 104     for mirror in sorted(common.config.get('mirrors', [])):
 105         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 106         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
 107             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 108             mirrorcheckfailed = True
 109         # must end with / or urljoin strips a whole path segment
 110         if mirror.endswith('/'):
 111             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
 112         else:
 113             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 114     for mirror in common.config.get('servergitmirrors', []):
 115         mirror = get_raw_mirror(mirror)
 116         if mirror is not None:
 117             mirrors.append(mirror + '/')
 118     if mirrorcheckfailed:
 119         sys.exit(1)
 120     if mirrors:
 121         repodict['mirrors'] = mirrors
 122
 123     appsWithPackages = collections.OrderedDict()
 124     for packageName in sortedids:
 125         app = apps[packageName]
 126         if app['Disabled']:
 127             continue
 128
 129         # only include apps with packages
 130         for apk in apks:
 131             if apk['packageName'] == packageName:
 132                 newapp = copy.copy(app)  # update wiki needs unmodified description
 133                 newapp['Description'] = metadata.description_html(app['Description'],
 134                                                                   _resolve_description_link)
 135                 appsWithPackages[packageName] = newapp
 136                 break
 137
 138     requestsdict = dict()
 139     for command in ('install', 'uninstall'):
 140         packageNames = []
 141         key = command + '_list'
 142         if key in common.config:
 143             if isinstance(common.config[key], str):
 144                 packageNames = [common.config[key]]
 145             elif all(isinstance(item, str) for item in common.config[key]):
 146                 packageNames = common.config[key]
 147             else:
 148                 raise TypeError('only accepts strings, lists, and tuples')
 149         requestsdict[command] = packageNames
 150
 151     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
 152     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
 153
 154
 155 def make_v1(apps, packages, repodir, repodict, requestsdict):
 156
 157     def _index_encoder_default(obj):
 158         if isinstance(obj, set):
 159             return list(obj)
 160         if isinstance(obj, datetime):
 161             return int(obj.timestamp() * 1000)  # Java expects milliseconds
 162         raise TypeError(repr(obj) + " is not JSON serializable")
 163
 164     output = collections.OrderedDict()
 165     output['repo'] = repodict
 166     output['requests'] = requestsdict
 167
 168     appslist = []
 169     output['apps'] = appslist
 170     for appid, appdict in apps.items():
 171         d = collections.OrderedDict()
 172         appslist.append(d)
 173         for k, v in sorted(appdict.items()):
 174             if not v:
 175                 continue
 176             if k in ('builds', 'comments', 'metadatapath',
 177                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 178                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 179                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 180                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 181                 continue
 182
 183             # name things after the App class fields in fdroidclient
 184             if k == 'id':
 185                 k = 'packageName'
 186             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 187                 k = 'suggestedVersionCode'
 188             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 189                 k = 'suggestedVersionName'
 190             elif k == 'AutoName':
 191                 if 'Name' not in apps[appid]:
 192                     d['name'] = v
 193                 continue
 194             else:
 195                 k = k[:1].lower() + k[1:]
 196             d[k] = v
 197
 198     output_packages = dict()
 199     output['packages'] = output_packages
 200     for package in packages:
 201         packageName = package['packageName']
 202         if packageName in output_packages:
 203             packagelist = output_packages[packageName]
 204         else:
 205             packagelist = []
 206             output_packages[packageName] = packagelist
 207         d = collections.OrderedDict()
 208         packagelist.append(d)
 209         for k, v in sorted(package.items()):
 210             if not v:
 211                 continue
 212             if k in ('icon', 'icons', 'icons_src', 'name', ):
 213                 continue
 214             d[k] = v
 215
 216     json_name = 'index-v1.json'
 217     index_file = os.path.join(repodir, json_name)
 218     with open(index_file, 'w') as fp:
 219         json.dump(output, fp, default=_index_encoder_default)
 220
 221     if common.options.nosign:
 222         logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
 223     else:
 224         signindex.config = common.config
 225         signindex.sign_index_v1(repodir, json_name)
 226
 227
 228 def make_v0(apps, apks, repodir, repodict, requestsdict):
 229     """
 230     aka index.jar aka index.xml
 231     """
 232
 233     doc = Document()
 234
 235     def addElement(name, value, doc, parent):
 236         el = doc.createElement(name)
 237         el.appendChild(doc.createTextNode(value))
 238         parent.appendChild(el)
 239
 240     def addElementNonEmpty(name, value, doc, parent):
 241         if not value:
 242             return
 243         addElement(name, value, doc, parent)
 244
 245     def addElementIfInApk(name, apk, key, doc, parent):
 246         if key not in apk:
 247             return
 248         value = str(apk[key])
 249         addElement(name, value, doc, parent)
 250
 251     def addElementCDATA(name, value, doc, parent):
 252         el = doc.createElement(name)
 253         el.appendChild(doc.createCDATASection(value))
 254         parent.appendChild(el)
 255
 256     root = doc.createElement("fdroid")
 257     doc.appendChild(root)
 258
 259     repoel = doc.createElement("repo")
 260
 261     repoel.setAttribute("name", repodict['name'])
 262     if 'maxage' in repodict:
 263         repoel.setAttribute("maxage", str(repodict['maxage']))
 264     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 265     repoel.setAttribute("url", repodict['address'])
 266     addElement('description', repodict['description'], doc, repoel)
 267     for mirror in repodict.get('mirrors', []):
 268         addElement('mirror', mirror, doc, repoel)
 269
 270     repoel.setAttribute("version", str(repodict['version']))
 271     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 272
 273     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 274     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 275     root.appendChild(repoel)
 276
 277     for command in ('install', 'uninstall'):
 278         for packageName in requestsdict[command]:
 279             element = doc.createElement(command)
 280             root.appendChild(element)
 281             element.setAttribute('packageName', packageName)
 282
 283     for appid, appdict in apps.items():
 284         app = metadata.App(appdict)
 285
 286         if app.Disabled is not None:
 287             continue
 288
 289         # Get a list of the apks for this app...
 290         apklist = []
 291         for apk in apks:
 292             if apk['packageName'] == appid:
 293                 apklist.append(apk)
 294
 295         if len(apklist) == 0:
 296             continue
 297
 298         apel = doc.createElement("application")
 299         apel.setAttribute("id", app.id)
 300         root.appendChild(apel)
 301
 302         addElement('id', app.id, doc, apel)
 303         if app.added:
 304             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 305         if app.lastUpdated:
 306             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 307         addElement('name', app.Name, doc, apel)
 308         addElement('summary', app.Summary, doc, apel)
 309         if app.icon:
 310             addElement('icon', app.icon, doc, apel)
 311
 312         if app.get('Description'):
 313             description = app.Description
 314         else:
 315             description = '<p>No description available</p>'
 316         addElement('desc', description, doc, apel)
 317         addElement('license', app.License, doc, apel)
 318         if app.Categories:
 319             addElement('categories', ','.join(app.Categories), doc, apel)
 320             # We put the first (primary) category in LAST, which will have
 321             # the desired effect of making clients that only understand one
 322             # category see that one.
 323             addElement('category', app.Categories[0], doc, apel)
 324         addElement('web', app.WebSite, doc, apel)
 325         addElement('source', app.SourceCode, doc, apel)
 326         addElement('tracker', app.IssueTracker, doc, apel)
 327         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 328         addElementNonEmpty('author', app.AuthorName, doc, apel)
 329         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 330         addElementNonEmpty('donate', app.Donate, doc, apel)
 331         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 332         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 333         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 334
 335         # These elements actually refer to the current version (i.e. which
 336         # one is recommended. They are historically mis-named, and need
 337         # changing, but stay like this for now to support existing clients.
 338         addElement('marketversion', app.CurrentVersion, doc, apel)
 339         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 340
 341         if app.Provides:
 342             pv = app.Provides.split(',')
 343             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 344         if app.RequiresRoot:
 345             addElement('requirements', 'root', doc, apel)
 346
 347         # Sort the apk list into version order, just so the web site
 348         # doesn't have to do any work by default...
 349         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 350
 351         if 'antiFeatures' in apklist[0]:
 352             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 353         if app.AntiFeatures:
 354             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 355
 356         # Check for duplicates - they will make the client unhappy...
 357         for i in range(len(apklist) - 1):
 358             if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
 359                 logging.critical("duplicate versions: '%s' - '%s'" % (
 360                     apklist[i]['apkName'], apklist[i + 1]['apkName']))
 361                 sys.exit(1)
 362
 363         current_version_code = 0
 364         current_version_file = None
 365         for apk in apklist:
 366             file_extension = common.get_file_extension(apk['apkName'])
 367             # find the APK for the "Current Version"
 368             if current_version_code < apk['versionCode']:
 369                 current_version_code = apk['versionCode']
 370             if current_version_code < int(app.CurrentVersionCode):
 371                 current_version_file = apk['apkName']
 372
 373             apkel = doc.createElement("package")
 374             apel.appendChild(apkel)
 375             addElement('version', apk['versionName'], doc, apkel)
 376             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 377             addElement('apkname', apk['apkName'], doc, apkel)
 378             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 379
 380             hashel = doc.createElement("hash")
 381             hashel.setAttribute('type', 'sha256')
 382             hashel.appendChild(doc.createTextNode(apk['hash']))
 383             apkel.appendChild(hashel)
 384
 385             addElement('size', str(apk['size']), doc, apkel)
 386             addElementIfInApk('sdkver', apk,
 387                               'minSdkVersion', doc, apkel)
 388             addElementIfInApk('targetSdkVersion', apk,
 389                               'targetSdkVersion', doc, apkel)
 390             addElementIfInApk('maxsdkver', apk,
 391                               'maxSdkVersion', doc, apkel)
 392             addElementIfInApk('obbMainFile', apk,
 393                               'obbMainFile', doc, apkel)
 394             addElementIfInApk('obbMainFileSha256', apk,
 395                               'obbMainFileSha256', doc, apkel)
 396             addElementIfInApk('obbPatchFile', apk,
 397                               'obbPatchFile', doc, apkel)
 398             addElementIfInApk('obbPatchFileSha256', apk,
 399                               'obbPatchFileSha256', doc, apkel)
 400             if 'added' in apk:
 401                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 402
 403             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 404                 addElement('sig', apk['sig'], doc, apkel)
 405
 406                 old_permissions = set()
 407                 sorted_permissions = sorted(apk['uses-permission'])
 408                 for perm in sorted_permissions:
 409                     perm_name = perm.name
 410                     if perm_name.startswith("android.permission."):
 411                         perm_name = perm_name[19:]
 412                     old_permissions.add(perm_name)
 413                 addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
 414
 415                 for permission in sorted_permissions:
 416                     permel = doc.createElement('uses-permission')
 417                     permel.setAttribute('name', permission.name)
 418                     if permission.maxSdkVersion is not None:
 419                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 420                         apkel.appendChild(permel)
 421                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 422                     permel = doc.createElement('uses-permission-sdk-23')
 423                     permel.setAttribute('name', permission_sdk_23.name)
 424                     if permission_sdk_23.maxSdkVersion is not None:
 425                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 426                         apkel.appendChild(permel)
 427                 if 'nativecode' in apk:
 428                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 429                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 430
 431         if current_version_file is not None \
 432                 and common.config['make_current_version_link'] \
 433                 and repodir == 'repo':  # only create these
 434             namefield = common.config['current_version_name_source']
 435             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
 436             apklinkname = sanitized_name + '.apk'
 437             current_version_path = os.path.join(repodir, current_version_file)
 438             if os.path.islink(apklinkname):
 439                 os.remove(apklinkname)
 440             os.symlink(current_version_path, apklinkname)
 441             # also symlink gpg signature, if it exists
 442             for extension in ('.asc', '.sig'):
 443                 sigfile_path = current_version_path + extension
 444                 if os.path.exists(sigfile_path):
 445                     siglinkname = apklinkname + extension
 446                     if os.path.islink(siglinkname):
 447                         os.remove(siglinkname)
 448                     os.symlink(sigfile_path, siglinkname)
 449
 450     if common.options.pretty:
 451         output = doc.toprettyxml(encoding='utf-8')
 452     else:
 453         output = doc.toxml(encoding='utf-8')
 454
 455     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 456         f.write(output)
 457
 458     if 'repo_keyalias' in common.config:
 459
 460         if common.options.nosign:
 461             logging.info("Creating unsigned index in preparation for signing")
 462         else:
 463             logging.info("Creating signed index with this key (SHA256):")
 464             logging.info("%s" % repo_pubkey_fingerprint)
 465
 466         # Create a jar of the index...
 467         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 468         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 469         if p.returncode != 0:
 470             logging.critical("Failed to create {0}".format(jar_output))
 471             sys.exit(1)
 472
 473         # Sign the index...
 474         signed = os.path.join(repodir, 'index.jar')
 475         if common.options.nosign:
 476             # Remove old signed index if not signing
 477             if os.path.exists(signed):
 478                 os.remove(signed)
 479         else:
 480             signindex.config = common.config
 481             signindex.sign_jar(signed)
 482
 483     # Copy the repo icon into the repo directory...
 484     icon_dir = os.path.join(repodir, 'icons')
 485     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 486     shutil.copyfile(common.config['repo_icon'], iconfilename)
 487
 488
 489 def extract_pubkey():
 490     """
 491     Extracts and returns the repository's public key from the keystore.
 492     :return: public key in hex, repository fingerprint
 493     """
 494     if 'repo_pubkey' in common.config:
 495         pubkey = unhexlify(common.config['repo_pubkey'])
 496     else:
 497         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 498                               '-alias', common.config['repo_keyalias'],
 499                               '-keystore', common.config['keystore'],
 500                               '-storepass:file', common.config['keystorepassfile']]
 501                              + common.config['smartcardoptions'],
 502                              output=False, stderr_to_stdout=False)
 503         if p.returncode != 0 or len(p.output) < 20:
 504             msg = "Failed to get repo pubkey!"
 505             if common.config['keystore'] == 'NONE':
 506                 msg += ' Is your crypto smartcard plugged in?'
 507             logging.critical(msg)
 508             sys.exit(1)
 509         pubkey = p.output
 510     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 511     return hexlify(pubkey), repo_pubkey_fingerprint
 512
 513
 514 # Get raw URL from git service for mirroring
 515 def get_raw_mirror(url):
 516     # Divide urls in parts
 517     url = url.split("/")
 518
 519     # Get the hostname
 520     hostname = url[2]
 521
 522     # fdroidserver will use always 'master' branch for git-mirroring
 523     branch = "master"
 524     folder = "fdroid"
 525
 526     if hostname == "github.com":
 527         # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
 528         url[2] = "raw.githubusercontent.com"
 529         url.extend([branch, folder])
 530     elif hostname == "gitlab.com":
 531         # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
 532         url.extend(["raw", branch, folder])
 533     else:
 534         return None
 535
 536     url = "/".join(url)
 537     return url