fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import sys
  31 import urllib.parse
  32 from binascii import hexlify, unhexlify
  33 from datetime import datetime
  34 from xml.dom.minidom import Document
  35
  36 from fdroidserver import metadata, signindex, common
  37 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
  38 from fdroidserver.metadata import MetaDataException
  39
  40 options = None
  41 config = None
  42
  43
  44 def make(apps, sortedids, apks, repodir, archive):
  45     """Generate the repo index files.
  46
  47     This requires properly initialized options and config objects.
  48
  49     :param apps: fully populated apps list
  50     :param sortedids: app package IDs, sorted
  51     :param apks: full populated apks list
  52     :param repodir: the repo directory
  53     :param archive: True if this is the archive repo, False if it's the
  54                     main one.
  55     """
  56     from fdroidserver.update import METADATA_VERSION
  57
  58     def _resolve_description_link(appid):
  59         if appid in apps:
  60             return "fdroid.app:" + appid, apps[appid].Name
  61         raise MetaDataException("Cannot resolve app id " + appid)
  62
  63     nosigningkey = False
  64     if not options.nosign:
  65         if 'repo_keyalias' not in config:
  66             nosigningkey = True
  67             logging.critical("'repo_keyalias' not found in config.py!")
  68         if 'keystore' not in config:
  69             nosigningkey = True
  70             logging.critical("'keystore' not found in config.py!")
  71         if 'keystorepass' not in config and 'keystorepassfile' not in config:
  72             nosigningkey = True
  73             logging.critical("'keystorepass' not found in config.py!")
  74         if 'keypass' not in config and 'keypassfile' not in config:
  75             nosigningkey = True
  76             logging.critical("'keypass' not found in config.py!")
  77         if not os.path.exists(config['keystore']):
  78             nosigningkey = True
  79             logging.critical("'" + config['keystore'] + "' does not exist!")
  80         if nosigningkey:
  81             logging.warning("`fdroid update` requires a signing key, you can create one using:")
  82             logging.warning("\tfdroid update --create-key")
  83             sys.exit(1)
  84
  85     repodict = collections.OrderedDict()
  86     repodict['timestamp'] = datetime.utcnow()
  87     repodict['version'] = METADATA_VERSION
  88
  89     if config['repo_maxage'] != 0:
  90         repodict['maxage'] = config['repo_maxage']
  91
  92     if archive:
  93         repodict['name'] = config['archive_name']
  94         repodict['icon'] = os.path.basename(config['archive_icon'])
  95         repodict['address'] = config['archive_url']
  96         repodict['description'] = config['archive_description']
  97         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
  98     else:
  99         repodict['name'] = config['repo_name']
 100         repodict['icon'] = os.path.basename(config['repo_icon'])
 101         repodict['address'] = config['repo_url']
 102         repodict['description'] = config['repo_description']
 103         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 104
 105     mirrorcheckfailed = False
 106     mirrors = []
 107     for mirror in sorted(config.get('mirrors', [])):
 108         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 109         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 110             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 111             mirrorcheckfailed = True
 112         # must end with / or urljoin strips a whole path segment
 113         if mirror.endswith('/'):
 114             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
 115         else:
 116             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 117     for mirror in config.get('servergitmirrors', []):
 118         mirror = get_raw_mirror(mirror)
 119         if mirror is not None:
 120             mirrors.append(mirror + '/')
 121     if mirrorcheckfailed:
 122         sys.exit(1)
 123     if mirrors:
 124         repodict['mirrors'] = mirrors
 125
 126     appsWithPackages = collections.OrderedDict()
 127     for packageName in sortedids:
 128         app = apps[packageName]
 129         if app['Disabled']:
 130             continue
 131
 132         # only include apps with packages
 133         for apk in apks:
 134             if apk['packageName'] == packageName:
 135                 newapp = copy.copy(app)  # update wiki needs unmodified description
 136                 newapp['Description'] = metadata.description_html(app['Description'],
 137                                                                   _resolve_description_link)
 138                 appsWithPackages[packageName] = newapp
 139                 break
 140
 141     requestsdict = dict()
 142     for command in ('install', 'uninstall'):
 143         packageNames = []
 144         key = command + '_list'
 145         if key in config:
 146             if isinstance(config[key], str):
 147                 packageNames = [config[key]]
 148             elif all(isinstance(item, str) for item in config[key]):
 149                 packageNames = config[key]
 150             else:
 151                 raise TypeError('only accepts strings, lists, and tuples')
 152         requestsdict[command] = packageNames
 153
 154     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
 155     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
 156
 157
 158 def make_v1(apps, packages, repodir, repodict, requestsdict):
 159
 160     def _index_encoder_default(obj):
 161         if isinstance(obj, set):
 162             return list(obj)
 163         if isinstance(obj, datetime):
 164             return int(obj.timestamp() * 1000)  # Java expects milliseconds
 165         raise TypeError(repr(obj) + " is not JSON serializable")
 166
 167     output = collections.OrderedDict()
 168     output['repo'] = repodict
 169     output['requests'] = requestsdict
 170
 171     appslist = []
 172     output['apps'] = appslist
 173     for appid, appdict in apps.items():
 174         d = collections.OrderedDict()
 175         appslist.append(d)
 176         for k, v in sorted(appdict.items()):
 177             if not v:
 178                 continue
 179             if k in ('builds', 'comments', 'metadatapath',
 180                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 181                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 182                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 183                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 184                 continue
 185
 186             # name things after the App class fields in fdroidclient
 187             if k == 'id':
 188                 k = 'packageName'
 189             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 190                 k = 'suggestedVersionCode'
 191             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 192                 k = 'suggestedVersionName'
 193             elif k == 'AutoName':
 194                 if 'Name' not in apps[appid]:
 195                     d['name'] = v
 196                 continue
 197             else:
 198                 k = k[:1].lower() + k[1:]
 199             d[k] = v
 200
 201     output_packages = dict()
 202     output['packages'] = output_packages
 203     for package in packages:
 204         packageName = package['packageName']
 205         if packageName in output_packages:
 206             packagelist = output_packages[packageName]
 207         else:
 208             packagelist = []
 209             output_packages[packageName] = packagelist
 210         d = collections.OrderedDict()
 211         packagelist.append(d)
 212         for k, v in sorted(package.items()):
 213             if not v:
 214                 continue
 215             if k in ('icon', 'icons', 'icons_src', 'name', ):
 216                 continue
 217             d[k] = v
 218
 219     json_name = 'index-v1.json'
 220     index_file = os.path.join(repodir, json_name)
 221     with open(index_file, 'w') as fp:
 222         json.dump(output, fp, default=_index_encoder_default)
 223
 224     if options.nosign:
 225         logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
 226     else:
 227         signindex.config = config
 228         signindex.sign_index_v1(repodir, json_name)
 229
 230
 231 def make_v0(apps, apks, repodir, repodict, requestsdict):
 232     """
 233     aka index.jar aka index.xml
 234     """
 235
 236     doc = Document()
 237
 238     def addElement(name, value, doc, parent):
 239         el = doc.createElement(name)
 240         el.appendChild(doc.createTextNode(value))
 241         parent.appendChild(el)
 242
 243     def addElementNonEmpty(name, value, doc, parent):
 244         if not value:
 245             return
 246         addElement(name, value, doc, parent)
 247
 248     def addElementIfInApk(name, apk, key, doc, parent):
 249         if key not in apk:
 250             return
 251         value = str(apk[key])
 252         addElement(name, value, doc, parent)
 253
 254     def addElementCDATA(name, value, doc, parent):
 255         el = doc.createElement(name)
 256         el.appendChild(doc.createCDATASection(value))
 257         parent.appendChild(el)
 258
 259     root = doc.createElement("fdroid")
 260     doc.appendChild(root)
 261
 262     repoel = doc.createElement("repo")
 263
 264     repoel.setAttribute("name", repodict['name'])
 265     if 'maxage' in repodict:
 266         repoel.setAttribute("maxage", str(repodict['maxage']))
 267     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 268     repoel.setAttribute("url", repodict['address'])
 269     addElement('description', repodict['description'], doc, repoel)
 270     for mirror in repodict.get('mirrors', []):
 271         addElement('mirror', mirror, doc, repoel)
 272
 273     repoel.setAttribute("version", str(repodict['version']))
 274     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 275
 276     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 277     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 278     root.appendChild(repoel)
 279
 280     for command in ('install', 'uninstall'):
 281         for packageName in requestsdict[command]:
 282             element = doc.createElement(command)
 283             root.appendChild(element)
 284             element.setAttribute('packageName', packageName)
 285
 286     for appid, appdict in apps.items():
 287         app = metadata.App(appdict)
 288
 289         if app.Disabled is not None:
 290             continue
 291
 292         # Get a list of the apks for this app...
 293         apklist = []
 294         for apk in apks:
 295             if apk['packageName'] == appid:
 296                 apklist.append(apk)
 297
 298         if len(apklist) == 0:
 299             continue
 300
 301         apel = doc.createElement("application")
 302         apel.setAttribute("id", app.id)
 303         root.appendChild(apel)
 304
 305         addElement('id', app.id, doc, apel)
 306         if app.added:
 307             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 308         if app.lastUpdated:
 309             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 310         addElement('name', app.Name, doc, apel)
 311         addElement('summary', app.Summary, doc, apel)
 312         if app.icon:
 313             addElement('icon', app.icon, doc, apel)
 314
 315         if app.get('Description'):
 316             description = app.Description
 317         else:
 318             description = '<p>No description available</p>'
 319         addElement('desc', description, doc, apel)
 320         addElement('license', app.License, doc, apel)
 321         if app.Categories:
 322             addElement('categories', ','.join(app.Categories), doc, apel)
 323             # We put the first (primary) category in LAST, which will have
 324             # the desired effect of making clients that only understand one
 325             # category see that one.
 326             addElement('category', app.Categories[0], doc, apel)
 327         addElement('web', app.WebSite, doc, apel)
 328         addElement('source', app.SourceCode, doc, apel)
 329         addElement('tracker', app.IssueTracker, doc, apel)
 330         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 331         addElementNonEmpty('author', app.AuthorName, doc, apel)
 332         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 333         addElementNonEmpty('donate', app.Donate, doc, apel)
 334         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 335         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 336         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 337
 338         # These elements actually refer to the current version (i.e. which
 339         # one is recommended. They are historically mis-named, and need
 340         # changing, but stay like this for now to support existing clients.
 341         addElement('marketversion', app.CurrentVersion, doc, apel)
 342         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 343
 344         if app.Provides:
 345             pv = app.Provides.split(',')
 346             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 347         if app.RequiresRoot:
 348             addElement('requirements', 'root', doc, apel)
 349
 350         # Sort the apk list into version order, just so the web site
 351         # doesn't have to do any work by default...
 352         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 353
 354         if 'antiFeatures' in apklist[0]:
 355             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 356         if app.AntiFeatures:
 357             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 358
 359         # Check for duplicates - they will make the client unhappy...
 360         for i in range(len(apklist) - 1):
 361             if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
 362                 logging.critical("duplicate versions: '%s' - '%s'" % (
 363                     apklist[i]['apkName'], apklist[i + 1]['apkName']))
 364                 sys.exit(1)
 365
 366         current_version_code = 0
 367         current_version_file = None
 368         for apk in apklist:
 369             file_extension = common.get_file_extension(apk['apkName'])
 370             # find the APK for the "Current Version"
 371             if current_version_code < apk['versionCode']:
 372                 current_version_code = apk['versionCode']
 373             if current_version_code < int(app.CurrentVersionCode):
 374                 current_version_file = apk['apkName']
 375
 376             apkel = doc.createElement("package")
 377             apel.appendChild(apkel)
 378             addElement('version', apk['versionName'], doc, apkel)
 379             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 380             addElement('apkname', apk['apkName'], doc, apkel)
 381             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 382
 383             hashel = doc.createElement("hash")
 384             hashel.setAttribute('type', 'sha256')
 385             hashel.appendChild(doc.createTextNode(apk['hash']))
 386             apkel.appendChild(hashel)
 387
 388             addElement('size', str(apk['size']), doc, apkel)
 389             addElementIfInApk('sdkver', apk,
 390                               'minSdkVersion', doc, apkel)
 391             addElementIfInApk('targetSdkVersion', apk,
 392                               'targetSdkVersion', doc, apkel)
 393             addElementIfInApk('maxsdkver', apk,
 394                               'maxSdkVersion', doc, apkel)
 395             addElementIfInApk('obbMainFile', apk,
 396                               'obbMainFile', doc, apkel)
 397             addElementIfInApk('obbMainFileSha256', apk,
 398                               'obbMainFileSha256', doc, apkel)
 399             addElementIfInApk('obbPatchFile', apk,
 400                               'obbPatchFile', doc, apkel)
 401             addElementIfInApk('obbPatchFileSha256', apk,
 402                               'obbPatchFileSha256', doc, apkel)
 403             if 'added' in apk:
 404                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 405
 406             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 407                 addElement('sig', apk['sig'], doc, apkel)
 408
 409                 old_permissions = set()
 410                 sorted_permissions = sorted(apk['uses-permission'])
 411                 for perm in sorted_permissions:
 412                     perm_name = perm.name
 413                     if perm_name.startswith("android.permission."):
 414                         perm_name = perm_name[19:]
 415                     old_permissions.add(perm_name)
 416                 addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
 417
 418                 for permission in sorted_permissions:
 419                     permel = doc.createElement('uses-permission')
 420                     permel.setAttribute('name', permission.name)
 421                     if permission.maxSdkVersion is not None:
 422                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 423                         apkel.appendChild(permel)
 424                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 425                     permel = doc.createElement('uses-permission-sdk-23')
 426                     permel.setAttribute('name', permission_sdk_23.name)
 427                     if permission_sdk_23.maxSdkVersion is not None:
 428                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 429                         apkel.appendChild(permel)
 430                 if 'nativecode' in apk:
 431                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 432                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 433
 434         if current_version_file is not None \
 435                 and config['make_current_version_link'] \
 436                 and repodir == 'repo':  # only create these
 437             namefield = config['current_version_name_source']
 438             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
 439             apklinkname = sanitized_name + '.apk'
 440             current_version_path = os.path.join(repodir, current_version_file)
 441             if os.path.islink(apklinkname):
 442                 os.remove(apklinkname)
 443             os.symlink(current_version_path, apklinkname)
 444             # also symlink gpg signature, if it exists
 445             for extension in ('.asc', '.sig'):
 446                 sigfile_path = current_version_path + extension
 447                 if os.path.exists(sigfile_path):
 448                     siglinkname = apklinkname + extension
 449                     if os.path.islink(siglinkname):
 450                         os.remove(siglinkname)
 451                     os.symlink(sigfile_path, siglinkname)
 452
 453     if options.pretty:
 454         output = doc.toprettyxml(encoding='utf-8')
 455     else:
 456         output = doc.toxml(encoding='utf-8')
 457
 458     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 459         f.write(output)
 460
 461     if 'repo_keyalias' in config:
 462
 463         if options.nosign:
 464             logging.info("Creating unsigned index in preparation for signing")
 465         else:
 466             logging.info("Creating signed index with this key (SHA256):")
 467             logging.info("%s" % repo_pubkey_fingerprint)
 468
 469         # Create a jar of the index...
 470         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 471         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 472         if p.returncode != 0:
 473             logging.critical("Failed to create {0}".format(jar_output))
 474             sys.exit(1)
 475
 476         # Sign the index...
 477         signed = os.path.join(repodir, 'index.jar')
 478         if options.nosign:
 479             # Remove old signed index if not signing
 480             if os.path.exists(signed):
 481                 os.remove(signed)
 482         else:
 483             signindex.config = config
 484             signindex.sign_jar(signed)
 485
 486     # Copy the repo icon into the repo directory...
 487     icon_dir = os.path.join(repodir, 'icons')
 488     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 489     shutil.copyfile(config['repo_icon'], iconfilename)
 490
 491
 492 def extract_pubkey():
 493     """
 494     Extracts and returns the repository's public key from the keystore.
 495     :return: public key in hex, repository fingerprint
 496     """
 497     if 'repo_pubkey' in config:
 498         pubkey = unhexlify(config['repo_pubkey'])
 499     else:
 500         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 501                               '-alias', config['repo_keyalias'],
 502                               '-keystore', config['keystore'],
 503                               '-storepass:file', config['keystorepassfile']]
 504                              + config['smartcardoptions'],
 505                              output=False, stderr_to_stdout=False)
 506         if p.returncode != 0 or len(p.output) < 20:
 507             msg = "Failed to get repo pubkey!"
 508             if config['keystore'] == 'NONE':
 509                 msg += ' Is your crypto smartcard plugged in?'
 510             logging.critical(msg)
 511             sys.exit(1)
 512         pubkey = p.output
 513     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 514     return hexlify(pubkey), repo_pubkey_fingerprint
 515
 516
 517 # Get raw URL from git service for mirroring
 518 def get_raw_mirror(url):
 519     # Divide urls in parts
 520     url = url.split("/")
 521
 522     # Get the hostname
 523     hostname = url[2]
 524
 525     # fdroidserver will use always 'master' branch for git-mirroring
 526     branch = "master"
 527     folder = "fdroid"
 528
 529     if hostname == "github.com":
 530         # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
 531         url[2] = "raw.githubusercontent.com"
 532         url.extend([branch, folder])
 533     elif hostname == "gitlab.com":
 534         # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
 535         url.extend(["raw", branch, folder])
 536     else:
 537         return None
 538
 539     url = "/".join(url)
 540     return url