fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import tempfile
  31 import urllib.parse
  32 import zipfile
  33 from binascii import hexlify, unhexlify
  34 from datetime import datetime
  35 from xml.dom.minidom import Document
  36
  37 from fdroidserver import metadata, signindex, common, net
  38 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
  39 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
  40
  41
  42 def make(apps, sortedids, apks, repodir, archive):
  43     """Generate the repo index files.
  44
  45     This requires properly initialized options and config objects.
  46
  47     :param apps: fully populated apps list
  48     :param sortedids: app package IDs, sorted
  49     :param apks: full populated apks list
  50     :param repodir: the repo directory
  51     :param archive: True if this is the archive repo, False if it's the
  52                     main one.
  53     """
  54     from fdroidserver.update import METADATA_VERSION
  55
  56     def _resolve_description_link(appid):
  57         if appid in apps:
  58             return "fdroid.app:" + appid, apps[appid].Name
  59         raise MetaDataException("Cannot resolve app id " + appid)
  60
  61     nosigningkey = False
  62     if not common.options.nosign:
  63         if 'repo_keyalias' not in common.config:
  64             nosigningkey = True
  65             logging.critical("'repo_keyalias' not found in config.py!")
  66         if 'keystore' not in common.config:
  67             nosigningkey = True
  68             logging.critical("'keystore' not found in config.py!")
  69         if 'keystorepass' not in common.config:
  70             nosigningkey = True
  71             logging.critical("'keystorepass' not found in config.py!")
  72         if 'keypass' not in common.config:
  73             nosigningkey = True
  74             logging.critical("'keypass' not found in config.py!")
  75         if not os.path.exists(common.config['keystore']):
  76             nosigningkey = True
  77             logging.critical("'" + common.config['keystore'] + "' does not exist!")
  78         if nosigningkey:
  79             raise FDroidException("`fdroid update` requires a signing key, " +
  80                                   "you can create one using: fdroid update --create-key")
  81
  82     repodict = collections.OrderedDict()
  83     repodict['timestamp'] = datetime.utcnow()
  84     repodict['version'] = METADATA_VERSION
  85
  86     if common.config['repo_maxage'] != 0:
  87         repodict['maxage'] = common.config['repo_maxage']
  88
  89     if archive:
  90         repodict['name'] = common.config['archive_name']
  91         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  92         repodict['address'] = common.config['archive_url']
  93         repodict['description'] = common.config['archive_description']
  94         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
  95     else:
  96         repodict['name'] = common.config['repo_name']
  97         repodict['icon'] = os.path.basename(common.config['repo_icon'])
  98         repodict['address'] = common.config['repo_url']
  99         repodict['description'] = common.config['repo_description']
 100         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
 101
 102     mirrorcheckfailed = False
 103     mirrors = []
 104     for mirror in sorted(common.config.get('mirrors', [])):
 105         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 106         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
 107             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 108             mirrorcheckfailed = True
 109         # must end with / or urljoin strips a whole path segment
 110         if mirror.endswith('/'):
 111             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
 112         else:
 113             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 114     for mirror in common.config.get('servergitmirrors', []):
 115         for url in get_mirror_service_urls(mirror):
 116             mirrors.append(url + '/' + repodir)
 117     if mirrorcheckfailed:
 118         raise FDroidException("Malformed repository mirrors.")
 119     if mirrors:
 120         repodict['mirrors'] = mirrors
 121
 122     appsWithPackages = collections.OrderedDict()
 123     for packageName in sortedids:
 124         app = apps[packageName]
 125         if app['Disabled']:
 126             continue
 127
 128         # only include apps with packages
 129         for apk in apks:
 130             if apk['packageName'] == packageName:
 131                 newapp = copy.copy(app)  # update wiki needs unmodified description
 132                 newapp['Description'] = metadata.description_html(app['Description'],
 133                                                                   _resolve_description_link)
 134                 appsWithPackages[packageName] = newapp
 135                 break
 136
 137     requestsdict = collections.OrderedDict()
 138     for command in ('install', 'uninstall'):
 139         packageNames = []
 140         key = command + '_list'
 141         if key in common.config:
 142             if isinstance(common.config[key], str):
 143                 packageNames = [common.config[key]]
 144             elif all(isinstance(item, str) for item in common.config[key]):
 145                 packageNames = common.config[key]
 146             else:
 147                 raise TypeError('only accepts strings, lists, and tuples')
 148         requestsdict[command] = packageNames
 149
 150     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
 151     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
 152
 153
 154 def make_v1(apps, packages, repodir, repodict, requestsdict):
 155
 156     def _index_encoder_default(obj):
 157         if isinstance(obj, set):
 158             return list(obj)
 159         if isinstance(obj, datetime):
 160             return int(obj.timestamp() * 1000)  # Java expects milliseconds
 161         raise TypeError(repr(obj) + " is not JSON serializable")
 162
 163     output = collections.OrderedDict()
 164     output['repo'] = repodict
 165     output['requests'] = requestsdict
 166
 167     appslist = []
 168     output['apps'] = appslist
 169     for packageName, appdict in apps.items():
 170         d = collections.OrderedDict()
 171         appslist.append(d)
 172         for k, v in sorted(appdict.items()):
 173             if not v:
 174                 continue
 175             if k in ('builds', 'comments', 'metadatapath',
 176                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 177                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 178                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 179                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 180                 continue
 181
 182             # name things after the App class fields in fdroidclient
 183             if k == 'id':
 184                 k = 'packageName'
 185             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 186                 k = 'suggestedVersionCode'
 187             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 188                 k = 'suggestedVersionName'
 189             elif k == 'AutoName':
 190                 if 'Name' not in apps[packageName]:
 191                     d['name'] = v
 192                 continue
 193             else:
 194                 k = k[:1].lower() + k[1:]
 195             d[k] = v
 196
 197     output_packages = collections.OrderedDict()
 198     output['packages'] = output_packages
 199     for package in packages:
 200         packageName = package['packageName']
 201         if packageName not in apps:
 202             logging.info('Ignoring package without metadata: ' + package['apkName'])
 203             continue
 204         if packageName in output_packages:
 205             packagelist = output_packages[packageName]
 206         else:
 207             packagelist = []
 208             output_packages[packageName] = packagelist
 209         d = collections.OrderedDict()
 210         packagelist.append(d)
 211         for k, v in sorted(package.items()):
 212             if not v:
 213                 continue
 214             if k in ('icon', 'icons', 'icons_src', 'name', ):
 215                 continue
 216             d[k] = v
 217
 218     json_name = 'index-v1.json'
 219     index_file = os.path.join(repodir, json_name)
 220     with open(index_file, 'w') as fp:
 221         if common.options.pretty:
 222             json.dump(output, fp, default=_index_encoder_default, indent=2)
 223         else:
 224             json.dump(output, fp, default=_index_encoder_default)
 225
 226     if common.options.nosign:
 227         logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
 228     else:
 229         signindex.config = common.config
 230         signindex.sign_index_v1(repodir, json_name)
 231
 232
 233 def make_v0(apps, apks, repodir, repodict, requestsdict):
 234     """
 235     aka index.jar aka index.xml
 236     """
 237
 238     doc = Document()
 239
 240     def addElement(name, value, doc, parent):
 241         el = doc.createElement(name)
 242         el.appendChild(doc.createTextNode(value))
 243         parent.appendChild(el)
 244
 245     def addElementNonEmpty(name, value, doc, parent):
 246         if not value:
 247             return
 248         addElement(name, value, doc, parent)
 249
 250     def addElementIfInApk(name, apk, key, doc, parent):
 251         if key not in apk:
 252             return
 253         value = str(apk[key])
 254         addElement(name, value, doc, parent)
 255
 256     def addElementCDATA(name, value, doc, parent):
 257         el = doc.createElement(name)
 258         el.appendChild(doc.createCDATASection(value))
 259         parent.appendChild(el)
 260
 261     root = doc.createElement("fdroid")
 262     doc.appendChild(root)
 263
 264     repoel = doc.createElement("repo")
 265
 266     repoel.setAttribute("name", repodict['name'])
 267     if 'maxage' in repodict:
 268         repoel.setAttribute("maxage", str(repodict['maxage']))
 269     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 270     repoel.setAttribute("url", repodict['address'])
 271     addElement('description', repodict['description'], doc, repoel)
 272     for mirror in repodict.get('mirrors', []):
 273         addElement('mirror', mirror, doc, repoel)
 274
 275     repoel.setAttribute("version", str(repodict['version']))
 276     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 277
 278     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 279     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 280     root.appendChild(repoel)
 281
 282     for command in ('install', 'uninstall'):
 283         for packageName in requestsdict[command]:
 284             element = doc.createElement(command)
 285             root.appendChild(element)
 286             element.setAttribute('packageName', packageName)
 287
 288     for appid, appdict in apps.items():
 289         app = metadata.App(appdict)
 290
 291         if app.Disabled is not None:
 292             continue
 293
 294         # Get a list of the apks for this app...
 295         apklist = []
 296         versionCodes = []
 297         for apk in apks:
 298             if apk['packageName'] == appid:
 299                 if apk['versionCode'] not in versionCodes:
 300                     apklist.append(apk)
 301                     versionCodes.append(apk['versionCode'])
 302
 303         if len(apklist) == 0:
 304             continue
 305
 306         apel = doc.createElement("application")
 307         apel.setAttribute("id", app.id)
 308         root.appendChild(apel)
 309
 310         addElement('id', app.id, doc, apel)
 311         if app.added:
 312             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 313         if app.lastUpdated:
 314             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 315         addElement('name', app.Name, doc, apel)
 316         addElement('summary', app.Summary, doc, apel)
 317         if app.icon:
 318             addElement('icon', app.icon, doc, apel)
 319
 320         if app.get('Description'):
 321             description = app.Description
 322         else:
 323             description = '<p>No description available</p>'
 324         addElement('desc', description, doc, apel)
 325         addElement('license', app.License, doc, apel)
 326         if app.Categories:
 327             addElement('categories', ','.join(app.Categories), doc, apel)
 328             # We put the first (primary) category in LAST, which will have
 329             # the desired effect of making clients that only understand one
 330             # category see that one.
 331             addElement('category', app.Categories[0], doc, apel)
 332         addElement('web', app.WebSite, doc, apel)
 333         addElement('source', app.SourceCode, doc, apel)
 334         addElement('tracker', app.IssueTracker, doc, apel)
 335         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 336         addElementNonEmpty('author', app.AuthorName, doc, apel)
 337         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 338         addElementNonEmpty('donate', app.Donate, doc, apel)
 339         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 340         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 341         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 342
 343         # These elements actually refer to the current version (i.e. which
 344         # one is recommended. They are historically mis-named, and need
 345         # changing, but stay like this for now to support existing clients.
 346         addElement('marketversion', app.CurrentVersion, doc, apel)
 347         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 348
 349         if app.Provides:
 350             pv = app.Provides.split(',')
 351             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 352         if app.RequiresRoot:
 353             addElement('requirements', 'root', doc, apel)
 354
 355         # Sort the apk list into version order, just so the web site
 356         # doesn't have to do any work by default...
 357         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 358
 359         if 'antiFeatures' in apklist[0]:
 360             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 361         if app.AntiFeatures:
 362             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 363
 364         # Check for duplicates - they will make the client unhappy...
 365         for i in range(len(apklist) - 1):
 366             first = apklist[i]
 367             second = apklist[i + 1]
 368             if first['versionCode'] == second['versionCode'] \
 369                and first['sig'] == second['sig']:
 370                 if first['hash'] == second['hash']:
 371                     raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
 372                         repodir, first['apkName'], second['apkName']))
 373                 else:
 374                     raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
 375                         repodir, first['apkName'], second['apkName']))
 376
 377         current_version_code = 0
 378         current_version_file = None
 379         for apk in apklist:
 380             file_extension = common.get_file_extension(apk['apkName'])
 381             # find the APK for the "Current Version"
 382             if current_version_code < apk['versionCode']:
 383                 current_version_code = apk['versionCode']
 384             if current_version_code < int(app.CurrentVersionCode):
 385                 current_version_file = apk['apkName']
 386
 387             apkel = doc.createElement("package")
 388             apel.appendChild(apkel)
 389             addElement('version', apk['versionName'], doc, apkel)
 390             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 391             addElement('apkname', apk['apkName'], doc, apkel)
 392             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 393
 394             hashel = doc.createElement("hash")
 395             hashel.setAttribute('type', 'sha256')
 396             hashel.appendChild(doc.createTextNode(apk['hash']))
 397             apkel.appendChild(hashel)
 398
 399             addElement('size', str(apk['size']), doc, apkel)
 400             addElementIfInApk('sdkver', apk,
 401                               'minSdkVersion', doc, apkel)
 402             addElementIfInApk('targetSdkVersion', apk,
 403                               'targetSdkVersion', doc, apkel)
 404             addElementIfInApk('maxsdkver', apk,
 405                               'maxSdkVersion', doc, apkel)
 406             addElementIfInApk('obbMainFile', apk,
 407                               'obbMainFile', doc, apkel)
 408             addElementIfInApk('obbMainFileSha256', apk,
 409                               'obbMainFileSha256', doc, apkel)
 410             addElementIfInApk('obbPatchFile', apk,
 411                               'obbPatchFile', doc, apkel)
 412             addElementIfInApk('obbPatchFileSha256', apk,
 413                               'obbPatchFileSha256', doc, apkel)
 414             if 'added' in apk:
 415                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 416
 417             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 418                 addElement('sig', apk['sig'], doc, apkel)
 419
 420                 old_permissions = set()
 421                 sorted_permissions = sorted(apk['uses-permission'])
 422                 for perm in sorted_permissions:
 423                     perm_name = perm.name
 424                     if perm_name.startswith("android.permission."):
 425                         perm_name = perm_name[19:]
 426                     old_permissions.add(perm_name)
 427                 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 428
 429                 for permission in sorted_permissions:
 430                     permel = doc.createElement('uses-permission')
 431                     permel.setAttribute('name', permission.name)
 432                     if permission.maxSdkVersion is not None:
 433                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 434                         apkel.appendChild(permel)
 435                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 436                     permel = doc.createElement('uses-permission-sdk-23')
 437                     permel.setAttribute('name', permission_sdk_23.name)
 438                     if permission_sdk_23.maxSdkVersion is not None:
 439                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 440                         apkel.appendChild(permel)
 441                 if 'nativecode' in apk:
 442                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 443                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 444
 445         if current_version_file is not None \
 446                 and common.config['make_current_version_link'] \
 447                 and repodir == 'repo':  # only create these
 448             namefield = common.config['current_version_name_source']
 449             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
 450             apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
 451             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
 452             if os.path.islink(apklinkname):
 453                 os.remove(apklinkname)
 454             os.symlink(current_version_path, apklinkname)
 455             # also symlink gpg signature, if it exists
 456             for extension in (b'.asc', b'.sig'):
 457                 sigfile_path = current_version_path + extension
 458                 if os.path.exists(sigfile_path):
 459                     siglinkname = apklinkname + extension
 460                     if os.path.islink(siglinkname):
 461                         os.remove(siglinkname)
 462                     os.symlink(sigfile_path, siglinkname)
 463
 464     if common.options.pretty:
 465         output = doc.toprettyxml(encoding='utf-8')
 466     else:
 467         output = doc.toxml(encoding='utf-8')
 468
 469     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 470         f.write(output)
 471
 472     if 'repo_keyalias' in common.config:
 473
 474         if common.options.nosign:
 475             logging.info("Creating unsigned index in preparation for signing")
 476         else:
 477             logging.info("Creating signed index with this key (SHA256):")
 478             logging.info("%s" % repo_pubkey_fingerprint)
 479
 480         # Create a jar of the index...
 481         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 482         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 483         if p.returncode != 0:
 484             raise FDroidException("Failed to create {0}".format(jar_output))
 485
 486         # Sign the index...
 487         signed = os.path.join(repodir, 'index.jar')
 488         if common.options.nosign:
 489             # Remove old signed index if not signing
 490             if os.path.exists(signed):
 491                 os.remove(signed)
 492         else:
 493             signindex.config = common.config
 494             signindex.sign_jar(signed)
 495
 496     # Copy the repo icon into the repo directory...
 497     icon_dir = os.path.join(repodir, 'icons')
 498     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 499     shutil.copyfile(common.config['repo_icon'], iconfilename)
 500
 501
 502 def extract_pubkey():
 503     """
 504     Extracts and returns the repository's public key from the keystore.
 505     :return: public key in hex, repository fingerprint
 506     """
 507     if 'repo_pubkey' in common.config:
 508         pubkey = unhexlify(common.config['repo_pubkey'])
 509     else:
 510         env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
 511         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 512                               '-alias', common.config['repo_keyalias'],
 513                               '-keystore', common.config['keystore'],
 514                               '-storepass:env', 'FDROID_KEY_STORE_PASS']
 515                              + common.config['smartcardoptions'],
 516                              envs=env_vars, output=False, stderr_to_stdout=False)
 517         if p.returncode != 0 or len(p.output) < 20:
 518             msg = "Failed to get repo pubkey!"
 519             if common.config['keystore'] == 'NONE':
 520                 msg += ' Is your crypto smartcard plugged in?'
 521             raise FDroidException(msg)
 522         pubkey = p.output
 523     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 524     return hexlify(pubkey), repo_pubkey_fingerprint
 525
 526
 527 def get_mirror_service_urls(url):
 528     '''Get direct URLs from git service for use by fdroidclient
 529
 530     Via 'servergitmirrors', fdroidserver can create and push a mirror
 531     to certain well known git services like gitlab or github.  This
 532     will always use the 'master' branch since that is the default
 533     branch in git. The files are then accessible via alternate URLs,
 534     where they are served in their raw format via a CDN rather than
 535     from git.
 536     '''
 537
 538     if url.startswith('git@'):
 539         url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 540
 541     segments = url.split("/")
 542
 543     if segments[4].endswith('.git'):
 544         segments[4] = segments[4][:-4]
 545
 546     hostname = segments[2]
 547     user = segments[3]
 548     repo = segments[4]
 549     branch = "master"
 550     folder = "fdroid"
 551
 552     urls = []
 553     if hostname == "github.com":
 554         # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
 555         segments[2] = "raw.githubusercontent.com"
 556         segments.extend([branch, folder])
 557         urls.append('/'.join(segments))
 558     elif hostname == "gitlab.com":
 559         # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
 560         gitlab_raw = segments + ['raw', branch, folder]
 561         urls.append('/'.join(gitlab_raw))
 562         # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
 563         gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
 564         urls.append('/'.join(gitlab_pages))
 565         return urls
 566
 567     return urls
 568
 569
 570
 571 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
 572     """
 573     Downloads the repository index from the given :param url_str
 574     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 575
 576     :raises: VerificationException() if the repository could not be verified
 577
 578     :return: A tuple consisting of:
 579         - The index in JSON format or None if the index did not change
 580         - The new eTag as returned by the HTTP request
 581     """
 582     url = urllib.parse.urlsplit(url_str)
 583
 584     fingerprint = None
 585     if verify_fingerprint:
 586         query = urllib.parse.parse_qs(url.query)
 587         if 'fingerprint' not in query:
 588             raise VerificationException("No fingerprint in URL.")
 589         fingerprint = query['fingerprint'][0]
 590
 591     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
 592     download, new_etag = net.http_get(url.geturl(), etag)
 593
 594     if download is None:
 595         return None, new_etag
 596
 597     with tempfile.NamedTemporaryFile() as fp:
 598         # write and open JAR file
 599         fp.write(download)
 600         jar = zipfile.ZipFile(fp)
 601
 602         # verify that the JAR signature is valid
 603         verify_jar_signature(fp.name)
 604
 605         # get public key and its fingerprint from JAR
 606         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 607
 608         # compare the fingerprint if verify_fingerprint is True
 609         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
 610             raise VerificationException("The repository's fingerprint does not match.")
 611
 612         # load repository index from JSON
 613         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
 614         index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
 615         index["repo"]["fingerprint"] = public_key_fingerprint
 616
 617         # turn the apps into App objects
 618         index["apps"] = [metadata.App(app) for app in index["apps"]]
 619
 620         return index, new_etag
 621
 622
 623 def verify_jar_signature(file):
 624     """
 625     Verifies the signature of a given JAR file.
 626
 627     :raises: VerificationException() if the JAR's signature could not be verified
 628     """
 629     if not common.verify_apk_signature(file, jar=True):
 630         raise VerificationException("The repository's index could not be verified.")
 631
 632
 633 def get_public_key_from_jar(jar):
 634     """
 635     Get the public key and its fingerprint from a JAR file.
 636
 637     :raises: VerificationException() if the JAR was not signed exactly once
 638
 639     :param jar: a zipfile.ZipFile object
 640     :return: the public key from the jar and its fingerprint
 641     """
 642     # extract certificate from jar
 643     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
 644     if len(certs) < 1:
 645         raise VerificationException("Found no signing certificates for repository.")
 646     if len(certs) > 1:
 647         raise VerificationException("Found multiple signing certificates for repository.")
 648
 649     # extract public key from certificate
 650     public_key = common.get_certificate(jar.read(certs[0]))
 651     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 652
 653     return public_key, public_key_fingerprint