fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import tempfile
  31 import urllib.parse
  32 import zipfile
  33 from binascii import hexlify, unhexlify
  34 from datetime import datetime
  35 from xml.dom.minidom import Document
  36
  37 from fdroidserver import metadata, signindex, common, net
  38 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
  39 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
  40
  41
  42 def make(apps, sortedids, apks, repodir, archive):
  43     """Generate the repo index files.
  44
  45     This requires properly initialized options and config objects.
  46
  47     :param apps: fully populated apps list
  48     :param sortedids: app package IDs, sorted
  49     :param apks: full populated apks list
  50     :param repodir: the repo directory
  51     :param archive: True if this is the archive repo, False if it's the
  52                     main one.
  53     """
  54     from fdroidserver.update import METADATA_VERSION
  55
  56     def _resolve_description_link(appid):
  57         if appid in apps:
  58             return "fdroid.app:" + appid, apps[appid].Name
  59         raise MetaDataException("Cannot resolve app id " + appid)
  60
  61     nosigningkey = False
  62     if not common.options.nosign:
  63         if 'repo_keyalias' not in common.config:
  64             nosigningkey = True
  65             logging.critical("'repo_keyalias' not found in config.py!")
  66         if 'keystore' not in common.config:
  67             nosigningkey = True
  68             logging.critical("'keystore' not found in config.py!")
  69         if 'keystorepass' not in common.config:
  70             nosigningkey = True
  71             logging.critical("'keystorepass' not found in config.py!")
  72         if 'keypass' not in common.config:
  73             nosigningkey = True
  74             logging.critical("'keypass' not found in config.py!")
  75         if not os.path.exists(common.config['keystore']):
  76             nosigningkey = True
  77             logging.critical("'" + common.config['keystore'] + "' does not exist!")
  78         if nosigningkey:
  79             raise FDroidException("`fdroid update` requires a signing key, " +
  80                                   "you can create one using: fdroid update --create-key")
  81
  82     repodict = collections.OrderedDict()
  83     repodict['timestamp'] = datetime.utcnow()
  84     repodict['version'] = METADATA_VERSION
  85
  86     if common.config['repo_maxage'] != 0:
  87         repodict['maxage'] = common.config['repo_maxage']
  88
  89     if archive:
  90         repodict['name'] = common.config['archive_name']
  91         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  92         repodict['address'] = common.config['archive_url']
  93         repodict['description'] = common.config['archive_description']
  94         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
  95     else:
  96         repodict['name'] = common.config['repo_name']
  97         repodict['icon'] = os.path.basename(common.config['repo_icon'])
  98         repodict['address'] = common.config['repo_url']
  99         repodict['description'] = common.config['repo_description']
 100         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
 101
 102     mirrorcheckfailed = False
 103     mirrors = []
 104     for mirror in sorted(common.config.get('mirrors', [])):
 105         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 106         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
 107             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 108             mirrorcheckfailed = True
 109         # must end with / or urljoin strips a whole path segment
 110         if mirror.endswith('/'):
 111             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
 112         else:
 113             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 114     for mirror in common.config.get('servergitmirrors', []):
 115         mirror = get_mirror_service_url(mirror)
 116         if mirror is not None:
 117             mirrors.append(mirror + '/')
 118     if mirrorcheckfailed:
 119         raise FDroidException("Malformed repository mirrors.")
 120     if mirrors:
 121         repodict['mirrors'] = mirrors
 122
 123     appsWithPackages = collections.OrderedDict()
 124     for packageName in sortedids:
 125         app = apps[packageName]
 126         if app['Disabled']:
 127             continue
 128
 129         # only include apps with packages
 130         for apk in apks:
 131             if apk['packageName'] == packageName:
 132                 newapp = copy.copy(app)  # update wiki needs unmodified description
 133                 newapp['Description'] = metadata.description_html(app['Description'],
 134                                                                   _resolve_description_link)
 135                 appsWithPackages[packageName] = newapp
 136                 break
 137
 138     requestsdict = collections.OrderedDict()
 139     for command in ('install', 'uninstall'):
 140         packageNames = []
 141         key = command + '_list'
 142         if key in common.config:
 143             if isinstance(common.config[key], str):
 144                 packageNames = [common.config[key]]
 145             elif all(isinstance(item, str) for item in common.config[key]):
 146                 packageNames = common.config[key]
 147             else:
 148                 raise TypeError('only accepts strings, lists, and tuples')
 149         requestsdict[command] = packageNames
 150
 151     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
 152     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
 153
 154
 155 def make_v1(apps, packages, repodir, repodict, requestsdict):
 156
 157     def _index_encoder_default(obj):
 158         if isinstance(obj, set):
 159             return list(obj)
 160         if isinstance(obj, datetime):
 161             return int(obj.timestamp() * 1000)  # Java expects milliseconds
 162         raise TypeError(repr(obj) + " is not JSON serializable")
 163
 164     output = collections.OrderedDict()
 165     output['repo'] = repodict
 166     output['requests'] = requestsdict
 167
 168     appslist = []
 169     output['apps'] = appslist
 170     for packageName, appdict in apps.items():
 171         d = collections.OrderedDict()
 172         appslist.append(d)
 173         for k, v in sorted(appdict.items()):
 174             if not v:
 175                 continue
 176             if k in ('builds', 'comments', 'metadatapath',
 177                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 178                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 179                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 180                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 181                 continue
 182
 183             # name things after the App class fields in fdroidclient
 184             if k == 'id':
 185                 k = 'packageName'
 186             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 187                 k = 'suggestedVersionCode'
 188             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 189                 k = 'suggestedVersionName'
 190             elif k == 'AutoName':
 191                 if 'Name' not in apps[packageName]:
 192                     d['name'] = v
 193                 continue
 194             else:
 195                 k = k[:1].lower() + k[1:]
 196             d[k] = v
 197
 198     output_packages = collections.OrderedDict()
 199     output['packages'] = output_packages
 200     for package in packages:
 201         packageName = package['packageName']
 202         if packageName not in apps:
 203             logging.info('Ignoring package without metadata: ' + package['apkName'])
 204             continue
 205         if packageName in output_packages:
 206             packagelist = output_packages[packageName]
 207         else:
 208             packagelist = []
 209             output_packages[packageName] = packagelist
 210         d = collections.OrderedDict()
 211         packagelist.append(d)
 212         for k, v in sorted(package.items()):
 213             if not v:
 214                 continue
 215             if k in ('icon', 'icons', 'icons_src', 'name', ):
 216                 continue
 217             d[k] = v
 218
 219     json_name = 'index-v1.json'
 220     index_file = os.path.join(repodir, json_name)
 221     with open(index_file, 'w') as fp:
 222         if common.options.pretty:
 223             json.dump(output, fp, default=_index_encoder_default, indent=2)
 224         else:
 225             json.dump(output, fp, default=_index_encoder_default)
 226
 227     if common.options.nosign:
 228         logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
 229     else:
 230         signindex.config = common.config
 231         signindex.sign_index_v1(repodir, json_name)
 232
 233
 234 def make_v0(apps, apks, repodir, repodict, requestsdict):
 235     """
 236     aka index.jar aka index.xml
 237     """
 238
 239     doc = Document()
 240
 241     def addElement(name, value, doc, parent):
 242         el = doc.createElement(name)
 243         el.appendChild(doc.createTextNode(value))
 244         parent.appendChild(el)
 245
 246     def addElementNonEmpty(name, value, doc, parent):
 247         if not value:
 248             return
 249         addElement(name, value, doc, parent)
 250
 251     def addElementIfInApk(name, apk, key, doc, parent):
 252         if key not in apk:
 253             return
 254         value = str(apk[key])
 255         addElement(name, value, doc, parent)
 256
 257     def addElementCDATA(name, value, doc, parent):
 258         el = doc.createElement(name)
 259         el.appendChild(doc.createCDATASection(value))
 260         parent.appendChild(el)
 261
 262     root = doc.createElement("fdroid")
 263     doc.appendChild(root)
 264
 265     repoel = doc.createElement("repo")
 266
 267     repoel.setAttribute("name", repodict['name'])
 268     if 'maxage' in repodict:
 269         repoel.setAttribute("maxage", str(repodict['maxage']))
 270     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 271     repoel.setAttribute("url", repodict['address'])
 272     addElement('description', repodict['description'], doc, repoel)
 273     for mirror in repodict.get('mirrors', []):
 274         addElement('mirror', mirror, doc, repoel)
 275
 276     repoel.setAttribute("version", str(repodict['version']))
 277     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 278
 279     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 280     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 281     root.appendChild(repoel)
 282
 283     for command in ('install', 'uninstall'):
 284         for packageName in requestsdict[command]:
 285             element = doc.createElement(command)
 286             root.appendChild(element)
 287             element.setAttribute('packageName', packageName)
 288
 289     for appid, appdict in apps.items():
 290         app = metadata.App(appdict)
 291
 292         if app.Disabled is not None:
 293             continue
 294
 295         # Get a list of the apks for this app...
 296         apklist = []
 297         for apk in apks:
 298             if apk['packageName'] == appid:
 299                 apklist.append(apk)
 300
 301         if len(apklist) == 0:
 302             continue
 303
 304         apel = doc.createElement("application")
 305         apel.setAttribute("id", app.id)
 306         root.appendChild(apel)
 307
 308         addElement('id', app.id, doc, apel)
 309         if app.added:
 310             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 311         if app.lastUpdated:
 312             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 313         addElement('name', app.Name, doc, apel)
 314         addElement('summary', app.Summary, doc, apel)
 315         if app.icon:
 316             addElement('icon', app.icon, doc, apel)
 317
 318         if app.get('Description'):
 319             description = app.Description
 320         else:
 321             description = '<p>No description available</p>'
 322         addElement('desc', description, doc, apel)
 323         addElement('license', app.License, doc, apel)
 324         if app.Categories:
 325             addElement('categories', ','.join(app.Categories), doc, apel)
 326             # We put the first (primary) category in LAST, which will have
 327             # the desired effect of making clients that only understand one
 328             # category see that one.
 329             addElement('category', app.Categories[0], doc, apel)
 330         addElement('web', app.WebSite, doc, apel)
 331         addElement('source', app.SourceCode, doc, apel)
 332         addElement('tracker', app.IssueTracker, doc, apel)
 333         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 334         addElementNonEmpty('author', app.AuthorName, doc, apel)
 335         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 336         addElementNonEmpty('donate', app.Donate, doc, apel)
 337         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 338         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 339         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 340
 341         # These elements actually refer to the current version (i.e. which
 342         # one is recommended. They are historically mis-named, and need
 343         # changing, but stay like this for now to support existing clients.
 344         addElement('marketversion', app.CurrentVersion, doc, apel)
 345         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 346
 347         if app.Provides:
 348             pv = app.Provides.split(',')
 349             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 350         if app.RequiresRoot:
 351             addElement('requirements', 'root', doc, apel)
 352
 353         # Sort the apk list into version order, just so the web site
 354         # doesn't have to do any work by default...
 355         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 356
 357         if 'antiFeatures' in apklist[0]:
 358             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 359         if app.AntiFeatures:
 360             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 361
 362         # Check for duplicates - they will make the client unhappy...
 363         for i in range(len(apklist) - 1):
 364             first = apklist[i]
 365             second = apklist[i + 1]
 366             if first['versionCode'] == second['versionCode'] \
 367                and first['sig'] == second['sig']:
 368                 if first['hash'] == second['hash']:
 369                     raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
 370                         repodir, first['apkName'], second['apkName']))
 371                 else:
 372                     raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
 373                         repodir, first['apkName'], second['apkName']))
 374
 375         current_version_code = 0
 376         current_version_file = None
 377         for apk in apklist:
 378             file_extension = common.get_file_extension(apk['apkName'])
 379             # find the APK for the "Current Version"
 380             if current_version_code < apk['versionCode']:
 381                 current_version_code = apk['versionCode']
 382             if current_version_code < int(app.CurrentVersionCode):
 383                 current_version_file = apk['apkName']
 384
 385             apkel = doc.createElement("package")
 386             apel.appendChild(apkel)
 387             addElement('version', apk['versionName'], doc, apkel)
 388             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 389             addElement('apkname', apk['apkName'], doc, apkel)
 390             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 391
 392             hashel = doc.createElement("hash")
 393             hashel.setAttribute('type', 'sha256')
 394             hashel.appendChild(doc.createTextNode(apk['hash']))
 395             apkel.appendChild(hashel)
 396
 397             addElement('size', str(apk['size']), doc, apkel)
 398             addElementIfInApk('sdkver', apk,
 399                               'minSdkVersion', doc, apkel)
 400             addElementIfInApk('targetSdkVersion', apk,
 401                               'targetSdkVersion', doc, apkel)
 402             addElementIfInApk('maxsdkver', apk,
 403                               'maxSdkVersion', doc, apkel)
 404             addElementIfInApk('obbMainFile', apk,
 405                               'obbMainFile', doc, apkel)
 406             addElementIfInApk('obbMainFileSha256', apk,
 407                               'obbMainFileSha256', doc, apkel)
 408             addElementIfInApk('obbPatchFile', apk,
 409                               'obbPatchFile', doc, apkel)
 410             addElementIfInApk('obbPatchFileSha256', apk,
 411                               'obbPatchFileSha256', doc, apkel)
 412             if 'added' in apk:
 413                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 414
 415             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 416                 addElement('sig', apk['sig'], doc, apkel)
 417
 418                 old_permissions = set()
 419                 sorted_permissions = sorted(apk['uses-permission'])
 420                 for perm in sorted_permissions:
 421                     perm_name = perm.name
 422                     if perm_name.startswith("android.permission."):
 423                         perm_name = perm_name[19:]
 424                     old_permissions.add(perm_name)
 425                 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 426
 427                 for permission in sorted_permissions:
 428                     permel = doc.createElement('uses-permission')
 429                     permel.setAttribute('name', permission.name)
 430                     if permission.maxSdkVersion is not None:
 431                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 432                         apkel.appendChild(permel)
 433                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 434                     permel = doc.createElement('uses-permission-sdk-23')
 435                     permel.setAttribute('name', permission_sdk_23.name)
 436                     if permission_sdk_23.maxSdkVersion is not None:
 437                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 438                         apkel.appendChild(permel)
 439                 if 'nativecode' in apk:
 440                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 441                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 442
 443         if current_version_file is not None \
 444                 and common.config['make_current_version_link'] \
 445                 and repodir == 'repo':  # only create these
 446             namefield = common.config['current_version_name_source']
 447             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
 448             apklinkname = sanitized_name + b'.apk'
 449             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
 450             if os.path.islink(apklinkname):
 451                 os.remove(apklinkname)
 452             os.symlink(current_version_path, apklinkname)
 453             # also symlink gpg signature, if it exists
 454             for extension in (b'.asc', b'.sig'):
 455                 sigfile_path = current_version_path + extension
 456                 if os.path.exists(sigfile_path):
 457                     siglinkname = apklinkname + extension
 458                     if os.path.islink(siglinkname):
 459                         os.remove(siglinkname)
 460                     os.symlink(sigfile_path, siglinkname)
 461
 462     if common.options.pretty:
 463         output = doc.toprettyxml(encoding='utf-8')
 464     else:
 465         output = doc.toxml(encoding='utf-8')
 466
 467     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 468         f.write(output)
 469
 470     if 'repo_keyalias' in common.config:
 471
 472         if common.options.nosign:
 473             logging.info("Creating unsigned index in preparation for signing")
 474         else:
 475             logging.info("Creating signed index with this key (SHA256):")
 476             logging.info("%s" % repo_pubkey_fingerprint)
 477
 478         # Create a jar of the index...
 479         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 480         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 481         if p.returncode != 0:
 482             raise FDroidException("Failed to create {0}".format(jar_output))
 483
 484         # Sign the index...
 485         signed = os.path.join(repodir, 'index.jar')
 486         if common.options.nosign:
 487             # Remove old signed index if not signing
 488             if os.path.exists(signed):
 489                 os.remove(signed)
 490         else:
 491             signindex.config = common.config
 492             signindex.sign_jar(signed)
 493
 494     # Copy the repo icon into the repo directory...
 495     icon_dir = os.path.join(repodir, 'icons')
 496     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 497     shutil.copyfile(common.config['repo_icon'], iconfilename)
 498
 499
 500 def extract_pubkey():
 501     """
 502     Extracts and returns the repository's public key from the keystore.
 503     :return: public key in hex, repository fingerprint
 504     """
 505     if 'repo_pubkey' in common.config:
 506         pubkey = unhexlify(common.config['repo_pubkey'])
 507     else:
 508         env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
 509         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 510                               '-alias', common.config['repo_keyalias'],
 511                               '-keystore', common.config['keystore'],
 512                               '-storepass:env', 'FDROID_KEY_STORE_PASS']
 513                              + common.config['smartcardoptions'],
 514                              envs=env_vars, output=False, stderr_to_stdout=False)
 515         if p.returncode != 0 or len(p.output) < 20:
 516             msg = "Failed to get repo pubkey!"
 517             if common.config['keystore'] == 'NONE':
 518                 msg += ' Is your crypto smartcard plugged in?'
 519             raise FDroidException(msg)
 520         pubkey = p.output
 521     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 522     return hexlify(pubkey), repo_pubkey_fingerprint
 523
 524
 525 def get_mirror_service_url(url):
 526     '''Get direct URL from git service for use by fdroidclient
 527
 528     Via 'servergitmirrors', fdroidserver can create and push a mirror
 529     to certain well known git services like gitlab or github.  This
 530     will always use the 'master' branch since that is the default
 531     branch in git.
 532
 533     '''
 534
 535     if url.startswith('git@'):
 536         url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 537
 538     segments = url.split("/")
 539
 540     if segments[4].endswith('.git'):
 541         segments[4] = segments[4][:-4]
 542
 543     hostname = segments[2]
 544     user = segments[3]
 545     repo = segments[4]
 546     branch = "master"
 547     folder = "fdroid"
 548
 549     if hostname == "github.com":
 550         # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/master/fdroid"
 551         segments[2] = "raw.githubusercontent.com"
 552         segments.extend([branch, folder])
 553     elif hostname == "gitlab.com":
 554         # Gitlab-like Pages segments "https://user.gitlab.com/repo/fdroid"
 555         gitlab_url = ["https:", "", user + ".gitlab.io", repo, folder]
 556         segments = gitlab_url
 557     else:
 558         return None
 559
 560     return '/'.join(segments)
 561
 562
 563 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
 564     """
 565     Downloads the repository index from the given :param url_str
 566     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 567
 568     :raises: VerificationException() if the repository could not be verified
 569
 570     :return: A tuple consisting of:
 571         - The index in JSON format or None if the index did not change
 572         - The new eTag as returned by the HTTP request
 573     """
 574     url = urllib.parse.urlsplit(url_str)
 575
 576     fingerprint = None
 577     if verify_fingerprint:
 578         query = urllib.parse.parse_qs(url.query)
 579         if 'fingerprint' not in query:
 580             raise VerificationException("No fingerprint in URL.")
 581         fingerprint = query['fingerprint'][0]
 582
 583     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
 584     download, new_etag = net.http_get(url.geturl(), etag)
 585
 586     if download is None:
 587         return None, new_etag
 588
 589     with tempfile.NamedTemporaryFile() as fp:
 590         # write and open JAR file
 591         fp.write(download)
 592         jar = zipfile.ZipFile(fp)
 593
 594         # verify that the JAR signature is valid
 595         verify_jar_signature(fp.name)
 596
 597         # get public key and its fingerprint from JAR
 598         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 599
 600         # compare the fingerprint if verify_fingerprint is True
 601         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
 602             raise VerificationException("The repository's fingerprint does not match.")
 603
 604         # load repository index from JSON
 605         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
 606         index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
 607         index["repo"]["fingerprint"] = public_key_fingerprint
 608
 609         # turn the apps into App objects
 610         index["apps"] = [metadata.App(app) for app in index["apps"]]
 611
 612         return index, new_etag
 613
 614
 615 def verify_jar_signature(file):
 616     """
 617     Verifies the signature of a given JAR file.
 618
 619     :raises: VerificationException() if the JAR's signature could not be verified
 620     """
 621     if not common.verify_apk_signature(file, jar=True):
 622         raise VerificationException("The repository's index could not be verified.")
 623
 624
 625 def get_public_key_from_jar(jar):
 626     """
 627     Get the public key and its fingerprint from a JAR file.
 628
 629     :raises: VerificationException() if the JAR was not signed exactly once
 630
 631     :param jar: a zipfile.ZipFile object
 632     :return: the public key from the jar and its fingerprint
 633     """
 634     # extract certificate from jar
 635     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
 636     if len(certs) < 1:
 637         raise VerificationException("Found no signing certificates for repository.")
 638     if len(certs) > 1:
 639         raise VerificationException("Found multiple signing certificates for repository.")
 640
 641     # extract public key from certificate
 642     public_key = common.get_certificate(jar.read(certs[0]))
 643     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 644
 645     return public_key, public_key_fingerprint