fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import sys
  31 import tempfile
  32 import urllib.parse
  33 import zipfile
  34 from binascii import hexlify, unhexlify
  35 from datetime import datetime
  36 from xml.dom.minidom import Document
  37
  38 from fdroidserver import metadata, signindex, common, net
  39 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
  40 from fdroidserver.metadata import MetaDataException
  41
  42
  43 def make(apps, sortedids, apks, repodir, archive):
  44     """Generate the repo index files.
  45
  46     This requires properly initialized options and config objects.
  47
  48     :param apps: fully populated apps list
  49     :param sortedids: app package IDs, sorted
  50     :param apks: full populated apks list
  51     :param repodir: the repo directory
  52     :param archive: True if this is the archive repo, False if it's the
  53                     main one.
  54     """
  55     from fdroidserver.update import METADATA_VERSION
  56
  57     def _resolve_description_link(appid):
  58         if appid in apps:
  59             return "fdroid.app:" + appid, apps[appid].Name
  60         raise MetaDataException("Cannot resolve app id " + appid)
  61
  62     nosigningkey = False
  63     if not common.options.nosign:
  64         if 'repo_keyalias' not in common.config:
  65             nosigningkey = True
  66             logging.critical("'repo_keyalias' not found in config.py!")
  67         if 'keystore' not in common.config:
  68             nosigningkey = True
  69             logging.critical("'keystore' not found in config.py!")
  70         if 'keystorepass' not in common.config:
  71             nosigningkey = True
  72             logging.critical("'keystorepass' not found in config.py!")
  73         if 'keypass' not in common.config:
  74             nosigningkey = True
  75             logging.critical("'keypass' not found in config.py!")
  76         if not os.path.exists(common.config['keystore']):
  77             nosigningkey = True
  78             logging.critical("'" + common.config['keystore'] + "' does not exist!")
  79         if nosigningkey:
  80             logging.warning("`fdroid update` requires a signing key, you can create one using:")
  81             logging.warning("\tfdroid update --create-key")
  82             sys.exit(1)
  83
  84     repodict = collections.OrderedDict()
  85     repodict['timestamp'] = datetime.utcnow()
  86     repodict['version'] = METADATA_VERSION
  87
  88     if common.config['repo_maxage'] != 0:
  89         repodict['maxage'] = common.config['repo_maxage']
  90
  91     if archive:
  92         repodict['name'] = common.config['archive_name']
  93         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  94         repodict['address'] = common.config['archive_url']
  95         repodict['description'] = common.config['archive_description']
  96         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
  97     else:
  98         repodict['name'] = common.config['repo_name']
  99         repodict['icon'] = os.path.basename(common.config['repo_icon'])
 100         repodict['address'] = common.config['repo_url']
 101         repodict['description'] = common.config['repo_description']
 102         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
 103
 104     mirrorcheckfailed = False
 105     mirrors = []
 106     for mirror in sorted(common.config.get('mirrors', [])):
 107         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 108         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
 109             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 110             mirrorcheckfailed = True
 111         # must end with / or urljoin strips a whole path segment
 112         if mirror.endswith('/'):
 113             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
 114         else:
 115             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 116     for mirror in common.config.get('servergitmirrors', []):
 117         mirror = get_mirror_service_url(mirror)
 118         if mirror is not None:
 119             mirrors.append(mirror + '/')
 120     if mirrorcheckfailed:
 121         sys.exit(1)
 122     if mirrors:
 123         repodict['mirrors'] = mirrors
 124
 125     appsWithPackages = collections.OrderedDict()
 126     for packageName in sortedids:
 127         app = apps[packageName]
 128         if app['Disabled']:
 129             continue
 130
 131         # only include apps with packages
 132         for apk in apks:
 133             if apk['packageName'] == packageName:
 134                 newapp = copy.copy(app)  # update wiki needs unmodified description
 135                 newapp['Description'] = metadata.description_html(app['Description'],
 136                                                                   _resolve_description_link)
 137                 appsWithPackages[packageName] = newapp
 138                 break
 139
 140     requestsdict = collections.OrderedDict()
 141     for command in ('install', 'uninstall'):
 142         packageNames = []
 143         key = command + '_list'
 144         if key in common.config:
 145             if isinstance(common.config[key], str):
 146                 packageNames = [common.config[key]]
 147             elif all(isinstance(item, str) for item in common.config[key]):
 148                 packageNames = common.config[key]
 149             else:
 150                 raise TypeError('only accepts strings, lists, and tuples')
 151         requestsdict[command] = packageNames
 152
 153     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
 154     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
 155
 156
 157 def make_v1(apps, packages, repodir, repodict, requestsdict):
 158
 159     def _index_encoder_default(obj):
 160         if isinstance(obj, set):
 161             return list(obj)
 162         if isinstance(obj, datetime):
 163             return int(obj.timestamp() * 1000)  # Java expects milliseconds
 164         raise TypeError(repr(obj) + " is not JSON serializable")
 165
 166     output = collections.OrderedDict()
 167     output['repo'] = repodict
 168     output['requests'] = requestsdict
 169
 170     appslist = []
 171     output['apps'] = appslist
 172     for packageName, appdict in apps.items():
 173         d = collections.OrderedDict()
 174         appslist.append(d)
 175         for k, v in sorted(appdict.items()):
 176             if not v:
 177                 continue
 178             if k in ('builds', 'comments', 'metadatapath',
 179                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 180                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 181                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 182                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 183                 continue
 184
 185             # name things after the App class fields in fdroidclient
 186             if k == 'id':
 187                 k = 'packageName'
 188             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 189                 k = 'suggestedVersionCode'
 190             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 191                 k = 'suggestedVersionName'
 192             elif k == 'AutoName':
 193                 if 'Name' not in apps[packageName]:
 194                     d['name'] = v
 195                 continue
 196             else:
 197                 k = k[:1].lower() + k[1:]
 198             d[k] = v
 199
 200     output_packages = collections.OrderedDict()
 201     output['packages'] = output_packages
 202     for package in packages:
 203         packageName = package['packageName']
 204         if packageName not in apps:
 205             logging.info('Ignoring package without metadata: ' + package['apkName'])
 206             continue
 207         if packageName in output_packages:
 208             packagelist = output_packages[packageName]
 209         else:
 210             packagelist = []
 211             output_packages[packageName] = packagelist
 212         d = collections.OrderedDict()
 213         packagelist.append(d)
 214         for k, v in sorted(package.items()):
 215             if not v:
 216                 continue
 217             if k in ('icon', 'icons', 'icons_src', 'name', ):
 218                 continue
 219             d[k] = v
 220
 221     json_name = 'index-v1.json'
 222     index_file = os.path.join(repodir, json_name)
 223     with open(index_file, 'w') as fp:
 224         if common.options.pretty:
 225             json.dump(output, fp, default=_index_encoder_default, indent=2)
 226         else:
 227             json.dump(output, fp, default=_index_encoder_default)
 228
 229     if common.options.nosign:
 230         logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
 231     else:
 232         signindex.config = common.config
 233         signindex.sign_index_v1(repodir, json_name)
 234
 235
 236 def make_v0(apps, apks, repodir, repodict, requestsdict):
 237     """
 238     aka index.jar aka index.xml
 239     """
 240
 241     doc = Document()
 242
 243     def addElement(name, value, doc, parent):
 244         el = doc.createElement(name)
 245         el.appendChild(doc.createTextNode(value))
 246         parent.appendChild(el)
 247
 248     def addElementNonEmpty(name, value, doc, parent):
 249         if not value:
 250             return
 251         addElement(name, value, doc, parent)
 252
 253     def addElementIfInApk(name, apk, key, doc, parent):
 254         if key not in apk:
 255             return
 256         value = str(apk[key])
 257         addElement(name, value, doc, parent)
 258
 259     def addElementCDATA(name, value, doc, parent):
 260         el = doc.createElement(name)
 261         el.appendChild(doc.createCDATASection(value))
 262         parent.appendChild(el)
 263
 264     root = doc.createElement("fdroid")
 265     doc.appendChild(root)
 266
 267     repoel = doc.createElement("repo")
 268
 269     repoel.setAttribute("name", repodict['name'])
 270     if 'maxage' in repodict:
 271         repoel.setAttribute("maxage", str(repodict['maxage']))
 272     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 273     repoel.setAttribute("url", repodict['address'])
 274     addElement('description', repodict['description'], doc, repoel)
 275     for mirror in repodict.get('mirrors', []):
 276         addElement('mirror', mirror, doc, repoel)
 277
 278     repoel.setAttribute("version", str(repodict['version']))
 279     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 280
 281     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 282     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 283     root.appendChild(repoel)
 284
 285     for command in ('install', 'uninstall'):
 286         for packageName in requestsdict[command]:
 287             element = doc.createElement(command)
 288             root.appendChild(element)
 289             element.setAttribute('packageName', packageName)
 290
 291     for appid, appdict in apps.items():
 292         app = metadata.App(appdict)
 293
 294         if app.Disabled is not None:
 295             continue
 296
 297         # Get a list of the apks for this app...
 298         apklist = []
 299         for apk in apks:
 300             if apk['packageName'] == appid:
 301                 apklist.append(apk)
 302
 303         if len(apklist) == 0:
 304             continue
 305
 306         apel = doc.createElement("application")
 307         apel.setAttribute("id", app.id)
 308         root.appendChild(apel)
 309
 310         addElement('id', app.id, doc, apel)
 311         if app.added:
 312             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 313         if app.lastUpdated:
 314             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 315         addElement('name', app.Name, doc, apel)
 316         addElement('summary', app.Summary, doc, apel)
 317         if app.icon:
 318             addElement('icon', app.icon, doc, apel)
 319
 320         if app.get('Description'):
 321             description = app.Description
 322         else:
 323             description = '<p>No description available</p>'
 324         addElement('desc', description, doc, apel)
 325         addElement('license', app.License, doc, apel)
 326         if app.Categories:
 327             addElement('categories', ','.join(app.Categories), doc, apel)
 328             # We put the first (primary) category in LAST, which will have
 329             # the desired effect of making clients that only understand one
 330             # category see that one.
 331             addElement('category', app.Categories[0], doc, apel)
 332         addElement('web', app.WebSite, doc, apel)
 333         addElement('source', app.SourceCode, doc, apel)
 334         addElement('tracker', app.IssueTracker, doc, apel)
 335         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 336         addElementNonEmpty('author', app.AuthorName, doc, apel)
 337         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 338         addElementNonEmpty('donate', app.Donate, doc, apel)
 339         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 340         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 341         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 342
 343         # These elements actually refer to the current version (i.e. which
 344         # one is recommended. They are historically mis-named, and need
 345         # changing, but stay like this for now to support existing clients.
 346         addElement('marketversion', app.CurrentVersion, doc, apel)
 347         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 348
 349         if app.Provides:
 350             pv = app.Provides.split(',')
 351             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 352         if app.RequiresRoot:
 353             addElement('requirements', 'root', doc, apel)
 354
 355         # Sort the apk list into version order, just so the web site
 356         # doesn't have to do any work by default...
 357         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 358
 359         if 'antiFeatures' in apklist[0]:
 360             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 361         if app.AntiFeatures:
 362             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 363
 364         # Check for duplicates - they will make the client unhappy...
 365         for i in range(len(apklist) - 1):
 366             if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
 367                 logging.critical("duplicate versions: '%s' - '%s'" % (
 368                     apklist[i]['apkName'], apklist[i + 1]['apkName']))
 369                 sys.exit(1)
 370
 371         current_version_code = 0
 372         current_version_file = None
 373         for apk in apklist:
 374             file_extension = common.get_file_extension(apk['apkName'])
 375             # find the APK for the "Current Version"
 376             if current_version_code < apk['versionCode']:
 377                 current_version_code = apk['versionCode']
 378             if current_version_code < int(app.CurrentVersionCode):
 379                 current_version_file = apk['apkName']
 380
 381             apkel = doc.createElement("package")
 382             apel.appendChild(apkel)
 383             addElement('version', apk['versionName'], doc, apkel)
 384             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 385             addElement('apkname', apk['apkName'], doc, apkel)
 386             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 387
 388             hashel = doc.createElement("hash")
 389             hashel.setAttribute('type', 'sha256')
 390             hashel.appendChild(doc.createTextNode(apk['hash']))
 391             apkel.appendChild(hashel)
 392
 393             addElement('size', str(apk['size']), doc, apkel)
 394             addElementIfInApk('sdkver', apk,
 395                               'minSdkVersion', doc, apkel)
 396             addElementIfInApk('targetSdkVersion', apk,
 397                               'targetSdkVersion', doc, apkel)
 398             addElementIfInApk('maxsdkver', apk,
 399                               'maxSdkVersion', doc, apkel)
 400             addElementIfInApk('obbMainFile', apk,
 401                               'obbMainFile', doc, apkel)
 402             addElementIfInApk('obbMainFileSha256', apk,
 403                               'obbMainFileSha256', doc, apkel)
 404             addElementIfInApk('obbPatchFile', apk,
 405                               'obbPatchFile', doc, apkel)
 406             addElementIfInApk('obbPatchFileSha256', apk,
 407                               'obbPatchFileSha256', doc, apkel)
 408             if 'added' in apk:
 409                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 410
 411             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 412                 addElement('sig', apk['sig'], doc, apkel)
 413
 414                 old_permissions = set()
 415                 sorted_permissions = sorted(apk['uses-permission'])
 416                 for perm in sorted_permissions:
 417                     perm_name = perm.name
 418                     if perm_name.startswith("android.permission."):
 419                         perm_name = perm_name[19:]
 420                     old_permissions.add(perm_name)
 421                 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 422
 423                 for permission in sorted_permissions:
 424                     permel = doc.createElement('uses-permission')
 425                     permel.setAttribute('name', permission.name)
 426                     if permission.maxSdkVersion is not None:
 427                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 428                         apkel.appendChild(permel)
 429                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 430                     permel = doc.createElement('uses-permission-sdk-23')
 431                     permel.setAttribute('name', permission_sdk_23.name)
 432                     if permission_sdk_23.maxSdkVersion is not None:
 433                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 434                         apkel.appendChild(permel)
 435                 if 'nativecode' in apk:
 436                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 437                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 438
 439         if current_version_file is not None \
 440                 and common.config['make_current_version_link'] \
 441                 and repodir == 'repo':  # only create these
 442             namefield = common.config['current_version_name_source']
 443             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
 444             apklinkname = sanitized_name + b'.apk'
 445             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
 446             if os.path.islink(apklinkname):
 447                 os.remove(apklinkname)
 448             os.symlink(current_version_path, apklinkname)
 449             # also symlink gpg signature, if it exists
 450             for extension in (b'.asc', b'.sig'):
 451                 sigfile_path = current_version_path + extension
 452                 if os.path.exists(sigfile_path):
 453                     siglinkname = apklinkname + extension
 454                     if os.path.islink(siglinkname):
 455                         os.remove(siglinkname)
 456                     os.symlink(sigfile_path, siglinkname)
 457
 458     if common.options.pretty:
 459         output = doc.toprettyxml(encoding='utf-8')
 460     else:
 461         output = doc.toxml(encoding='utf-8')
 462
 463     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 464         f.write(output)
 465
 466     if 'repo_keyalias' in common.config:
 467
 468         if common.options.nosign:
 469             logging.info("Creating unsigned index in preparation for signing")
 470         else:
 471             logging.info("Creating signed index with this key (SHA256):")
 472             logging.info("%s" % repo_pubkey_fingerprint)
 473
 474         # Create a jar of the index...
 475         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 476         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 477         if p.returncode != 0:
 478             logging.critical("Failed to create {0}".format(jar_output))
 479             sys.exit(1)
 480
 481         # Sign the index...
 482         signed = os.path.join(repodir, 'index.jar')
 483         if common.options.nosign:
 484             # Remove old signed index if not signing
 485             if os.path.exists(signed):
 486                 os.remove(signed)
 487         else:
 488             signindex.config = common.config
 489             signindex.sign_jar(signed)
 490
 491     # Copy the repo icon into the repo directory...
 492     icon_dir = os.path.join(repodir, 'icons')
 493     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 494     shutil.copyfile(common.config['repo_icon'], iconfilename)
 495
 496
 497 def extract_pubkey():
 498     """
 499     Extracts and returns the repository's public key from the keystore.
 500     :return: public key in hex, repository fingerprint
 501     """
 502     if 'repo_pubkey' in common.config:
 503         pubkey = unhexlify(common.config['repo_pubkey'])
 504     else:
 505         env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
 506         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 507                               '-alias', common.config['repo_keyalias'],
 508                               '-keystore', common.config['keystore'],
 509                               '-storepass:env', 'FDROID_KEY_STORE_PASS']
 510                              + common.config['smartcardoptions'],
 511                              envs=env_vars, output=False, stderr_to_stdout=False)
 512         if p.returncode != 0 or len(p.output) < 20:
 513             msg = "Failed to get repo pubkey!"
 514             if common.config['keystore'] == 'NONE':
 515                 msg += ' Is your crypto smartcard plugged in?'
 516             logging.critical(msg)
 517             sys.exit(1)
 518         pubkey = p.output
 519     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 520     return hexlify(pubkey), repo_pubkey_fingerprint
 521
 522
 523 def get_mirror_service_url(url):
 524     '''Get direct URL from git service for use by fdroidclient
 525
 526     Via 'servergitmirrors', fdroidserver can create and push a mirror
 527     to certain well known git services like gitlab or github.  This
 528     will always use the 'master' branch since that is the default
 529     branch in git.
 530
 531     '''
 532
 533     if url.startswith('git@'):
 534         url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 535
 536     segments = url.split("/")
 537
 538     if segments[4].endswith('.git'):
 539         segments[4] = segments[4][:-4]
 540
 541     hostname = segments[2]
 542     user = segments[3]
 543     repo = segments[4]
 544     branch = "master"
 545     folder = "fdroid"
 546
 547     if hostname == "github.com":
 548         # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/master/fdroid"
 549         segments[2] = "raw.githubusercontent.com"
 550         segments.extend([branch, folder])
 551     elif hostname == "gitlab.com":
 552         # Gitlab-like Pages segments "https://user.gitlab.com/repo/fdroid"
 553         gitlab_url = ["https:", "", user + ".gitlab.io", repo, folder]
 554         segments = gitlab_url
 555     else:
 556         return None
 557
 558     return '/'.join(segments)
 559
 560
 561 class VerificationException(Exception):
 562     pass
 563
 564
 565 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
 566     """
 567     Downloads the repository index from the given :param url_str
 568     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 569
 570     :raises: VerificationException() if the repository could not be verified
 571
 572     :return: A tuple consisting of:
 573         - The index in JSON format or None if the index did not change
 574         - The new eTag as returned by the HTTP request
 575     """
 576     url = urllib.parse.urlsplit(url_str)
 577
 578     fingerprint = None
 579     if verify_fingerprint:
 580         query = urllib.parse.parse_qs(url.query)
 581         if 'fingerprint' not in query:
 582             raise VerificationException("No fingerprint in URL.")
 583         fingerprint = query['fingerprint'][0]
 584
 585     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
 586     download, new_etag = net.http_get(url.geturl(), etag)
 587
 588     if download is None:
 589         return None, new_etag
 590
 591     with tempfile.NamedTemporaryFile() as fp:
 592         # write and open JAR file
 593         fp.write(download)
 594         jar = zipfile.ZipFile(fp)
 595
 596         # verify that the JAR signature is valid
 597         verify_jar_signature(fp.name)
 598
 599         # get public key and its fingerprint from JAR
 600         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 601
 602         # compare the fingerprint if verify_fingerprint is True
 603         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
 604             raise VerificationException("The repository's fingerprint does not match.")
 605
 606         # load repository index from JSON
 607         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
 608         index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
 609         index["repo"]["fingerprint"] = public_key_fingerprint
 610
 611         # turn the apps into App objects
 612         index["apps"] = [metadata.App(app) for app in index["apps"]]
 613
 614         return index, new_etag
 615
 616
 617 def verify_jar_signature(file):
 618     """
 619     Verifies the signature of a given JAR file.
 620
 621     :raises: VerificationException() if the JAR's signature could not be verified
 622     """
 623     if not common.verify_apk_signature(file, jar=True):
 624         raise VerificationException("The repository's index could not be verified.")
 625
 626
 627 def get_public_key_from_jar(jar):
 628     """
 629     Get the public key and its fingerprint from a JAR file.
 630
 631     :raises: VerificationException() if the JAR was not signed exactly once
 632
 633     :param jar: a zipfile.ZipFile object
 634     :return: the public key from the jar and its fingerprint
 635     """
 636     # extract certificate from jar
 637     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
 638     if len(certs) < 1:
 639         raise VerificationException("Found no signing certificates for repository.")
 640     if len(certs) > 1:
 641         raise VerificationException("Found multiple signing certificates for repository.")
 642
 643     # extract public key from certificate
 644     public_key = common.get_certificate(jar.read(certs[0]))
 645     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 646
 647     return public_key, public_key_fingerprint