fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import tempfile
  31 import urllib.parse
  32 import zipfile
  33 from binascii import hexlify, unhexlify
  34 from datetime import datetime
  35 from xml.dom.minidom import Document
  36
  37 from fdroidserver import metadata, signindex, common, net
  38 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
  39 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
  40
  41
  42 def make(apps, sortedids, apks, repodir, archive):
  43     """Generate the repo index files.
  44
  45     This requires properly initialized options and config objects.
  46
  47     :param apps: fully populated apps list
  48     :param sortedids: app package IDs, sorted
  49     :param apks: full populated apks list
  50     :param repodir: the repo directory
  51     :param archive: True if this is the archive repo, False if it's the
  52                     main one.
  53     """
  54     from fdroidserver.update import METADATA_VERSION
  55
  56     def _resolve_description_link(appid):
  57         if appid in apps:
  58             return "fdroid.app:" + appid, apps[appid].Name
  59         raise MetaDataException("Cannot resolve app id " + appid)
  60
  61     nosigningkey = False
  62     if not common.options.nosign:
  63         if 'repo_keyalias' not in common.config:
  64             nosigningkey = True
  65             logging.critical("'repo_keyalias' not found in config.py!")
  66         if 'keystore' not in common.config:
  67             nosigningkey = True
  68             logging.critical("'keystore' not found in config.py!")
  69         if 'keystorepass' not in common.config:
  70             nosigningkey = True
  71             logging.critical("'keystorepass' not found in config.py!")
  72         if 'keypass' not in common.config:
  73             nosigningkey = True
  74             logging.critical("'keypass' not found in config.py!")
  75         if not os.path.exists(common.config['keystore']):
  76             nosigningkey = True
  77             logging.critical("'" + common.config['keystore'] + "' does not exist!")
  78         if nosigningkey:
  79             raise FDroidException("`fdroid update` requires a signing key, " +
  80                                   "you can create one using: fdroid update --create-key")
  81
  82     repodict = collections.OrderedDict()
  83     repodict['timestamp'] = datetime.utcnow()
  84     repodict['version'] = METADATA_VERSION
  85
  86     if common.config['repo_maxage'] != 0:
  87         repodict['maxage'] = common.config['repo_maxage']
  88
  89     if archive:
  90         repodict['name'] = common.config['archive_name']
  91         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  92         repodict['address'] = common.config['archive_url']
  93         repodict['description'] = common.config['archive_description']
  94         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
  95     else:
  96         repodict['name'] = common.config['repo_name']
  97         repodict['icon'] = os.path.basename(common.config['repo_icon'])
  98         repodict['address'] = common.config['repo_url']
  99         repodict['description'] = common.config['repo_description']
 100         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
 101
 102     mirrorcheckfailed = False
 103     mirrors = []
 104     for mirror in sorted(common.config.get('mirrors', [])):
 105         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 106         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
 107             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 108             mirrorcheckfailed = True
 109         # must end with / or urljoin strips a whole path segment
 110         if mirror.endswith('/'):
 111             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
 112         else:
 113             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 114     for mirror in common.config.get('servergitmirrors', []):
 115         mirror = get_mirror_service_url(mirror)
 116         if mirror:
 117             mirrors.append(mirror + '/' + repodir)
 118     if mirrorcheckfailed:
 119         raise FDroidException("Malformed repository mirrors.")
 120     if mirrors:
 121         repodict['mirrors'] = mirrors
 122
 123     appsWithPackages = collections.OrderedDict()
 124     for packageName in sortedids:
 125         app = apps[packageName]
 126         if app['Disabled']:
 127             continue
 128
 129         # only include apps with packages
 130         for apk in apks:
 131             if apk['packageName'] == packageName:
 132                 newapp = copy.copy(app)  # update wiki needs unmodified description
 133                 newapp['Description'] = metadata.description_html(app['Description'],
 134                                                                   _resolve_description_link)
 135                 appsWithPackages[packageName] = newapp
 136                 break
 137
 138     requestsdict = collections.OrderedDict()
 139     for command in ('install', 'uninstall'):
 140         packageNames = []
 141         key = command + '_list'
 142         if key in common.config:
 143             if isinstance(common.config[key], str):
 144                 packageNames = [common.config[key]]
 145             elif all(isinstance(item, str) for item in common.config[key]):
 146                 packageNames = common.config[key]
 147             else:
 148                 raise TypeError('only accepts strings, lists, and tuples')
 149         requestsdict[command] = packageNames
 150
 151     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
 152     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
 153
 154
 155 def make_v1(apps, packages, repodir, repodict, requestsdict):
 156
 157     def _index_encoder_default(obj):
 158         if isinstance(obj, set):
 159             return list(obj)
 160         if isinstance(obj, datetime):
 161             return int(obj.timestamp() * 1000)  # Java expects milliseconds
 162         raise TypeError(repr(obj) + " is not JSON serializable")
 163
 164     output = collections.OrderedDict()
 165     output['repo'] = repodict
 166     output['requests'] = requestsdict
 167
 168     appslist = []
 169     output['apps'] = appslist
 170     for packageName, appdict in apps.items():
 171         d = collections.OrderedDict()
 172         appslist.append(d)
 173         for k, v in sorted(appdict.items()):
 174             if not v:
 175                 continue
 176             if k in ('builds', 'comments', 'metadatapath',
 177                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 178                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 179                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 180                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 181                 continue
 182
 183             # name things after the App class fields in fdroidclient
 184             if k == 'id':
 185                 k = 'packageName'
 186             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 187                 k = 'suggestedVersionCode'
 188             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 189                 k = 'suggestedVersionName'
 190             elif k == 'AutoName':
 191                 if 'Name' not in apps[packageName]:
 192                     d['name'] = v
 193                 continue
 194             else:
 195                 k = k[:1].lower() + k[1:]
 196             d[k] = v
 197
 198     output_packages = collections.OrderedDict()
 199     output['packages'] = output_packages
 200     for package in packages:
 201         packageName = package['packageName']
 202         if packageName not in apps:
 203             logging.info('Ignoring package without metadata: ' + package['apkName'])
 204             continue
 205         if packageName in output_packages:
 206             packagelist = output_packages[packageName]
 207         else:
 208             packagelist = []
 209             output_packages[packageName] = packagelist
 210         d = collections.OrderedDict()
 211         packagelist.append(d)
 212         for k, v in sorted(package.items()):
 213             if not v:
 214                 continue
 215             if k in ('icon', 'icons', 'icons_src', 'name', ):
 216                 continue
 217             d[k] = v
 218
 219     json_name = 'index-v1.json'
 220     index_file = os.path.join(repodir, json_name)
 221     with open(index_file, 'w') as fp:
 222         if common.options.pretty:
 223             json.dump(output, fp, default=_index_encoder_default, indent=2)
 224         else:
 225             json.dump(output, fp, default=_index_encoder_default)
 226
 227     if common.options.nosign:
 228         logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
 229     else:
 230         signindex.config = common.config
 231         signindex.sign_index_v1(repodir, json_name)
 232
 233
 234 def make_v0(apps, apks, repodir, repodict, requestsdict):
 235     """
 236     aka index.jar aka index.xml
 237     """
 238
 239     doc = Document()
 240
 241     def addElement(name, value, doc, parent):
 242         el = doc.createElement(name)
 243         el.appendChild(doc.createTextNode(value))
 244         parent.appendChild(el)
 245
 246     def addElementNonEmpty(name, value, doc, parent):
 247         if not value:
 248             return
 249         addElement(name, value, doc, parent)
 250
 251     def addElementIfInApk(name, apk, key, doc, parent):
 252         if key not in apk:
 253             return
 254         value = str(apk[key])
 255         addElement(name, value, doc, parent)
 256
 257     def addElementCDATA(name, value, doc, parent):
 258         el = doc.createElement(name)
 259         el.appendChild(doc.createCDATASection(value))
 260         parent.appendChild(el)
 261
 262     root = doc.createElement("fdroid")
 263     doc.appendChild(root)
 264
 265     repoel = doc.createElement("repo")
 266
 267     repoel.setAttribute("name", repodict['name'])
 268     if 'maxage' in repodict:
 269         repoel.setAttribute("maxage", str(repodict['maxage']))
 270     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 271     repoel.setAttribute("url", repodict['address'])
 272     addElement('description', repodict['description'], doc, repoel)
 273     for mirror in repodict.get('mirrors', []):
 274         addElement('mirror', mirror, doc, repoel)
 275
 276     repoel.setAttribute("version", str(repodict['version']))
 277     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 278
 279     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 280     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 281     root.appendChild(repoel)
 282
 283     for command in ('install', 'uninstall'):
 284         for packageName in requestsdict[command]:
 285             element = doc.createElement(command)
 286             root.appendChild(element)
 287             element.setAttribute('packageName', packageName)
 288
 289     for appid, appdict in apps.items():
 290         app = metadata.App(appdict)
 291
 292         if app.Disabled is not None:
 293             continue
 294
 295         # Get a list of the apks for this app...
 296         apklist = []
 297         versionCodes = []
 298         for apk in apks:
 299             if apk['packageName'] == appid:
 300                 if apk['versionCode'] not in versionCodes:
 301                     apklist.append(apk)
 302                     versionCodes.append(apk['versionCode'])
 303
 304         if len(apklist) == 0:
 305             continue
 306
 307         apel = doc.createElement("application")
 308         apel.setAttribute("id", app.id)
 309         root.appendChild(apel)
 310
 311         addElement('id', app.id, doc, apel)
 312         if app.added:
 313             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 314         if app.lastUpdated:
 315             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 316         addElement('name', app.Name, doc, apel)
 317         addElement('summary', app.Summary, doc, apel)
 318         if app.icon:
 319             addElement('icon', app.icon, doc, apel)
 320
 321         if app.get('Description'):
 322             description = app.Description
 323         else:
 324             description = '<p>No description available</p>'
 325         addElement('desc', description, doc, apel)
 326         addElement('license', app.License, doc, apel)
 327         if app.Categories:
 328             addElement('categories', ','.join(app.Categories), doc, apel)
 329             # We put the first (primary) category in LAST, which will have
 330             # the desired effect of making clients that only understand one
 331             # category see that one.
 332             addElement('category', app.Categories[0], doc, apel)
 333         addElement('web', app.WebSite, doc, apel)
 334         addElement('source', app.SourceCode, doc, apel)
 335         addElement('tracker', app.IssueTracker, doc, apel)
 336         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 337         addElementNonEmpty('author', app.AuthorName, doc, apel)
 338         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 339         addElementNonEmpty('donate', app.Donate, doc, apel)
 340         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 341         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 342         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 343
 344         # These elements actually refer to the current version (i.e. which
 345         # one is recommended. They are historically mis-named, and need
 346         # changing, but stay like this for now to support existing clients.
 347         addElement('marketversion', app.CurrentVersion, doc, apel)
 348         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 349
 350         if app.Provides:
 351             pv = app.Provides.split(',')
 352             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 353         if app.RequiresRoot:
 354             addElement('requirements', 'root', doc, apel)
 355
 356         # Sort the apk list into version order, just so the web site
 357         # doesn't have to do any work by default...
 358         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 359
 360         if 'antiFeatures' in apklist[0]:
 361             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 362         if app.AntiFeatures:
 363             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 364
 365         # Check for duplicates - they will make the client unhappy...
 366         for i in range(len(apklist) - 1):
 367             first = apklist[i]
 368             second = apklist[i + 1]
 369             if first['versionCode'] == second['versionCode'] \
 370                and first['sig'] == second['sig']:
 371                 if first['hash'] == second['hash']:
 372                     raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
 373                         repodir, first['apkName'], second['apkName']))
 374                 else:
 375                     raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
 376                         repodir, first['apkName'], second['apkName']))
 377
 378         current_version_code = 0
 379         current_version_file = None
 380         for apk in apklist:
 381             file_extension = common.get_file_extension(apk['apkName'])
 382             # find the APK for the "Current Version"
 383             if current_version_code < apk['versionCode']:
 384                 current_version_code = apk['versionCode']
 385             if current_version_code < int(app.CurrentVersionCode):
 386                 current_version_file = apk['apkName']
 387
 388             apkel = doc.createElement("package")
 389             apel.appendChild(apkel)
 390             addElement('version', apk['versionName'], doc, apkel)
 391             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 392             addElement('apkname', apk['apkName'], doc, apkel)
 393             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 394
 395             hashel = doc.createElement("hash")
 396             hashel.setAttribute('type', 'sha256')
 397             hashel.appendChild(doc.createTextNode(apk['hash']))
 398             apkel.appendChild(hashel)
 399
 400             addElement('size', str(apk['size']), doc, apkel)
 401             addElementIfInApk('sdkver', apk,
 402                               'minSdkVersion', doc, apkel)
 403             addElementIfInApk('targetSdkVersion', apk,
 404                               'targetSdkVersion', doc, apkel)
 405             addElementIfInApk('maxsdkver', apk,
 406                               'maxSdkVersion', doc, apkel)
 407             addElementIfInApk('obbMainFile', apk,
 408                               'obbMainFile', doc, apkel)
 409             addElementIfInApk('obbMainFileSha256', apk,
 410                               'obbMainFileSha256', doc, apkel)
 411             addElementIfInApk('obbPatchFile', apk,
 412                               'obbPatchFile', doc, apkel)
 413             addElementIfInApk('obbPatchFileSha256', apk,
 414                               'obbPatchFileSha256', doc, apkel)
 415             if 'added' in apk:
 416                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 417
 418             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 419                 addElement('sig', apk['sig'], doc, apkel)
 420
 421                 old_permissions = set()
 422                 sorted_permissions = sorted(apk['uses-permission'])
 423                 for perm in sorted_permissions:
 424                     perm_name = perm.name
 425                     if perm_name.startswith("android.permission."):
 426                         perm_name = perm_name[19:]
 427                     old_permissions.add(perm_name)
 428                 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 429
 430                 for permission in sorted_permissions:
 431                     permel = doc.createElement('uses-permission')
 432                     permel.setAttribute('name', permission.name)
 433                     if permission.maxSdkVersion is not None:
 434                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 435                         apkel.appendChild(permel)
 436                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 437                     permel = doc.createElement('uses-permission-sdk-23')
 438                     permel.setAttribute('name', permission_sdk_23.name)
 439                     if permission_sdk_23.maxSdkVersion is not None:
 440                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 441                         apkel.appendChild(permel)
 442                 if 'nativecode' in apk:
 443                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 444                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 445
 446         if current_version_file is not None \
 447                 and common.config['make_current_version_link'] \
 448                 and repodir == 'repo':  # only create these
 449             namefield = common.config['current_version_name_source']
 450             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
 451             apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
 452             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
 453             if os.path.islink(apklinkname):
 454                 os.remove(apklinkname)
 455             os.symlink(current_version_path, apklinkname)
 456             # also symlink gpg signature, if it exists
 457             for extension in (b'.asc', b'.sig'):
 458                 sigfile_path = current_version_path + extension
 459                 if os.path.exists(sigfile_path):
 460                     siglinkname = apklinkname + extension
 461                     if os.path.islink(siglinkname):
 462                         os.remove(siglinkname)
 463                     os.symlink(sigfile_path, siglinkname)
 464
 465     if common.options.pretty:
 466         output = doc.toprettyxml(encoding='utf-8')
 467     else:
 468         output = doc.toxml(encoding='utf-8')
 469
 470     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 471         f.write(output)
 472
 473     if 'repo_keyalias' in common.config:
 474
 475         if common.options.nosign:
 476             logging.info("Creating unsigned index in preparation for signing")
 477         else:
 478             logging.info("Creating signed index with this key (SHA256):")
 479             logging.info("%s" % repo_pubkey_fingerprint)
 480
 481         # Create a jar of the index...
 482         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 483         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 484         if p.returncode != 0:
 485             raise FDroidException("Failed to create {0}".format(jar_output))
 486
 487         # Sign the index...
 488         signed = os.path.join(repodir, 'index.jar')
 489         if common.options.nosign:
 490             # Remove old signed index if not signing
 491             if os.path.exists(signed):
 492                 os.remove(signed)
 493         else:
 494             signindex.config = common.config
 495             signindex.sign_jar(signed)
 496
 497     # Copy the repo icon into the repo directory...
 498     icon_dir = os.path.join(repodir, 'icons')
 499     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 500     shutil.copyfile(common.config['repo_icon'], iconfilename)
 501
 502
 503 def extract_pubkey():
 504     """
 505     Extracts and returns the repository's public key from the keystore.
 506     :return: public key in hex, repository fingerprint
 507     """
 508     if 'repo_pubkey' in common.config:
 509         pubkey = unhexlify(common.config['repo_pubkey'])
 510     else:
 511         env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
 512         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 513                               '-alias', common.config['repo_keyalias'],
 514                               '-keystore', common.config['keystore'],
 515                               '-storepass:env', 'FDROID_KEY_STORE_PASS']
 516                              + common.config['smartcardoptions'],
 517                              envs=env_vars, output=False, stderr_to_stdout=False)
 518         if p.returncode != 0 or len(p.output) < 20:
 519             msg = "Failed to get repo pubkey!"
 520             if common.config['keystore'] == 'NONE':
 521                 msg += ' Is your crypto smartcard plugged in?'
 522             raise FDroidException(msg)
 523         pubkey = p.output
 524     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 525     return hexlify(pubkey), repo_pubkey_fingerprint
 526
 527
 528 def get_mirror_service_url(url):
 529     '''Get direct URL from git service for use by fdroidclient
 530
 531     Via 'servergitmirrors', fdroidserver can create and push a mirror
 532     to certain well known git services like gitlab or github.  This
 533     will always use the 'master' branch since that is the default
 534     branch in git.
 535
 536     '''
 537
 538     if url.startswith('git@'):
 539         url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 540
 541     segments = url.split("/")
 542
 543     if segments[4].endswith('.git'):
 544         segments[4] = segments[4][:-4]
 545
 546     hostname = segments[2]
 547     user = segments[3]
 548     repo = segments[4]
 549     branch = "master"
 550     folder = "fdroid"
 551
 552     if hostname == "github.com":
 553         # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/master/fdroid"
 554         segments[2] = "raw.githubusercontent.com"
 555         segments.extend([branch, folder])
 556     elif hostname == "gitlab.com":
 557         # Gitlab-like Pages segments "https://user.gitlab.com/repo/fdroid"
 558         gitlab_url = ["https:", "", user + ".gitlab.io", repo, folder]
 559         segments = gitlab_url
 560     else:
 561         return None
 562
 563     return '/'.join(segments)
 564
 565
 566 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
 567     """
 568     Downloads the repository index from the given :param url_str
 569     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 570
 571     :raises: VerificationException() if the repository could not be verified
 572
 573     :return: A tuple consisting of:
 574         - The index in JSON format or None if the index did not change
 575         - The new eTag as returned by the HTTP request
 576     """
 577     url = urllib.parse.urlsplit(url_str)
 578
 579     fingerprint = None
 580     if verify_fingerprint:
 581         query = urllib.parse.parse_qs(url.query)
 582         if 'fingerprint' not in query:
 583             raise VerificationException("No fingerprint in URL.")
 584         fingerprint = query['fingerprint'][0]
 585
 586     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
 587     download, new_etag = net.http_get(url.geturl(), etag)
 588
 589     if download is None:
 590         return None, new_etag
 591
 592     with tempfile.NamedTemporaryFile() as fp:
 593         # write and open JAR file
 594         fp.write(download)
 595         jar = zipfile.ZipFile(fp)
 596
 597         # verify that the JAR signature is valid
 598         verify_jar_signature(fp.name)
 599
 600         # get public key and its fingerprint from JAR
 601         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 602
 603         # compare the fingerprint if verify_fingerprint is True
 604         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
 605             raise VerificationException("The repository's fingerprint does not match.")
 606
 607         # load repository index from JSON
 608         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
 609         index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
 610         index["repo"]["fingerprint"] = public_key_fingerprint
 611
 612         # turn the apps into App objects
 613         index["apps"] = [metadata.App(app) for app in index["apps"]]
 614
 615         return index, new_etag
 616
 617
 618 def verify_jar_signature(file):
 619     """
 620     Verifies the signature of a given JAR file.
 621
 622     :raises: VerificationException() if the JAR's signature could not be verified
 623     """
 624     if not common.verify_apk_signature(file, jar=True):
 625         raise VerificationException("The repository's index could not be verified.")
 626
 627
 628 def get_public_key_from_jar(jar):
 629     """
 630     Get the public key and its fingerprint from a JAR file.
 631
 632     :raises: VerificationException() if the JAR was not signed exactly once
 633
 634     :param jar: a zipfile.ZipFile object
 635     :return: the public key from the jar and its fingerprint
 636     """
 637     # extract certificate from jar
 638     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
 639     if len(certs) < 1:
 640         raise VerificationException("Found no signing certificates for repository.")
 641     if len(certs) > 1:
 642         raise VerificationException("Found multiple signing certificates for repository.")
 643
 644     # extract public key from certificate
 645     public_key = common.get_certificate(jar.read(certs[0]))
 646     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 647
 648     return public_key, public_key_fingerprint