fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import tempfile
  31 import urllib.parse
  32 import zipfile
  33 from binascii import hexlify, unhexlify
  34 from datetime import datetime
  35 from xml.dom.minidom import Document
  36
  37 from . import _
  38 from . import common
  39 from . import metadata
  40 from . import net
  41 from . import signindex
  42 from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
  43 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
  44
  45
  46 def make(apps, sortedids, apks, repodir, archive):
  47     """Generate the repo index files.
  48
  49     This requires properly initialized options and config objects.
  50
  51     :param apps: fully populated apps list
  52     :param sortedids: app package IDs, sorted
  53     :param apks: full populated apks list
  54     :param repodir: the repo directory
  55     :param archive: True if this is the archive repo, False if it's the
  56                     main one.
  57     """
  58     from fdroidserver.update import METADATA_VERSION
  59
  60     def _resolve_description_link(appid):
  61         if appid in apps:
  62             return "fdroid.app:" + appid, apps[appid].Name
  63         raise MetaDataException("Cannot resolve app id " + appid)
  64
  65     nosigningkey = False
  66     if not common.options.nosign:
  67         if 'repo_keyalias' not in common.config:
  68             nosigningkey = True
  69             logging.critical(_("'repo_keyalias' not found in config.py!"))
  70         if 'keystore' not in common.config:
  71             nosigningkey = True
  72             logging.critical(_("'keystore' not found in config.py!"))
  73         if 'keystorepass' not in common.config:
  74             nosigningkey = True
  75             logging.critical(_("'keystorepass' not found in config.py!"))
  76         if 'keypass' not in common.config:
  77             nosigningkey = True
  78             logging.critical(_("'keypass' not found in config.py!"))
  79         if not os.path.exists(common.config['keystore']):
  80             nosigningkey = True
  81             logging.critical("'" + common.config['keystore'] + "' does not exist!")
  82         if nosigningkey:
  83             raise FDroidException("`fdroid update` requires a signing key, " +
  84                                   "you can create one using: fdroid update --create-key")
  85
  86     repodict = collections.OrderedDict()
  87     repodict['timestamp'] = datetime.utcnow()
  88     repodict['version'] = METADATA_VERSION
  89
  90     if common.config['repo_maxage'] != 0:
  91         repodict['maxage'] = common.config['repo_maxage']
  92
  93     if archive:
  94         repodict['name'] = common.config['archive_name']
  95         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  96         repodict['address'] = common.config['archive_url']
  97         repodict['description'] = common.config['archive_description']
  98         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
  99     else:
 100         repodict['name'] = common.config['repo_name']
 101         repodict['icon'] = os.path.basename(common.config['repo_icon'])
 102         repodict['address'] = common.config['repo_url']
 103         repodict['description'] = common.config['repo_description']
 104         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
 105
 106     mirrorcheckfailed = False
 107     mirrors = []
 108     for mirror in sorted(common.config.get('mirrors', [])):
 109         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 110         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
 111             logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
 112             mirrorcheckfailed = True
 113         # must end with / or urljoin strips a whole path segment
 114         if mirror.endswith('/'):
 115             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
 116         else:
 117             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 118     for mirror in common.config.get('servergitmirrors', []):
 119         for url in get_mirror_service_urls(mirror):
 120             mirrors.append(url + '/' + repodir)
 121     if mirrorcheckfailed:
 122         raise FDroidException(_("Malformed repository mirrors."))
 123     if mirrors:
 124         repodict['mirrors'] = mirrors
 125
 126     appsWithPackages = collections.OrderedDict()
 127     for packageName in sortedids:
 128         app = apps[packageName]
 129         if app['Disabled']:
 130             continue
 131
 132         # only include apps with packages
 133         for apk in apks:
 134             if apk['packageName'] == packageName:
 135                 newapp = copy.copy(app)  # update wiki needs unmodified description
 136                 newapp['Description'] = metadata.description_html(app['Description'],
 137                                                                   _resolve_description_link)
 138                 appsWithPackages[packageName] = newapp
 139                 break
 140
 141     requestsdict = collections.OrderedDict()
 142     for command in ('install', 'uninstall'):
 143         packageNames = []
 144         key = command + '_list'
 145         if key in common.config:
 146             if isinstance(common.config[key], str):
 147                 packageNames = [common.config[key]]
 148             elif all(isinstance(item, str) for item in common.config[key]):
 149                 packageNames = common.config[key]
 150             else:
 151                 raise TypeError(_('only accepts strings, lists, and tuples'))
 152         requestsdict[command] = packageNames
 153
 154     fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
 155
 156     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
 157             fdroid_signing_key_fingerprints)
 158     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
 159             fdroid_signing_key_fingerprints)
 160
 161
 162 def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 163
 164     def _index_encoder_default(obj):
 165         if isinstance(obj, set):
 166             return list(obj)
 167         if isinstance(obj, datetime):
 168             return int(obj.timestamp() * 1000)  # Java expects milliseconds
 169         raise TypeError(repr(obj) + " is not JSON serializable")
 170
 171     output = collections.OrderedDict()
 172     output['repo'] = repodict
 173     output['requests'] = requestsdict
 174
 175     # establish sort order of the index
 176     v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
 177
 178     appslist = []
 179     output['apps'] = appslist
 180     for packageName, appdict in apps.items():
 181         d = collections.OrderedDict()
 182         appslist.append(d)
 183         for k, v in sorted(appdict.items()):
 184             if not v:
 185                 continue
 186             if k in ('builds', 'comments', 'metadatapath',
 187                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 188                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 189                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 190                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 191                 continue
 192
 193             # name things after the App class fields in fdroidclient
 194             if k == 'id':
 195                 k = 'packageName'
 196             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 197                 k = 'suggestedVersionCode'
 198             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 199                 k = 'suggestedVersionName'
 200             elif k == 'AutoName':
 201                 if 'Name' not in apps[packageName]:
 202                     d['name'] = v
 203                 continue
 204             else:
 205                 k = k[:1].lower() + k[1:]
 206             d[k] = v
 207
 208     output_packages = collections.OrderedDict()
 209     output['packages'] = output_packages
 210     for package in packages:
 211         packageName = package['packageName']
 212         if packageName not in apps:
 213             logging.info(_('Ignoring package without metadata: ') + package['apkName'])
 214             continue
 215         if packageName in output_packages:
 216             packagelist = output_packages[packageName]
 217         else:
 218             packagelist = []
 219             output_packages[packageName] = packagelist
 220         d = collections.OrderedDict()
 221         packagelist.append(d)
 222         for k, v in sorted(package.items()):
 223             if not v:
 224                 continue
 225             if k in ('icon', 'icons', 'icons_src', 'name', ):
 226                 continue
 227             d[k] = v
 228
 229     json_name = 'index-v1.json'
 230     index_file = os.path.join(repodir, json_name)
 231     with open(index_file, 'w') as fp:
 232         if common.options.pretty:
 233             json.dump(output, fp, default=_index_encoder_default, indent=2)
 234         else:
 235             json.dump(output, fp, default=_index_encoder_default)
 236
 237     if common.options.nosign:
 238         logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
 239     else:
 240         signindex.config = common.config
 241         signindex.sign_index_v1(repodir, json_name)
 242
 243
 244 def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
 245
 246     GROUP_DEV_SIGNED = 1
 247     GROUP_FDROID_SIGNED = 2
 248     GROUP_OTHER_SIGNED = 3
 249
 250     def v1_sort_keys(package):
 251         packageName = package.get('packageName', None)
 252
 253         sig = package.get('signer', None)
 254
 255         dev_sig = common.metadata_find_developer_signature(packageName)
 256         group = GROUP_OTHER_SIGNED
 257         if dev_sig and dev_sig == sig:
 258             group = GROUP_DEV_SIGNED
 259         else:
 260             fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
 261             if fdroidsig and fdroidsig == sig:
 262                 group = GROUP_FDROID_SIGNED
 263
 264         versionCode = None
 265         if package.get('versionCode', None):
 266             versionCode = -int(package['versionCode'])
 267
 268         return(packageName, group, sig, versionCode)
 269
 270     packages.sort(key=v1_sort_keys)
 271
 272
 273 def make_v0(apps, apks, repodir, repodict, requestsdict):
 274     """
 275     aka index.jar aka index.xml
 276     """
 277
 278     doc = Document()
 279
 280     def addElement(name, value, doc, parent):
 281         el = doc.createElement(name)
 282         el.appendChild(doc.createTextNode(value))
 283         parent.appendChild(el)
 284
 285     def addElementNonEmpty(name, value, doc, parent):
 286         if not value:
 287             return
 288         addElement(name, value, doc, parent)
 289
 290     def addElementIfInApk(name, apk, key, doc, parent):
 291         if key not in apk:
 292             return
 293         value = str(apk[key])
 294         addElement(name, value, doc, parent)
 295
 296     def addElementCDATA(name, value, doc, parent):
 297         el = doc.createElement(name)
 298         el.appendChild(doc.createCDATASection(value))
 299         parent.appendChild(el)
 300
 301     def addElementCheckLocalized(name, app, key, doc, parent, default=''):
 302         '''Fill in field from metadata or localized block
 303
 304         For name/summary/description, they can come only from the app source,
 305         or from a dir in fdroiddata.  They can be entirely missing from the
 306         metadata file if there is localized versions.  This will fetch those
 307         from the localized version if its not available in the metadata file.
 308         '''
 309
 310         el = doc.createElement(name)
 311         value = app.get(key)
 312         lkey = key[:1].lower() + key[1:]
 313         localized = app.get('localized')
 314         if not value and localized:
 315             for lang in ['en-US'] + [x for x in localized.keys()]:
 316                 if not lang.startswith('en'):
 317                     continue
 318                 if lang in localized:
 319                     value = localized[lang].get(lkey)
 320                     if value:
 321                         break
 322         if not value and localized and len(localized) > 1:
 323             lang = list(localized.keys())[0]
 324             value = localized[lang].get(lkey)
 325         if not value:
 326             value = default
 327         el.appendChild(doc.createTextNode(value))
 328         parent.appendChild(el)
 329
 330     root = doc.createElement("fdroid")
 331     doc.appendChild(root)
 332
 333     repoel = doc.createElement("repo")
 334
 335     repoel.setAttribute("name", repodict['name'])
 336     if 'maxage' in repodict:
 337         repoel.setAttribute("maxage", str(repodict['maxage']))
 338     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 339     repoel.setAttribute("url", repodict['address'])
 340     addElement('description', repodict['description'], doc, repoel)
 341     for mirror in repodict.get('mirrors', []):
 342         addElement('mirror', mirror, doc, repoel)
 343
 344     repoel.setAttribute("version", str(repodict['version']))
 345     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 346
 347     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 348     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 349     root.appendChild(repoel)
 350
 351     for command in ('install', 'uninstall'):
 352         for packageName in requestsdict[command]:
 353             element = doc.createElement(command)
 354             root.appendChild(element)
 355             element.setAttribute('packageName', packageName)
 356
 357     fdroid_signed = load_sigkeys(repodir)
 358
 359     for appid, appdict in apps.items():
 360         app = metadata.App(appdict)
 361
 362         if app.Disabled is not None:
 363             continue
 364
 365         # Get a list of the apks for this app...
 366         apklist = []
 367         apksbyversion = collections.defaultdict(lambda: [])
 368         for apk in apks:
 369             if apk.get('versionCode') and apk.get('packageName') == appid:
 370                 apksbyversion[apk['versionCode']].append(apk)
 371         for versionCode, apksforver in apksbyversion.items():
 372             fdroidsig = fdroid_signed.get(appid, {}).get('signer')
 373             fdroid_signed_apk = None
 374             name_match_apk = None
 375             for x in apksforver:
 376                 if fdroidsig and x.get('signer', None) == fdroidsig:
 377                     fdroid_signed_apk = x
 378                 if common.apk_release_filename.match(x.get('apkName', '')):
 379                     name_match_apk = x
 380             # choose which of the available versions is most
 381             # suiteable for index v0
 382             if fdroid_signed_apk:
 383                 apklist.append(fdroid_signed_apk)
 384             elif name_match_apk:
 385                 apklist.append(name_match_apk)
 386             else:
 387                 apklist.append(apksforver[0])
 388
 389         if len(apklist) == 0:
 390             continue
 391
 392         apel = doc.createElement("application")
 393         apel.setAttribute("id", app.id)
 394         root.appendChild(apel)
 395
 396         addElement('id', app.id, doc, apel)
 397         if app.added:
 398             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 399         if app.lastUpdated:
 400             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 401
 402         addElementCheckLocalized('name', app, 'Name', doc, apel)
 403         addElementCheckLocalized('summary', app, 'Summary', doc, apel)
 404
 405         if app.icon:
 406             addElement('icon', app.icon, doc, apel)
 407
 408         addElementCheckLocalized('desc', app, 'Description', doc, apel,
 409                                  '<p>No description available</p>')
 410
 411         addElement('license', app.License, doc, apel)
 412         if app.Categories:
 413             addElement('categories', ','.join(app.Categories), doc, apel)
 414             # We put the first (primary) category in LAST, which will have
 415             # the desired effect of making clients that only understand one
 416             # category see that one.
 417             addElement('category', app.Categories[0], doc, apel)
 418         addElement('web', app.WebSite, doc, apel)
 419         addElement('source', app.SourceCode, doc, apel)
 420         addElement('tracker', app.IssueTracker, doc, apel)
 421         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 422         addElementNonEmpty('author', app.AuthorName, doc, apel)
 423         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 424         addElementNonEmpty('donate', app.Donate, doc, apel)
 425         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 426         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 427         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 428
 429         # These elements actually refer to the current version (i.e. which
 430         # one is recommended. They are historically mis-named, and need
 431         # changing, but stay like this for now to support existing clients.
 432         addElement('marketversion', app.CurrentVersion, doc, apel)
 433         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 434
 435         if app.Provides:
 436             pv = app.Provides.split(',')
 437             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 438         if app.RequiresRoot:
 439             addElement('requirements', 'root', doc, apel)
 440
 441         # Sort the apk list into version order, just so the web site
 442         # doesn't have to do any work by default...
 443         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 444
 445         if 'antiFeatures' in apklist[0]:
 446             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 447         if app.AntiFeatures:
 448             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 449
 450         # Check for duplicates - they will make the client unhappy...
 451         for i in range(len(apklist) - 1):
 452             first = apklist[i]
 453             second = apklist[i + 1]
 454             if first['versionCode'] == second['versionCode'] \
 455                and first['sig'] == second['sig']:
 456                 if first['hash'] == second['hash']:
 457                     raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
 458                         repodir, first['apkName'], second['apkName']))
 459                 else:
 460                     raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
 461                         repodir, first['apkName'], second['apkName']))
 462
 463         current_version_code = 0
 464         current_version_file = None
 465         for apk in apklist:
 466             file_extension = common.get_file_extension(apk['apkName'])
 467             # find the APK for the "Current Version"
 468             if current_version_code < apk['versionCode']:
 469                 current_version_code = apk['versionCode']
 470             if current_version_code < int(app.CurrentVersionCode):
 471                 current_version_file = apk['apkName']
 472
 473             apkel = doc.createElement("package")
 474             apel.appendChild(apkel)
 475             addElement('version', apk['versionName'], doc, apkel)
 476             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 477             addElement('apkname', apk['apkName'], doc, apkel)
 478             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 479
 480             hashel = doc.createElement("hash")
 481             hashel.setAttribute('type', 'sha256')
 482             hashel.appendChild(doc.createTextNode(apk['hash']))
 483             apkel.appendChild(hashel)
 484
 485             addElement('size', str(apk['size']), doc, apkel)
 486             addElementIfInApk('sdkver', apk,
 487                               'minSdkVersion', doc, apkel)
 488             addElementIfInApk('targetSdkVersion', apk,
 489                               'targetSdkVersion', doc, apkel)
 490             addElementIfInApk('maxsdkver', apk,
 491                               'maxSdkVersion', doc, apkel)
 492             addElementIfInApk('obbMainFile', apk,
 493                               'obbMainFile', doc, apkel)
 494             addElementIfInApk('obbMainFileSha256', apk,
 495                               'obbMainFileSha256', doc, apkel)
 496             addElementIfInApk('obbPatchFile', apk,
 497                               'obbPatchFile', doc, apkel)
 498             addElementIfInApk('obbPatchFileSha256', apk,
 499                               'obbPatchFileSha256', doc, apkel)
 500             if 'added' in apk:
 501                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 502
 503             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 504                 addElement('sig', apk['sig'], doc, apkel)
 505
 506                 old_permissions = set()
 507                 sorted_permissions = sorted(apk['uses-permission'])
 508                 for perm in sorted_permissions:
 509                     perm_name = perm.name
 510                     if perm_name.startswith("android.permission."):
 511                         perm_name = perm_name[19:]
 512                     old_permissions.add(perm_name)
 513                 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 514
 515                 for permission in sorted_permissions:
 516                     permel = doc.createElement('uses-permission')
 517                     permel.setAttribute('name', permission.name)
 518                     if permission.maxSdkVersion is not None:
 519                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 520                         apkel.appendChild(permel)
 521                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 522                     permel = doc.createElement('uses-permission-sdk-23')
 523                     permel.setAttribute('name', permission_sdk_23.name)
 524                     if permission_sdk_23.maxSdkVersion is not None:
 525                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 526                         apkel.appendChild(permel)
 527                 if 'nativecode' in apk:
 528                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 529                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 530
 531         if current_version_file is not None \
 532                 and common.config['make_current_version_link'] \
 533                 and repodir == 'repo':  # only create these
 534             namefield = common.config['current_version_name_source']
 535             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
 536             apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
 537             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
 538             if os.path.islink(apklinkname):
 539                 os.remove(apklinkname)
 540             os.symlink(current_version_path, apklinkname)
 541             # also symlink gpg signature, if it exists
 542             for extension in (b'.asc', b'.sig'):
 543                 sigfile_path = current_version_path + extension
 544                 if os.path.exists(sigfile_path):
 545                     siglinkname = apklinkname + extension
 546                     if os.path.islink(siglinkname):
 547                         os.remove(siglinkname)
 548                     os.symlink(sigfile_path, siglinkname)
 549
 550     if common.options.pretty:
 551         output = doc.toprettyxml(encoding='utf-8')
 552     else:
 553         output = doc.toxml(encoding='utf-8')
 554
 555     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 556         f.write(output)
 557
 558     if 'repo_keyalias' in common.config:
 559
 560         if common.options.nosign:
 561             logging.info(_("Creating unsigned index in preparation for signing"))
 562         else:
 563             logging.info(_("Creating signed index with this key (SHA256):"))
 564             logging.info("%s" % repo_pubkey_fingerprint)
 565
 566         # Create a jar of the index...
 567         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 568         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 569         if p.returncode != 0:
 570             raise FDroidException("Failed to create {0}".format(jar_output))
 571
 572         # Sign the index...
 573         signed = os.path.join(repodir, 'index.jar')
 574         if common.options.nosign:
 575             # Remove old signed index if not signing
 576             if os.path.exists(signed):
 577                 os.remove(signed)
 578         else:
 579             signindex.config = common.config
 580             signindex.sign_jar(signed)
 581
 582     # Copy the repo icon into the repo directory...
 583     icon_dir = os.path.join(repodir, 'icons')
 584     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 585     shutil.copyfile(common.config['repo_icon'], iconfilename)
 586
 587
 588 def extract_pubkey():
 589     """
 590     Extracts and returns the repository's public key from the keystore.
 591     :return: public key in hex, repository fingerprint
 592     """
 593     if 'repo_pubkey' in common.config:
 594         pubkey = unhexlify(common.config['repo_pubkey'])
 595     else:
 596         env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
 597         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 598                               '-alias', common.config['repo_keyalias'],
 599                               '-keystore', common.config['keystore'],
 600                               '-storepass:env', 'FDROID_KEY_STORE_PASS']
 601                              + common.config['smartcardoptions'],
 602                              envs=env_vars, output=False, stderr_to_stdout=False)
 603         if p.returncode != 0 or len(p.output) < 20:
 604             msg = "Failed to get repo pubkey!"
 605             if common.config['keystore'] == 'NONE':
 606                 msg += ' Is your crypto smartcard plugged in?'
 607             raise FDroidException(msg)
 608         pubkey = p.output
 609     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 610     return hexlify(pubkey), repo_pubkey_fingerprint
 611
 612
 613 def get_mirror_service_urls(url):
 614     '''Get direct URLs from git service for use by fdroidclient
 615
 616     Via 'servergitmirrors', fdroidserver can create and push a mirror
 617     to certain well known git services like gitlab or github.  This
 618     will always use the 'master' branch since that is the default
 619     branch in git. The files are then accessible via alternate URLs,
 620     where they are served in their raw format via a CDN rather than
 621     from git.
 622     '''
 623
 624     if url.startswith('git@'):
 625         url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 626
 627     segments = url.split("/")
 628
 629     if segments[4].endswith('.git'):
 630         segments[4] = segments[4][:-4]
 631
 632     hostname = segments[2]
 633     user = segments[3]
 634     repo = segments[4]
 635     branch = "master"
 636     folder = "fdroid"
 637
 638     urls = []
 639     if hostname == "github.com":
 640         # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
 641         segments[2] = "raw.githubusercontent.com"
 642         segments.extend([branch, folder])
 643         urls.append('/'.join(segments))
 644     elif hostname == "gitlab.com":
 645         # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
 646         gitlab_raw = segments + ['raw', branch, folder]
 647         urls.append('/'.join(gitlab_raw))
 648         # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
 649         gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
 650         urls.append('/'.join(gitlab_pages))
 651         return urls
 652
 653     return urls
 654
 655
 656 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
 657     """
 658     Downloads the repository index from the given :param url_str
 659     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 660
 661     :raises: VerificationException() if the repository could not be verified
 662
 663     :return: A tuple consisting of:
 664         - The index in JSON format or None if the index did not change
 665         - The new eTag as returned by the HTTP request
 666     """
 667     url = urllib.parse.urlsplit(url_str)
 668
 669     fingerprint = None
 670     if verify_fingerprint:
 671         query = urllib.parse.parse_qs(url.query)
 672         if 'fingerprint' not in query:
 673             raise VerificationException(_("No fingerprint in URL."))
 674         fingerprint = query['fingerprint'][0]
 675
 676     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
 677     download, new_etag = net.http_get(url.geturl(), etag)
 678
 679     if download is None:
 680         return None, new_etag
 681
 682     with tempfile.NamedTemporaryFile() as fp:
 683         # write and open JAR file
 684         fp.write(download)
 685         jar = zipfile.ZipFile(fp)
 686
 687         # verify that the JAR signature is valid
 688         common.verify_jar_signature(fp.name)
 689
 690         # get public key and its fingerprint from JAR
 691         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 692
 693         # compare the fingerprint if verify_fingerprint is True
 694         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
 695             raise VerificationException(_("The repository's fingerprint does not match."))
 696
 697         # load repository index from JSON
 698         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
 699         index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
 700         index["repo"]["fingerprint"] = public_key_fingerprint
 701
 702         # turn the apps into App objects
 703         index["apps"] = [metadata.App(app) for app in index["apps"]]
 704
 705         return index, new_etag
 706
 707
 708 def get_public_key_from_jar(jar):
 709     """
 710     Get the public key and its fingerprint from a JAR file.
 711
 712     :raises: VerificationException() if the JAR was not signed exactly once
 713
 714     :param jar: a zipfile.ZipFile object
 715     :return: the public key from the jar and its fingerprint
 716     """
 717     # extract certificate from jar
 718     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
 719     if len(certs) < 1:
 720         raise VerificationException(_("Found no signing certificates for repository."))
 721     if len(certs) > 1:
 722         raise VerificationException(_("Found multiple signing certificates for repository."))
 723
 724     # extract public key from certificate
 725     public_key = common.get_certificate(jar.read(certs[0]))
 726     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 727
 728     return public_key, public_key_fingerprint