fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import tempfile
  31 import urllib.parse
  32 import zipfile
  33 import calendar
  34 from binascii import hexlify, unhexlify
  35 from datetime import datetime, timezone
  36 from xml.dom.minidom import Document
  37
  38 from . import _
  39 from . import common
  40 from . import metadata
  41 from . import net
  42 from . import signindex
  43 from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
  44 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
  45
  46
  47 def make(apps, sortedids, apks, repodir, archive):
  48     """Generate the repo index files.
  49
  50     This requires properly initialized options and config objects.
  51
  52     :param apps: fully populated apps list
  53     :param sortedids: app package IDs, sorted
  54     :param apks: full populated apks list
  55     :param repodir: the repo directory
  56     :param archive: True if this is the archive repo, False if it's the
  57                     main one.
  58     """
  59     from fdroidserver.update import METADATA_VERSION
  60
  61     def _resolve_description_link(appid):
  62         if appid in apps:
  63             return "fdroid.app:" + appid, apps[appid].Name
  64         raise MetaDataException("Cannot resolve app id " + appid)
  65
  66     if not common.options.nosign:
  67         common.assert_config_keystore(common.config)
  68
  69     repodict = collections.OrderedDict()
  70     repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
  71     repodict['version'] = METADATA_VERSION
  72
  73     if common.config['repo_maxage'] != 0:
  74         repodict['maxage'] = common.config['repo_maxage']
  75
  76     if archive:
  77         repodict['name'] = common.config['archive_name']
  78         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  79         repodict['address'] = common.config['archive_url']
  80         repodict['description'] = common.config['archive_description']
  81         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
  82     else:
  83         repodict['name'] = common.config['repo_name']
  84         repodict['icon'] = os.path.basename(common.config['repo_icon'])
  85         repodict['address'] = common.config['repo_url']
  86         repodict['description'] = common.config['repo_description']
  87         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
  88
  89     mirrorcheckfailed = False
  90     mirrors = []
  91     for mirror in sorted(common.config.get('mirrors', [])):
  92         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
  93         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
  94             logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
  95             mirrorcheckfailed = True
  96         # must end with / or urljoin strips a whole path segment
  97         if mirror.endswith('/'):
  98             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
  99         else:
 100             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 101     for mirror in common.config.get('servergitmirrors', []):
 102         for url in get_mirror_service_urls(mirror):
 103             mirrors.append(url + '/' + repodir)
 104     if mirrorcheckfailed:
 105         raise FDroidException(_("Malformed repository mirrors."))
 106     if mirrors:
 107         repodict['mirrors'] = mirrors
 108
 109     appsWithPackages = collections.OrderedDict()
 110     for packageName in sortedids:
 111         app = apps[packageName]
 112         if app['Disabled']:
 113             continue
 114
 115         # only include apps with packages
 116         for apk in apks:
 117             if apk['packageName'] == packageName:
 118                 newapp = copy.copy(app)  # update wiki needs unmodified description
 119                 newapp['Description'] = metadata.description_html(app['Description'],
 120                                                                   _resolve_description_link)
 121                 appsWithPackages[packageName] = newapp
 122                 break
 123
 124     requestsdict = collections.OrderedDict()
 125     for command in ('install', 'uninstall'):
 126         packageNames = []
 127         key = command + '_list'
 128         if key in common.config:
 129             if isinstance(common.config[key], str):
 130                 packageNames = [common.config[key]]
 131             elif all(isinstance(item, str) for item in common.config[key]):
 132                 packageNames = common.config[key]
 133             else:
 134                 raise TypeError(_('only accepts strings, lists, and tuples'))
 135         requestsdict[command] = packageNames
 136
 137     fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
 138
 139     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
 140             fdroid_signing_key_fingerprints)
 141     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
 142             fdroid_signing_key_fingerprints)
 143
 144
 145 def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 146
 147     def _index_encoder_default(obj):
 148         if isinstance(obj, set):
 149             return sorted(list(obj))
 150         if isinstance(obj, datetime):
 151             # Java prefers milliseconds
 152             # we also need to accound for time zone/daylight saving time
 153             return int(calendar.timegm(obj.timetuple()) * 1000)
 154         if isinstance(obj, dict):
 155             d = collections.OrderedDict()
 156             for key in sorted(obj.keys()):
 157                 d[key] = obj[key]
 158             return d
 159         raise TypeError(repr(obj) + " is not JSON serializable")
 160
 161     output = collections.OrderedDict()
 162     output['repo'] = repodict
 163     output['requests'] = requestsdict
 164
 165     # establish sort order of the index
 166     v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
 167
 168     appslist = []
 169     output['apps'] = appslist
 170     for packageName, appdict in apps.items():
 171         d = collections.OrderedDict()
 172         appslist.append(d)
 173         for k, v in sorted(appdict.items()):
 174             if not v:
 175                 continue
 176             if k in ('builds', 'comments', 'metadatapath',
 177                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 178                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 179                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 180                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 181                 continue
 182
 183             # name things after the App class fields in fdroidclient
 184             if k == 'id':
 185                 k = 'packageName'
 186             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 187                 k = 'suggestedVersionCode'
 188             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 189                 k = 'suggestedVersionName'
 190             elif k == 'AutoName':
 191                 if 'Name' not in apps[packageName]:
 192                     d['name'] = v
 193                 continue
 194             else:
 195                 k = k[:1].lower() + k[1:]
 196             d[k] = v
 197
 198     # establish sort order in localized dicts
 199     for app in output['apps']:
 200         localized = app.get('localized')
 201         if localized:
 202             lordered = collections.OrderedDict()
 203             for lkey, lvalue in sorted(localized.items()):
 204                 lordered[lkey] = collections.OrderedDict()
 205                 for ikey, iname in sorted(lvalue.items()):
 206                     lordered[lkey][ikey] = iname
 207             app['localized'] = lordered
 208
 209     output_packages = collections.OrderedDict()
 210     output['packages'] = output_packages
 211     for package in packages:
 212         packageName = package['packageName']
 213         if packageName not in apps:
 214             logging.info(_('Ignoring package without metadata: ') + package['apkName'])
 215             continue
 216         if not package.get('versionName'):
 217             app = apps[packageName]
 218             versionCodeStr = str(package['versionCode'])  # TODO build.versionCode should be int!
 219             for build in app['builds']:
 220                 if build['versionCode'] == versionCodeStr:
 221                     versionName = build.get('versionName')
 222                     logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}')
 223                                  .format(apkfilename=package['apkName'], version=versionName))
 224                     package['versionName'] = versionName
 225                     break
 226         if packageName in output_packages:
 227             packagelist = output_packages[packageName]
 228         else:
 229             packagelist = []
 230             output_packages[packageName] = packagelist
 231         d = collections.OrderedDict()
 232         packagelist.append(d)
 233         for k, v in sorted(package.items()):
 234             if not v:
 235                 continue
 236             if k in ('icon', 'icons', 'icons_src', 'name', ):
 237                 continue
 238             d[k] = v
 239
 240     json_name = 'index-v1.json'
 241     index_file = os.path.join(repodir, json_name)
 242     with open(index_file, 'w') as fp:
 243         if common.options.pretty:
 244             json.dump(output, fp, default=_index_encoder_default, indent=2)
 245         else:
 246             json.dump(output, fp, default=_index_encoder_default)
 247
 248     if common.options.nosign:
 249         logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
 250     else:
 251         signindex.config = common.config
 252         signindex.sign_index_v1(repodir, json_name)
 253
 254
 255 def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
 256     """Sorts the supplied list to ensure a deterministic sort order for
 257     package entries in the index file. This sort-order also expresses
 258     installation preference to the clients.
 259     (First in this list = first to install)
 260
 261     :param packages: list of packages which need to be sorted before but into index file.
 262     """
 263
 264     GROUP_DEV_SIGNED = 1
 265     GROUP_FDROID_SIGNED = 2
 266     GROUP_OTHER_SIGNED = 3
 267
 268     def v1_sort_keys(package):
 269         packageName = package.get('packageName', None)
 270
 271         sig = package.get('signer', None)
 272
 273         dev_sig = common.metadata_find_developer_signature(packageName)
 274         group = GROUP_OTHER_SIGNED
 275         if dev_sig and dev_sig == sig:
 276             group = GROUP_DEV_SIGNED
 277         else:
 278             fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
 279             if fdroidsig and fdroidsig == sig:
 280                 group = GROUP_FDROID_SIGNED
 281
 282         versionCode = None
 283         if package.get('versionCode', None):
 284             versionCode = -int(package['versionCode'])
 285
 286         return(packageName, group, sig, versionCode)
 287
 288     packages.sort(key=v1_sort_keys)
 289
 290
 291 def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 292     """
 293     aka index.jar aka index.xml
 294     """
 295
 296     doc = Document()
 297
 298     def addElement(name, value, doc, parent):
 299         el = doc.createElement(name)
 300         el.appendChild(doc.createTextNode(value))
 301         parent.appendChild(el)
 302
 303     def addElementNonEmpty(name, value, doc, parent):
 304         if not value:
 305             return
 306         addElement(name, value, doc, parent)
 307
 308     def addElementIfInApk(name, apk, key, doc, parent):
 309         if key not in apk:
 310             return
 311         value = str(apk[key])
 312         addElement(name, value, doc, parent)
 313
 314     def addElementCDATA(name, value, doc, parent):
 315         el = doc.createElement(name)
 316         el.appendChild(doc.createCDATASection(value))
 317         parent.appendChild(el)
 318
 319     def addElementCheckLocalized(name, app, key, doc, parent, default=''):
 320         '''Fill in field from metadata or localized block
 321
 322         For name/summary/description, they can come only from the app source,
 323         or from a dir in fdroiddata.  They can be entirely missing from the
 324         metadata file if there is localized versions.  This will fetch those
 325         from the localized version if its not available in the metadata file.
 326         '''
 327
 328         el = doc.createElement(name)
 329         value = app.get(key)
 330         lkey = key[:1].lower() + key[1:]
 331         localized = app.get('localized')
 332         if not value and localized:
 333             for lang in ['en-US'] + [x for x in localized.keys()]:
 334                 if not lang.startswith('en'):
 335                     continue
 336                 if lang in localized:
 337                     value = localized[lang].get(lkey)
 338                     if value:
 339                         break
 340         if not value and localized and len(localized) > 1:
 341             lang = list(localized.keys())[0]
 342             value = localized[lang].get(lkey)
 343         if not value:
 344             value = default
 345         el.appendChild(doc.createTextNode(value))
 346         parent.appendChild(el)
 347
 348     root = doc.createElement("fdroid")
 349     doc.appendChild(root)
 350
 351     repoel = doc.createElement("repo")
 352
 353     repoel.setAttribute("name", repodict['name'])
 354     if 'maxage' in repodict:
 355         repoel.setAttribute("maxage", str(repodict['maxage']))
 356     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 357     repoel.setAttribute("url", repodict['address'])
 358     addElement('description', repodict['description'], doc, repoel)
 359     for mirror in repodict.get('mirrors', []):
 360         addElement('mirror', mirror, doc, repoel)
 361
 362     repoel.setAttribute("version", str(repodict['version']))
 363     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 364
 365     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 366     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 367     root.appendChild(repoel)
 368
 369     for command in ('install', 'uninstall'):
 370         for packageName in requestsdict[command]:
 371             element = doc.createElement(command)
 372             root.appendChild(element)
 373             element.setAttribute('packageName', packageName)
 374
 375     for appid, appdict in apps.items():
 376         app = metadata.App(appdict)
 377
 378         if app.Disabled is not None:
 379             continue
 380
 381         # Get a list of the apks for this app...
 382         apklist = []
 383         apksbyversion = collections.defaultdict(lambda: [])
 384         for apk in apks:
 385             if apk.get('versionCode') and apk.get('packageName') == appid:
 386                 apksbyversion[apk['versionCode']].append(apk)
 387         for versionCode, apksforver in apksbyversion.items():
 388             fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
 389             fdroid_signed_apk = None
 390             name_match_apk = None
 391             for x in apksforver:
 392                 if fdroidsig and x.get('signer', None) == fdroidsig:
 393                     fdroid_signed_apk = x
 394                 if common.apk_release_filename.match(x.get('apkName', '')):
 395                     name_match_apk = x
 396             # choose which of the available versions is most
 397             # suiteable for index v0
 398             if fdroid_signed_apk:
 399                 apklist.append(fdroid_signed_apk)
 400             elif name_match_apk:
 401                 apklist.append(name_match_apk)
 402             else:
 403                 apklist.append(apksforver[0])
 404
 405         if len(apklist) == 0:
 406             continue
 407
 408         apel = doc.createElement("application")
 409         apel.setAttribute("id", app.id)
 410         root.appendChild(apel)
 411
 412         addElement('id', app.id, doc, apel)
 413         if app.added:
 414             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 415         if app.lastUpdated:
 416             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 417
 418         addElementCheckLocalized('name', app, 'Name', doc, apel)
 419         addElementCheckLocalized('summary', app, 'Summary', doc, apel)
 420
 421         if app.icon:
 422             addElement('icon', app.icon, doc, apel)
 423
 424         addElementCheckLocalized('desc', app, 'Description', doc, apel,
 425                                  '<p>No description available</p>')
 426
 427         addElement('license', app.License, doc, apel)
 428         if app.Categories:
 429             addElement('categories', ','.join(app.Categories), doc, apel)
 430             # We put the first (primary) category in LAST, which will have
 431             # the desired effect of making clients that only understand one
 432             # category see that one.
 433             addElement('category', app.Categories[0], doc, apel)
 434         addElement('web', app.WebSite, doc, apel)
 435         addElement('source', app.SourceCode, doc, apel)
 436         addElement('tracker', app.IssueTracker, doc, apel)
 437         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 438         addElementNonEmpty('author', app.AuthorName, doc, apel)
 439         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 440         addElementNonEmpty('donate', app.Donate, doc, apel)
 441         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 442         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 443         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 444         addElementNonEmpty('liberapay', app.LiberapayID, doc, apel)
 445
 446         # These elements actually refer to the current version (i.e. which
 447         # one is recommended. They are historically mis-named, and need
 448         # changing, but stay like this for now to support existing clients.
 449         addElement('marketversion', app.CurrentVersion, doc, apel)
 450         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 451
 452         if app.Provides:
 453             pv = app.Provides.split(',')
 454             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 455         if app.RequiresRoot:
 456             addElement('requirements', 'root', doc, apel)
 457
 458         # Sort the apk list into version order, just so the web site
 459         # doesn't have to do any work by default...
 460         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 461
 462         if 'antiFeatures' in apklist[0]:
 463             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 464         if app.AntiFeatures:
 465             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 466
 467         # Check for duplicates - they will make the client unhappy...
 468         for i in range(len(apklist) - 1):
 469             first = apklist[i]
 470             second = apklist[i + 1]
 471             if first['versionCode'] == second['versionCode'] \
 472                and first['sig'] == second['sig']:
 473                 if first['hash'] == second['hash']:
 474                     raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
 475                         repodir, first['apkName'], second['apkName']))
 476                 else:
 477                     raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
 478                         repodir, first['apkName'], second['apkName']))
 479
 480         current_version_code = 0
 481         current_version_file = None
 482         for apk in apklist:
 483             file_extension = common.get_file_extension(apk['apkName'])
 484             # find the APK for the "Current Version"
 485             if current_version_code < apk['versionCode']:
 486                 current_version_code = apk['versionCode']
 487             if current_version_code < int(app.CurrentVersionCode):
 488                 current_version_file = apk['apkName']
 489
 490             apkel = doc.createElement("package")
 491             apel.appendChild(apkel)
 492
 493             versionName = apk.get('versionName')
 494             if not versionName:
 495                 versionCodeStr = str(apk['versionCode'])  # TODO build.versionCode should be int!
 496                 for build in app.builds:
 497                     if build['versionCode'] == versionCodeStr and 'versionName' in build:
 498                         versionName = build['versionName']
 499                         break
 500             if versionName:
 501                 addElement('version', versionName, doc, apkel)
 502
 503             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 504             addElement('apkname', apk['apkName'], doc, apkel)
 505             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 506
 507             hashel = doc.createElement("hash")
 508             hashel.setAttribute('type', 'sha256')
 509             hashel.appendChild(doc.createTextNode(apk['hash']))
 510             apkel.appendChild(hashel)
 511
 512             addElement('size', str(apk['size']), doc, apkel)
 513             addElementIfInApk('sdkver', apk,
 514                               'minSdkVersion', doc, apkel)
 515             addElementIfInApk('targetSdkVersion', apk,
 516                               'targetSdkVersion', doc, apkel)
 517             addElementIfInApk('maxsdkver', apk,
 518                               'maxSdkVersion', doc, apkel)
 519             addElementIfInApk('obbMainFile', apk,
 520                               'obbMainFile', doc, apkel)
 521             addElementIfInApk('obbMainFileSha256', apk,
 522                               'obbMainFileSha256', doc, apkel)
 523             addElementIfInApk('obbPatchFile', apk,
 524                               'obbPatchFile', doc, apkel)
 525             addElementIfInApk('obbPatchFileSha256', apk,
 526                               'obbPatchFileSha256', doc, apkel)
 527             if 'added' in apk:
 528                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 529
 530             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 531                 addElement('sig', apk['sig'], doc, apkel)
 532
 533                 old_permissions = set()
 534                 sorted_permissions = sorted(apk['uses-permission'])
 535                 for perm in sorted_permissions:
 536                     perm_name = perm.name
 537                     if perm_name.startswith("android.permission."):
 538                         perm_name = perm_name[19:]
 539                     old_permissions.add(perm_name)
 540                 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 541
 542                 for permission in sorted_permissions:
 543                     permel = doc.createElement('uses-permission')
 544                     permel.setAttribute('name', permission.name)
 545                     if permission.maxSdkVersion is not None:
 546                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 547                         apkel.appendChild(permel)
 548                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 549                     permel = doc.createElement('uses-permission-sdk-23')
 550                     permel.setAttribute('name', permission_sdk_23.name)
 551                     if permission_sdk_23.maxSdkVersion is not None:
 552                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 553                         apkel.appendChild(permel)
 554                 if 'nativecode' in apk:
 555                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 556                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 557
 558         if current_version_file is not None \
 559                 and common.config['make_current_version_link'] \
 560                 and repodir == 'repo':  # only create these
 561             namefield = common.config['current_version_name_source']
 562             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
 563             apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
 564             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
 565             if os.path.islink(apklinkname):
 566                 os.remove(apklinkname)
 567             os.symlink(current_version_path, apklinkname)
 568             # also symlink gpg signature, if it exists
 569             for extension in (b'.asc', b'.sig'):
 570                 sigfile_path = current_version_path + extension
 571                 if os.path.exists(sigfile_path):
 572                     siglinkname = apklinkname + extension
 573                     if os.path.islink(siglinkname):
 574                         os.remove(siglinkname)
 575                     os.symlink(sigfile_path, siglinkname)
 576
 577     if common.options.pretty:
 578         output = doc.toprettyxml(encoding='utf-8')
 579     else:
 580         output = doc.toxml(encoding='utf-8')
 581
 582     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 583         f.write(output)
 584
 585     if 'repo_keyalias' in common.config:
 586
 587         if common.options.nosign:
 588             logging.info(_("Creating unsigned index in preparation for signing"))
 589         else:
 590             logging.info(_("Creating signed index with this key (SHA256):"))
 591             logging.info("%s" % repo_pubkey_fingerprint)
 592
 593         # Create a jar of the index...
 594         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 595         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 596         if p.returncode != 0:
 597             raise FDroidException("Failed to create {0}".format(jar_output))
 598
 599         # Sign the index...
 600         signed = os.path.join(repodir, 'index.jar')
 601         if common.options.nosign:
 602             # Remove old signed index if not signing
 603             if os.path.exists(signed):
 604                 os.remove(signed)
 605         else:
 606             signindex.config = common.config
 607             signindex.sign_jar(signed)
 608
 609     # Copy the repo icon into the repo directory...
 610     icon_dir = os.path.join(repodir, 'icons')
 611     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 612     shutil.copyfile(common.config['repo_icon'], iconfilename)
 613
 614
 615 def extract_pubkey():
 616     """
 617     Extracts and returns the repository's public key from the keystore.
 618     :return: public key in hex, repository fingerprint
 619     """
 620     if 'repo_pubkey' in common.config:
 621         pubkey = unhexlify(common.config['repo_pubkey'])
 622     else:
 623         env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
 624         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 625                               '-alias', common.config['repo_keyalias'],
 626                               '-keystore', common.config['keystore'],
 627                               '-storepass:env', 'FDROID_KEY_STORE_PASS']
 628                              + common.config['smartcardoptions'],
 629                              envs=env_vars, output=False, stderr_to_stdout=False)
 630         if p.returncode != 0 or len(p.output) < 20:
 631             msg = "Failed to get repo pubkey!"
 632             if common.config['keystore'] == 'NONE':
 633                 msg += ' Is your crypto smartcard plugged in?'
 634             raise FDroidException(msg)
 635         pubkey = p.output
 636     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 637     return hexlify(pubkey), repo_pubkey_fingerprint
 638
 639
 640 def get_mirror_service_urls(url):
 641     '''Get direct URLs from git service for use by fdroidclient
 642
 643     Via 'servergitmirrors', fdroidserver can create and push a mirror
 644     to certain well known git services like gitlab or github.  This
 645     will always use the 'master' branch since that is the default
 646     branch in git. The files are then accessible via alternate URLs,
 647     where they are served in their raw format via a CDN rather than
 648     from git.
 649     '''
 650
 651     if url.startswith('git@'):
 652         url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 653
 654     segments = url.split("/")
 655
 656     if segments[4].endswith('.git'):
 657         segments[4] = segments[4][:-4]
 658
 659     hostname = segments[2]
 660     user = segments[3]
 661     repo = segments[4]
 662     branch = "master"
 663     folder = "fdroid"
 664
 665     urls = []
 666     if hostname == "github.com":
 667         # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
 668         segments[2] = "raw.githubusercontent.com"
 669         segments.extend([branch, folder])
 670         urls.append('/'.join(segments))
 671     elif hostname == "gitlab.com":
 672         # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser
 673         # This is because the `raw` URLs are not served with the correct mime types, so any
 674         # index.html which is put in the repo will not be rendered. Putting an index.html file in
 675         # the repo root is a common way for to make information about the repo available to end user.
 676
 677         # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
 678         gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
 679         urls.append('/'.join(gitlab_pages))
 680         # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
 681         gitlab_raw = segments + ['raw', branch, folder]
 682         urls.append('/'.join(gitlab_raw))
 683         return urls
 684
 685     return urls
 686
 687
 688 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
 689     """
 690     Downloads the repository index from the given :param url_str
 691     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 692
 693     :raises: VerificationException() if the repository could not be verified
 694
 695     :return: A tuple consisting of:
 696         - The index in JSON format or None if the index did not change
 697         - The new eTag as returned by the HTTP request
 698     """
 699     url = urllib.parse.urlsplit(url_str)
 700
 701     fingerprint = None
 702     if verify_fingerprint:
 703         query = urllib.parse.parse_qs(url.query)
 704         if 'fingerprint' not in query:
 705             raise VerificationException(_("No fingerprint in URL."))
 706         fingerprint = query['fingerprint'][0]
 707
 708     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
 709     download, new_etag = net.http_get(url.geturl(), etag)
 710
 711     if download is None:
 712         return None, new_etag
 713
 714     with tempfile.NamedTemporaryFile() as fp:
 715         # write and open JAR file
 716         fp.write(download)
 717         jar = zipfile.ZipFile(fp)
 718
 719         # verify that the JAR signature is valid
 720         logging.debug(_('Verifying index signature:'))
 721         common.verify_jar_signature(fp.name)
 722
 723         # get public key and its fingerprint from JAR
 724         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 725
 726         # compare the fingerprint if verify_fingerprint is True
 727         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
 728             raise VerificationException(_("The repository's fingerprint does not match."))
 729
 730         # load repository index from JSON
 731         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
 732         index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
 733         index["repo"]["fingerprint"] = public_key_fingerprint
 734
 735         # turn the apps into App objects
 736         index["apps"] = [metadata.App(app) for app in index["apps"]]
 737
 738         return index, new_etag
 739
 740
 741 def get_public_key_from_jar(jar):
 742     """
 743     Get the public key and its fingerprint from a JAR file.
 744
 745     :raises: VerificationException() if the JAR was not signed exactly once
 746
 747     :param jar: a zipfile.ZipFile object
 748     :return: the public key from the jar and its fingerprint
 749     """
 750     # extract certificate from jar
 751     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
 752     if len(certs) < 1:
 753         raise VerificationException(_("Found no signing certificates for repository."))
 754     if len(certs) > 1:
 755         raise VerificationException(_("Found multiple signing certificates for repository."))
 756
 757     # extract public key from certificate
 758     public_key = common.get_certificate(jar.read(certs[0]))
 759     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 760
 761     return public_key, public_key_fingerprint