fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import tempfile
  31 import urllib.parse
  32 import zipfile
  33 import calendar
  34 from binascii import hexlify, unhexlify
  35 from datetime import datetime
  36 from xml.dom.minidom import Document
  37
  38 from . import _
  39 from . import common
  40 from . import metadata
  41 from . import net
  42 from . import signindex
  43 from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
  44 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
  45
  46
  47 def make(apps, sortedids, apks, repodir, archive):
  48     """Generate the repo index files.
  49
  50     This requires properly initialized options and config objects.
  51
  52     :param apps: fully populated apps list
  53     :param sortedids: app package IDs, sorted
  54     :param apks: full populated apks list
  55     :param repodir: the repo directory
  56     :param archive: True if this is the archive repo, False if it's the
  57                     main one.
  58     """
  59     from fdroidserver.update import METADATA_VERSION
  60
  61     def _resolve_description_link(appid):
  62         if appid in apps:
  63             return "fdroid.app:" + appid, apps[appid].Name
  64         raise MetaDataException("Cannot resolve app id " + appid)
  65
  66     nosigningkey = False
  67     if not common.options.nosign:
  68         if 'repo_keyalias' not in common.config:
  69             nosigningkey = True
  70             logging.critical(_("'repo_keyalias' not found in config.py!"))
  71         if 'keystore' not in common.config:
  72             nosigningkey = True
  73             logging.critical(_("'keystore' not found in config.py!"))
  74         if 'keystorepass' not in common.config:
  75             nosigningkey = True
  76             logging.critical(_("'keystorepass' not found in config.py!"))
  77         if 'keypass' not in common.config:
  78             nosigningkey = True
  79             logging.critical(_("'keypass' not found in config.py!"))
  80         if not os.path.exists(common.config['keystore']):
  81             nosigningkey = True
  82             logging.critical("'" + common.config['keystore'] + "' does not exist!")
  83         if nosigningkey:
  84             raise FDroidException("`fdroid update` requires a signing key, " +
  85                                   "you can create one using: fdroid update --create-key")
  86
  87     repodict = collections.OrderedDict()
  88     repodict['timestamp'] = datetime.utcnow()
  89     repodict['version'] = METADATA_VERSION
  90
  91     if common.config['repo_maxage'] != 0:
  92         repodict['maxage'] = common.config['repo_maxage']
  93
  94     if archive:
  95         repodict['name'] = common.config['archive_name']
  96         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  97         repodict['address'] = common.config['archive_url']
  98         repodict['description'] = common.config['archive_description']
  99         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
 100     else:
 101         repodict['name'] = common.config['repo_name']
 102         repodict['icon'] = os.path.basename(common.config['repo_icon'])
 103         repodict['address'] = common.config['repo_url']
 104         repodict['description'] = common.config['repo_description']
 105         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
 106
 107     mirrorcheckfailed = False
 108     mirrors = []
 109     for mirror in sorted(common.config.get('mirrors', [])):
 110         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 111         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
 112             logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
 113             mirrorcheckfailed = True
 114         # must end with / or urljoin strips a whole path segment
 115         if mirror.endswith('/'):
 116             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
 117         else:
 118             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 119     for mirror in common.config.get('servergitmirrors', []):
 120         for url in get_mirror_service_urls(mirror):
 121             mirrors.append(url + '/' + repodir)
 122     if mirrorcheckfailed:
 123         raise FDroidException(_("Malformed repository mirrors."))
 124     if mirrors:
 125         repodict['mirrors'] = mirrors
 126
 127     appsWithPackages = collections.OrderedDict()
 128     for packageName in sortedids:
 129         app = apps[packageName]
 130         if app['Disabled']:
 131             continue
 132
 133         # only include apps with packages
 134         for apk in apks:
 135             if apk['packageName'] == packageName:
 136                 newapp = copy.copy(app)  # update wiki needs unmodified description
 137                 newapp['Description'] = metadata.description_html(app['Description'],
 138                                                                   _resolve_description_link)
 139                 appsWithPackages[packageName] = newapp
 140                 break
 141
 142     requestsdict = collections.OrderedDict()
 143     for command in ('install', 'uninstall'):
 144         packageNames = []
 145         key = command + '_list'
 146         if key in common.config:
 147             if isinstance(common.config[key], str):
 148                 packageNames = [common.config[key]]
 149             elif all(isinstance(item, str) for item in common.config[key]):
 150                 packageNames = common.config[key]
 151             else:
 152                 raise TypeError(_('only accepts strings, lists, and tuples'))
 153         requestsdict[command] = packageNames
 154
 155     fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
 156
 157     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
 158             fdroid_signing_key_fingerprints)
 159     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
 160             fdroid_signing_key_fingerprints)
 161
 162
 163 def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 164
 165     def _index_encoder_default(obj):
 166         if isinstance(obj, set):
 167             return sorted(list(obj))
 168         if isinstance(obj, datetime):
 169             # Java prefers milliseconds
 170             # we also need to accound for time zone/daylight saving time
 171             return int(calendar.timegm(obj.timetuple()) * 1000)
 172         if isinstance(obj, dict):
 173             d = collections.OrderedDict()
 174             for key in sorted(obj.keys()):
 175                 d[key] = obj[key]
 176             return d
 177         raise TypeError(repr(obj) + " is not JSON serializable")
 178
 179     output = collections.OrderedDict()
 180     output['repo'] = repodict
 181     output['requests'] = requestsdict
 182
 183     # establish sort order of the index
 184     v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
 185
 186     appslist = []
 187     output['apps'] = appslist
 188     for packageName, appdict in apps.items():
 189         d = collections.OrderedDict()
 190         appslist.append(d)
 191         for k, v in sorted(appdict.items()):
 192             if not v:
 193                 continue
 194             if k in ('builds', 'comments', 'metadatapath',
 195                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 196                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 197                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 198                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 199                 continue
 200
 201             # name things after the App class fields in fdroidclient
 202             if k == 'id':
 203                 k = 'packageName'
 204             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 205                 k = 'suggestedVersionCode'
 206             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 207                 k = 'suggestedVersionName'
 208             elif k == 'AutoName':
 209                 if 'Name' not in apps[packageName]:
 210                     d['name'] = v
 211                 continue
 212             else:
 213                 k = k[:1].lower() + k[1:]
 214             d[k] = v
 215
 216     # establish sort order in localized dicts
 217     for app in output['apps']:
 218         localized = app.get('localized')
 219         if localized:
 220             lordered = collections.OrderedDict()
 221             for lkey, lvalue in sorted(localized.items()):
 222                 lordered[lkey] = collections.OrderedDict()
 223                 for ikey, iname in sorted(lvalue.items()):
 224                     lordered[lkey][ikey] = iname
 225             app['localized'] = lordered
 226
 227     output_packages = collections.OrderedDict()
 228     output['packages'] = output_packages
 229     for package in packages:
 230         packageName = package['packageName']
 231         if packageName not in apps:
 232             logging.info(_('Ignoring package without metadata: ') + package['apkName'])
 233             continue
 234         if packageName in output_packages:
 235             packagelist = output_packages[packageName]
 236         else:
 237             packagelist = []
 238             output_packages[packageName] = packagelist
 239         d = collections.OrderedDict()
 240         packagelist.append(d)
 241         for k, v in sorted(package.items()):
 242             if not v:
 243                 continue
 244             if k in ('icon', 'icons', 'icons_src', 'name', ):
 245                 continue
 246             d[k] = v
 247
 248     json_name = 'index-v1.json'
 249     index_file = os.path.join(repodir, json_name)
 250     with open(index_file, 'w') as fp:
 251         if common.options.pretty:
 252             json.dump(output, fp, default=_index_encoder_default, indent=2)
 253         else:
 254             json.dump(output, fp, default=_index_encoder_default)
 255
 256     if common.options.nosign:
 257         logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
 258     else:
 259         signindex.config = common.config
 260         signindex.sign_index_v1(repodir, json_name)
 261
 262
 263 def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
 264     """Sorts the supplied list to ensure a deterministic sort order for
 265     package entries in the index file. This sort-order also expresses
 266     installation preference to the clients.
 267     (First in this list = first to install)
 268
 269     :param packages: list of packages which need to be sorted before but into index file.
 270     """
 271
 272     GROUP_DEV_SIGNED = 1
 273     GROUP_FDROID_SIGNED = 2
 274     GROUP_OTHER_SIGNED = 3
 275
 276     def v1_sort_keys(package):
 277         packageName = package.get('packageName', None)
 278
 279         sig = package.get('signer', None)
 280
 281         dev_sig = common.metadata_find_developer_signature(packageName)
 282         group = GROUP_OTHER_SIGNED
 283         if dev_sig and dev_sig == sig:
 284             group = GROUP_DEV_SIGNED
 285         else:
 286             fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
 287             if fdroidsig and fdroidsig == sig:
 288                 group = GROUP_FDROID_SIGNED
 289
 290         versionCode = None
 291         if package.get('versionCode', None):
 292             versionCode = -int(package['versionCode'])
 293
 294         return(packageName, group, sig, versionCode)
 295
 296     packages.sort(key=v1_sort_keys)
 297
 298
 299 def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 300     """
 301     aka index.jar aka index.xml
 302     """
 303
 304     doc = Document()
 305
 306     def addElement(name, value, doc, parent):
 307         el = doc.createElement(name)
 308         el.appendChild(doc.createTextNode(value))
 309         parent.appendChild(el)
 310
 311     def addElementNonEmpty(name, value, doc, parent):
 312         if not value:
 313             return
 314         addElement(name, value, doc, parent)
 315
 316     def addElementIfInApk(name, apk, key, doc, parent):
 317         if key not in apk:
 318             return
 319         value = str(apk[key])
 320         addElement(name, value, doc, parent)
 321
 322     def addElementCDATA(name, value, doc, parent):
 323         el = doc.createElement(name)
 324         el.appendChild(doc.createCDATASection(value))
 325         parent.appendChild(el)
 326
 327     def addElementCheckLocalized(name, app, key, doc, parent, default=''):
 328         '''Fill in field from metadata or localized block
 329
 330         For name/summary/description, they can come only from the app source,
 331         or from a dir in fdroiddata.  They can be entirely missing from the
 332         metadata file if there is localized versions.  This will fetch those
 333         from the localized version if its not available in the metadata file.
 334         '''
 335
 336         el = doc.createElement(name)
 337         value = app.get(key)
 338         lkey = key[:1].lower() + key[1:]
 339         localized = app.get('localized')
 340         if not value and localized:
 341             for lang in ['en-US'] + [x for x in localized.keys()]:
 342                 if not lang.startswith('en'):
 343                     continue
 344                 if lang in localized:
 345                     value = localized[lang].get(lkey)
 346                     if value:
 347                         break
 348         if not value and localized and len(localized) > 1:
 349             lang = list(localized.keys())[0]
 350             value = localized[lang].get(lkey)
 351         if not value:
 352             value = default
 353         el.appendChild(doc.createTextNode(value))
 354         parent.appendChild(el)
 355
 356     root = doc.createElement("fdroid")
 357     doc.appendChild(root)
 358
 359     repoel = doc.createElement("repo")
 360
 361     repoel.setAttribute("name", repodict['name'])
 362     if 'maxage' in repodict:
 363         repoel.setAttribute("maxage", str(repodict['maxage']))
 364     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 365     repoel.setAttribute("url", repodict['address'])
 366     addElement('description', repodict['description'], doc, repoel)
 367     for mirror in repodict.get('mirrors', []):
 368         addElement('mirror', mirror, doc, repoel)
 369
 370     repoel.setAttribute("version", str(repodict['version']))
 371     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 372
 373     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 374     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 375     root.appendChild(repoel)
 376
 377     for command in ('install', 'uninstall'):
 378         for packageName in requestsdict[command]:
 379             element = doc.createElement(command)
 380             root.appendChild(element)
 381             element.setAttribute('packageName', packageName)
 382
 383     for appid, appdict in apps.items():
 384         app = metadata.App(appdict)
 385
 386         if app.Disabled is not None:
 387             continue
 388
 389         # Get a list of the apks for this app...
 390         apklist = []
 391         apksbyversion = collections.defaultdict(lambda: [])
 392         for apk in apks:
 393             if apk.get('versionCode') and apk.get('packageName') == appid:
 394                 apksbyversion[apk['versionCode']].append(apk)
 395         for versionCode, apksforver in apksbyversion.items():
 396             fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
 397             fdroid_signed_apk = None
 398             name_match_apk = None
 399             for x in apksforver:
 400                 if fdroidsig and x.get('signer', None) == fdroidsig:
 401                     fdroid_signed_apk = x
 402                 if common.apk_release_filename.match(x.get('apkName', '')):
 403                     name_match_apk = x
 404             # choose which of the available versions is most
 405             # suiteable for index v0
 406             if fdroid_signed_apk:
 407                 apklist.append(fdroid_signed_apk)
 408             elif name_match_apk:
 409                 apklist.append(name_match_apk)
 410             else:
 411                 apklist.append(apksforver[0])
 412
 413         if len(apklist) == 0:
 414             continue
 415
 416         apel = doc.createElement("application")
 417         apel.setAttribute("id", app.id)
 418         root.appendChild(apel)
 419
 420         addElement('id', app.id, doc, apel)
 421         if app.added:
 422             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 423         if app.lastUpdated:
 424             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 425
 426         addElementCheckLocalized('name', app, 'Name', doc, apel)
 427         addElementCheckLocalized('summary', app, 'Summary', doc, apel)
 428
 429         if app.icon:
 430             addElement('icon', app.icon, doc, apel)
 431
 432         addElementCheckLocalized('desc', app, 'Description', doc, apel,
 433                                  '<p>No description available</p>')
 434
 435         addElement('license', app.License, doc, apel)
 436         if app.Categories:
 437             addElement('categories', ','.join(app.Categories), doc, apel)
 438             # We put the first (primary) category in LAST, which will have
 439             # the desired effect of making clients that only understand one
 440             # category see that one.
 441             addElement('category', app.Categories[0], doc, apel)
 442         addElement('web', app.WebSite, doc, apel)
 443         addElement('source', app.SourceCode, doc, apel)
 444         addElement('tracker', app.IssueTracker, doc, apel)
 445         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 446         addElementNonEmpty('author', app.AuthorName, doc, apel)
 447         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 448         addElementNonEmpty('donate', app.Donate, doc, apel)
 449         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 450         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 451         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 452
 453         # These elements actually refer to the current version (i.e. which
 454         # one is recommended. They are historically mis-named, and need
 455         # changing, but stay like this for now to support existing clients.
 456         addElement('marketversion', app.CurrentVersion, doc, apel)
 457         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 458
 459         if app.Provides:
 460             pv = app.Provides.split(',')
 461             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 462         if app.RequiresRoot:
 463             addElement('requirements', 'root', doc, apel)
 464
 465         # Sort the apk list into version order, just so the web site
 466         # doesn't have to do any work by default...
 467         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 468
 469         if 'antiFeatures' in apklist[0]:
 470             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 471         if app.AntiFeatures:
 472             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 473
 474         # Check for duplicates - they will make the client unhappy...
 475         for i in range(len(apklist) - 1):
 476             first = apklist[i]
 477             second = apklist[i + 1]
 478             if first['versionCode'] == second['versionCode'] \
 479                and first['sig'] == second['sig']:
 480                 if first['hash'] == second['hash']:
 481                     raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
 482                         repodir, first['apkName'], second['apkName']))
 483                 else:
 484                     raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
 485                         repodir, first['apkName'], second['apkName']))
 486
 487         current_version_code = 0
 488         current_version_file = None
 489         for apk in apklist:
 490             file_extension = common.get_file_extension(apk['apkName'])
 491             # find the APK for the "Current Version"
 492             if current_version_code < apk['versionCode']:
 493                 current_version_code = apk['versionCode']
 494             if current_version_code < int(app.CurrentVersionCode):
 495                 current_version_file = apk['apkName']
 496
 497             apkel = doc.createElement("package")
 498             apel.appendChild(apkel)
 499             addElement('version', apk['versionName'], doc, apkel)
 500             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 501             addElement('apkname', apk['apkName'], doc, apkel)
 502             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 503
 504             hashel = doc.createElement("hash")
 505             hashel.setAttribute('type', 'sha256')
 506             hashel.appendChild(doc.createTextNode(apk['hash']))
 507             apkel.appendChild(hashel)
 508
 509             addElement('size', str(apk['size']), doc, apkel)
 510             addElementIfInApk('sdkver', apk,
 511                               'minSdkVersion', doc, apkel)
 512             addElementIfInApk('targetSdkVersion', apk,
 513                               'targetSdkVersion', doc, apkel)
 514             addElementIfInApk('maxsdkver', apk,
 515                               'maxSdkVersion', doc, apkel)
 516             addElementIfInApk('obbMainFile', apk,
 517                               'obbMainFile', doc, apkel)
 518             addElementIfInApk('obbMainFileSha256', apk,
 519                               'obbMainFileSha256', doc, apkel)
 520             addElementIfInApk('obbPatchFile', apk,
 521                               'obbPatchFile', doc, apkel)
 522             addElementIfInApk('obbPatchFileSha256', apk,
 523                               'obbPatchFileSha256', doc, apkel)
 524             if 'added' in apk:
 525                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 526
 527             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 528                 addElement('sig', apk['sig'], doc, apkel)
 529
 530                 old_permissions = set()
 531                 sorted_permissions = sorted(apk['uses-permission'])
 532                 for perm in sorted_permissions:
 533                     perm_name = perm.name
 534                     if perm_name.startswith("android.permission."):
 535                         perm_name = perm_name[19:]
 536                     old_permissions.add(perm_name)
 537                 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 538
 539                 for permission in sorted_permissions:
 540                     permel = doc.createElement('uses-permission')
 541                     permel.setAttribute('name', permission.name)
 542                     if permission.maxSdkVersion is not None:
 543                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 544                         apkel.appendChild(permel)
 545                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 546                     permel = doc.createElement('uses-permission-sdk-23')
 547                     permel.setAttribute('name', permission_sdk_23.name)
 548                     if permission_sdk_23.maxSdkVersion is not None:
 549                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 550                         apkel.appendChild(permel)
 551                 if 'nativecode' in apk:
 552                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 553                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 554
 555         if current_version_file is not None \
 556                 and common.config['make_current_version_link'] \
 557                 and repodir == 'repo':  # only create these
 558             namefield = common.config['current_version_name_source']
 559             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
 560             apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
 561             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
 562             if os.path.islink(apklinkname):
 563                 os.remove(apklinkname)
 564             os.symlink(current_version_path, apklinkname)
 565             # also symlink gpg signature, if it exists
 566             for extension in (b'.asc', b'.sig'):
 567                 sigfile_path = current_version_path + extension
 568                 if os.path.exists(sigfile_path):
 569                     siglinkname = apklinkname + extension
 570                     if os.path.islink(siglinkname):
 571                         os.remove(siglinkname)
 572                     os.symlink(sigfile_path, siglinkname)
 573
 574     if common.options.pretty:
 575         output = doc.toprettyxml(encoding='utf-8')
 576     else:
 577         output = doc.toxml(encoding='utf-8')
 578
 579     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 580         f.write(output)
 581
 582     if 'repo_keyalias' in common.config:
 583
 584         if common.options.nosign:
 585             logging.info(_("Creating unsigned index in preparation for signing"))
 586         else:
 587             logging.info(_("Creating signed index with this key (SHA256):"))
 588             logging.info("%s" % repo_pubkey_fingerprint)
 589
 590         # Create a jar of the index...
 591         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 592         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 593         if p.returncode != 0:
 594             raise FDroidException("Failed to create {0}".format(jar_output))
 595
 596         # Sign the index...
 597         signed = os.path.join(repodir, 'index.jar')
 598         if common.options.nosign:
 599             # Remove old signed index if not signing
 600             if os.path.exists(signed):
 601                 os.remove(signed)
 602         else:
 603             signindex.config = common.config
 604             signindex.sign_jar(signed)
 605
 606     # Copy the repo icon into the repo directory...
 607     icon_dir = os.path.join(repodir, 'icons')
 608     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 609     shutil.copyfile(common.config['repo_icon'], iconfilename)
 610
 611
 612 def extract_pubkey():
 613     """
 614     Extracts and returns the repository's public key from the keystore.
 615     :return: public key in hex, repository fingerprint
 616     """
 617     if 'repo_pubkey' in common.config:
 618         pubkey = unhexlify(common.config['repo_pubkey'])
 619     else:
 620         env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
 621         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 622                               '-alias', common.config['repo_keyalias'],
 623                               '-keystore', common.config['keystore'],
 624                               '-storepass:env', 'FDROID_KEY_STORE_PASS']
 625                              + common.config['smartcardoptions'],
 626                              envs=env_vars, output=False, stderr_to_stdout=False)
 627         if p.returncode != 0 or len(p.output) < 20:
 628             msg = "Failed to get repo pubkey!"
 629             if common.config['keystore'] == 'NONE':
 630                 msg += ' Is your crypto smartcard plugged in?'
 631             raise FDroidException(msg)
 632         pubkey = p.output
 633     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 634     return hexlify(pubkey), repo_pubkey_fingerprint
 635
 636
 637 def get_mirror_service_urls(url):
 638     '''Get direct URLs from git service for use by fdroidclient
 639
 640     Via 'servergitmirrors', fdroidserver can create and push a mirror
 641     to certain well known git services like gitlab or github.  This
 642     will always use the 'master' branch since that is the default
 643     branch in git. The files are then accessible via alternate URLs,
 644     where they are served in their raw format via a CDN rather than
 645     from git.
 646     '''
 647
 648     if url.startswith('git@'):
 649         url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 650
 651     segments = url.split("/")
 652
 653     if segments[4].endswith('.git'):
 654         segments[4] = segments[4][:-4]
 655
 656     hostname = segments[2]
 657     user = segments[3]
 658     repo = segments[4]
 659     branch = "master"
 660     folder = "fdroid"
 661
 662     urls = []
 663     if hostname == "github.com":
 664         # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
 665         segments[2] = "raw.githubusercontent.com"
 666         segments.extend([branch, folder])
 667         urls.append('/'.join(segments))
 668     elif hostname == "gitlab.com":
 669         # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
 670         gitlab_raw = segments + ['raw', branch, folder]
 671         urls.append('/'.join(gitlab_raw))
 672         # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
 673         gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
 674         urls.append('/'.join(gitlab_pages))
 675         return urls
 676
 677     return urls
 678
 679
 680 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
 681     """
 682     Downloads the repository index from the given :param url_str
 683     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 684
 685     :raises: VerificationException() if the repository could not be verified
 686
 687     :return: A tuple consisting of:
 688         - The index in JSON format or None if the index did not change
 689         - The new eTag as returned by the HTTP request
 690     """
 691     url = urllib.parse.urlsplit(url_str)
 692
 693     fingerprint = None
 694     if verify_fingerprint:
 695         query = urllib.parse.parse_qs(url.query)
 696         if 'fingerprint' not in query:
 697             raise VerificationException(_("No fingerprint in URL."))
 698         fingerprint = query['fingerprint'][0]
 699
 700     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
 701     download, new_etag = net.http_get(url.geturl(), etag)
 702
 703     if download is None:
 704         return None, new_etag
 705
 706     with tempfile.NamedTemporaryFile() as fp:
 707         # write and open JAR file
 708         fp.write(download)
 709         jar = zipfile.ZipFile(fp)
 710
 711         # verify that the JAR signature is valid
 712         common.verify_jar_signature(fp.name)
 713
 714         # get public key and its fingerprint from JAR
 715         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 716
 717         # compare the fingerprint if verify_fingerprint is True
 718         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
 719             raise VerificationException(_("The repository's fingerprint does not match."))
 720
 721         # load repository index from JSON
 722         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
 723         index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
 724         index["repo"]["fingerprint"] = public_key_fingerprint
 725
 726         # turn the apps into App objects
 727         index["apps"] = [metadata.App(app) for app in index["apps"]]
 728
 729         return index, new_etag
 730
 731
 732 def get_public_key_from_jar(jar):
 733     """
 734     Get the public key and its fingerprint from a JAR file.
 735
 736     :raises: VerificationException() if the JAR was not signed exactly once
 737
 738     :param jar: a zipfile.ZipFile object
 739     :return: the public key from the jar and its fingerprint
 740     """
 741     # extract certificate from jar
 742     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
 743     if len(certs) < 1:
 744         raise VerificationException(_("Found no signing certificates for repository."))
 745     if len(certs) > 1:
 746         raise VerificationException(_("Found multiple signing certificates for repository."))
 747
 748     # extract public key from certificate
 749     public_key = common.get_certificate(jar.read(certs[0]))
 750     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 751
 752     return public_key, public_key_fingerprint