fdroidserver/index.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2017, Torsten Grote <t at grobox dot de>
   5 # Copyright (C) 2016, Blue Jay Wireless
   6 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   7 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   8 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   9 #
  10 # This program is free software: you can redistribute it and/or modify
  11 # it under the terms of the GNU Affero General Public License as published by
  12 # the Free Software Foundation, either version 3 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU Affero General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Affero General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 import collections
  24 import copy
  25 import json
  26 import logging
  27 import os
  28 import re
  29 import shutil
  30 import tempfile
  31 import urllib.parse
  32 import zipfile
  33 import calendar
  34 from binascii import hexlify, unhexlify
  35 from datetime import datetime
  36 from xml.dom.minidom import Document
  37
  38 from . import _
  39 from . import common
  40 from . import metadata
  41 from . import net
  42 from . import signindex
  43 from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
  44 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
  45
  46
  47 def make(apps, sortedids, apks, repodir, archive):
  48     """Generate the repo index files.
  49
  50     This requires properly initialized options and config objects.
  51
  52     :param apps: fully populated apps list
  53     :param sortedids: app package IDs, sorted
  54     :param apks: full populated apks list
  55     :param repodir: the repo directory
  56     :param archive: True if this is the archive repo, False if it's the
  57                     main one.
  58     """
  59     from fdroidserver.update import METADATA_VERSION
  60
  61     def _resolve_description_link(appid):
  62         if appid in apps:
  63             return "fdroid.app:" + appid, apps[appid].Name
  64         raise MetaDataException("Cannot resolve app id " + appid)
  65
  66     if not common.options.nosign:
  67         common.assert_config_keystore(common.config)
  68
  69     repodict = collections.OrderedDict()
  70     repodict['timestamp'] = datetime.utcnow()
  71     repodict['version'] = METADATA_VERSION
  72
  73     if common.config['repo_maxage'] != 0:
  74         repodict['maxage'] = common.config['repo_maxage']
  75
  76     if archive:
  77         repodict['name'] = common.config['archive_name']
  78         repodict['icon'] = os.path.basename(common.config['archive_icon'])
  79         repodict['address'] = common.config['archive_url']
  80         repodict['description'] = common.config['archive_description']
  81         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path)
  82     else:
  83         repodict['name'] = common.config['repo_name']
  84         repodict['icon'] = os.path.basename(common.config['repo_icon'])
  85         repodict['address'] = common.config['repo_url']
  86         repodict['description'] = common.config['repo_description']
  87         urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path)
  88
  89     mirrorcheckfailed = False
  90     mirrors = []
  91     for mirror in sorted(common.config.get('mirrors', [])):
  92         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
  93         if common.config.get('nonstandardwebroot') is not True and base != 'fdroid':
  94             logging.error(_("mirror '%s' does not end with 'fdroid'!") % mirror)
  95             mirrorcheckfailed = True
  96         # must end with / or urljoin strips a whole path segment
  97         if mirror.endswith('/'):
  98             mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
  99         else:
 100             mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
 101     for mirror in common.config.get('servergitmirrors', []):
 102         for url in get_mirror_service_urls(mirror):
 103             mirrors.append(url + '/' + repodir)
 104     if mirrorcheckfailed:
 105         raise FDroidException(_("Malformed repository mirrors."))
 106     if mirrors:
 107         repodict['mirrors'] = mirrors
 108
 109     appsWithPackages = collections.OrderedDict()
 110     for packageName in sortedids:
 111         app = apps[packageName]
 112         if app['Disabled']:
 113             continue
 114
 115         # only include apps with packages
 116         for apk in apks:
 117             if apk['packageName'] == packageName:
 118                 newapp = copy.copy(app)  # update wiki needs unmodified description
 119                 newapp['Description'] = metadata.description_html(app['Description'],
 120                                                                   _resolve_description_link)
 121                 appsWithPackages[packageName] = newapp
 122                 break
 123
 124     requestsdict = collections.OrderedDict()
 125     for command in ('install', 'uninstall'):
 126         packageNames = []
 127         key = command + '_list'
 128         if key in common.config:
 129             if isinstance(common.config[key], str):
 130                 packageNames = [common.config[key]]
 131             elif all(isinstance(item, str) for item in common.config[key]):
 132                 packageNames = common.config[key]
 133             else:
 134                 raise TypeError(_('only accepts strings, lists, and tuples'))
 135         requestsdict[command] = packageNames
 136
 137     fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
 138
 139     make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
 140             fdroid_signing_key_fingerprints)
 141     make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
 142             fdroid_signing_key_fingerprints)
 143
 144
 145 def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 146
 147     def _index_encoder_default(obj):
 148         if isinstance(obj, set):
 149             return sorted(list(obj))
 150         if isinstance(obj, datetime):
 151             # Java prefers milliseconds
 152             # we also need to accound for time zone/daylight saving time
 153             return int(calendar.timegm(obj.timetuple()) * 1000)
 154         if isinstance(obj, dict):
 155             d = collections.OrderedDict()
 156             for key in sorted(obj.keys()):
 157                 d[key] = obj[key]
 158             return d
 159         raise TypeError(repr(obj) + " is not JSON serializable")
 160
 161     output = collections.OrderedDict()
 162     output['repo'] = repodict
 163     output['requests'] = requestsdict
 164
 165     # establish sort order of the index
 166     v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
 167
 168     appslist = []
 169     output['apps'] = appslist
 170     for packageName, appdict in apps.items():
 171         d = collections.OrderedDict()
 172         appslist.append(d)
 173         for k, v in sorted(appdict.items()):
 174             if not v:
 175                 continue
 176             if k in ('builds', 'comments', 'metadatapath',
 177                      'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
 178                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
 179                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
 180                      'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
 181                 continue
 182
 183             # name things after the App class fields in fdroidclient
 184             if k == 'id':
 185                 k = 'packageName'
 186             elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
 187                 k = 'suggestedVersionCode'
 188             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
 189                 k = 'suggestedVersionName'
 190             elif k == 'AutoName':
 191                 if 'Name' not in apps[packageName]:
 192                     d['name'] = v
 193                 continue
 194             else:
 195                 k = k[:1].lower() + k[1:]
 196             d[k] = v
 197
 198     # establish sort order in localized dicts
 199     for app in output['apps']:
 200         localized = app.get('localized')
 201         if localized:
 202             lordered = collections.OrderedDict()
 203             for lkey, lvalue in sorted(localized.items()):
 204                 lordered[lkey] = collections.OrderedDict()
 205                 for ikey, iname in sorted(lvalue.items()):
 206                     lordered[lkey][ikey] = iname
 207             app['localized'] = lordered
 208
 209     output_packages = collections.OrderedDict()
 210     output['packages'] = output_packages
 211     for package in packages:
 212         packageName = package['packageName']
 213         if packageName not in apps:
 214             logging.info(_('Ignoring package without metadata: ') + package['apkName'])
 215             continue
 216         if packageName in output_packages:
 217             packagelist = output_packages[packageName]
 218         else:
 219             packagelist = []
 220             output_packages[packageName] = packagelist
 221         d = collections.OrderedDict()
 222         packagelist.append(d)
 223         for k, v in sorted(package.items()):
 224             if not v:
 225                 continue
 226             if k in ('icon', 'icons', 'icons_src', 'name', ):
 227                 continue
 228             d[k] = v
 229
 230     json_name = 'index-v1.json'
 231     index_file = os.path.join(repodir, json_name)
 232     with open(index_file, 'w') as fp:
 233         if common.options.pretty:
 234             json.dump(output, fp, default=_index_encoder_default, indent=2)
 235         else:
 236             json.dump(output, fp, default=_index_encoder_default)
 237
 238     if common.options.nosign:
 239         logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
 240     else:
 241         signindex.config = common.config
 242         signindex.sign_index_v1(repodir, json_name)
 243
 244
 245 def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
 246     """Sorts the supplied list to ensure a deterministic sort order for
 247     package entries in the index file. This sort-order also expresses
 248     installation preference to the clients.
 249     (First in this list = first to install)
 250
 251     :param packages: list of packages which need to be sorted before but into index file.
 252     """
 253
 254     GROUP_DEV_SIGNED = 1
 255     GROUP_FDROID_SIGNED = 2
 256     GROUP_OTHER_SIGNED = 3
 257
 258     def v1_sort_keys(package):
 259         packageName = package.get('packageName', None)
 260
 261         sig = package.get('signer', None)
 262
 263         dev_sig = common.metadata_find_developer_signature(packageName)
 264         group = GROUP_OTHER_SIGNED
 265         if dev_sig and dev_sig == sig:
 266             group = GROUP_DEV_SIGNED
 267         else:
 268             fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
 269             if fdroidsig and fdroidsig == sig:
 270                 group = GROUP_FDROID_SIGNED
 271
 272         versionCode = None
 273         if package.get('versionCode', None):
 274             versionCode = -int(package['versionCode'])
 275
 276         return(packageName, group, sig, versionCode)
 277
 278     packages.sort(key=v1_sort_keys)
 279
 280
 281 def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 282     """
 283     aka index.jar aka index.xml
 284     """
 285
 286     doc = Document()
 287
 288     def addElement(name, value, doc, parent):
 289         el = doc.createElement(name)
 290         el.appendChild(doc.createTextNode(value))
 291         parent.appendChild(el)
 292
 293     def addElementNonEmpty(name, value, doc, parent):
 294         if not value:
 295             return
 296         addElement(name, value, doc, parent)
 297
 298     def addElementIfInApk(name, apk, key, doc, parent):
 299         if key not in apk:
 300             return
 301         value = str(apk[key])
 302         addElement(name, value, doc, parent)
 303
 304     def addElementCDATA(name, value, doc, parent):
 305         el = doc.createElement(name)
 306         el.appendChild(doc.createCDATASection(value))
 307         parent.appendChild(el)
 308
 309     def addElementCheckLocalized(name, app, key, doc, parent, default=''):
 310         '''Fill in field from metadata or localized block
 311
 312         For name/summary/description, they can come only from the app source,
 313         or from a dir in fdroiddata.  They can be entirely missing from the
 314         metadata file if there is localized versions.  This will fetch those
 315         from the localized version if its not available in the metadata file.
 316         '''
 317
 318         el = doc.createElement(name)
 319         value = app.get(key)
 320         lkey = key[:1].lower() + key[1:]
 321         localized = app.get('localized')
 322         if not value and localized:
 323             for lang in ['en-US'] + [x for x in localized.keys()]:
 324                 if not lang.startswith('en'):
 325                     continue
 326                 if lang in localized:
 327                     value = localized[lang].get(lkey)
 328                     if value:
 329                         break
 330         if not value and localized and len(localized) > 1:
 331             lang = list(localized.keys())[0]
 332             value = localized[lang].get(lkey)
 333         if not value:
 334             value = default
 335         el.appendChild(doc.createTextNode(value))
 336         parent.appendChild(el)
 337
 338     root = doc.createElement("fdroid")
 339     doc.appendChild(root)
 340
 341     repoel = doc.createElement("repo")
 342
 343     repoel.setAttribute("name", repodict['name'])
 344     if 'maxage' in repodict:
 345         repoel.setAttribute("maxage", str(repodict['maxage']))
 346     repoel.setAttribute("icon", os.path.basename(repodict['icon']))
 347     repoel.setAttribute("url", repodict['address'])
 348     addElement('description', repodict['description'], doc, repoel)
 349     for mirror in repodict.get('mirrors', []):
 350         addElement('mirror', mirror, doc, repoel)
 351
 352     repoel.setAttribute("version", str(repodict['version']))
 353     repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
 354
 355     pubkey, repo_pubkey_fingerprint = extract_pubkey()
 356     repoel.setAttribute("pubkey", pubkey.decode('utf-8'))
 357     root.appendChild(repoel)
 358
 359     for command in ('install', 'uninstall'):
 360         for packageName in requestsdict[command]:
 361             element = doc.createElement(command)
 362             root.appendChild(element)
 363             element.setAttribute('packageName', packageName)
 364
 365     for appid, appdict in apps.items():
 366         app = metadata.App(appdict)
 367
 368         if app.Disabled is not None:
 369             continue
 370
 371         # Get a list of the apks for this app...
 372         apklist = []
 373         apksbyversion = collections.defaultdict(lambda: [])
 374         for apk in apks:
 375             if apk.get('versionCode') and apk.get('packageName') == appid:
 376                 apksbyversion[apk['versionCode']].append(apk)
 377         for versionCode, apksforver in apksbyversion.items():
 378             fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
 379             fdroid_signed_apk = None
 380             name_match_apk = None
 381             for x in apksforver:
 382                 if fdroidsig and x.get('signer', None) == fdroidsig:
 383                     fdroid_signed_apk = x
 384                 if common.apk_release_filename.match(x.get('apkName', '')):
 385                     name_match_apk = x
 386             # choose which of the available versions is most
 387             # suiteable for index v0
 388             if fdroid_signed_apk:
 389                 apklist.append(fdroid_signed_apk)
 390             elif name_match_apk:
 391                 apklist.append(name_match_apk)
 392             else:
 393                 apklist.append(apksforver[0])
 394
 395         if len(apklist) == 0:
 396             continue
 397
 398         apel = doc.createElement("application")
 399         apel.setAttribute("id", app.id)
 400         root.appendChild(apel)
 401
 402         addElement('id', app.id, doc, apel)
 403         if app.added:
 404             addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
 405         if app.lastUpdated:
 406             addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
 407
 408         addElementCheckLocalized('name', app, 'Name', doc, apel)
 409         addElementCheckLocalized('summary', app, 'Summary', doc, apel)
 410
 411         if app.icon:
 412             addElement('icon', app.icon, doc, apel)
 413
 414         addElementCheckLocalized('desc', app, 'Description', doc, apel,
 415                                  '<p>No description available</p>')
 416
 417         addElement('license', app.License, doc, apel)
 418         if app.Categories:
 419             addElement('categories', ','.join(app.Categories), doc, apel)
 420             # We put the first (primary) category in LAST, which will have
 421             # the desired effect of making clients that only understand one
 422             # category see that one.
 423             addElement('category', app.Categories[0], doc, apel)
 424         addElement('web', app.WebSite, doc, apel)
 425         addElement('source', app.SourceCode, doc, apel)
 426         addElement('tracker', app.IssueTracker, doc, apel)
 427         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 428         addElementNonEmpty('author', app.AuthorName, doc, apel)
 429         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 430         addElementNonEmpty('donate', app.Donate, doc, apel)
 431         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 432         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 433         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 434         addElementNonEmpty('liberapay', app.LiberapayID, doc, apel)
 435
 436         # These elements actually refer to the current version (i.e. which
 437         # one is recommended. They are historically mis-named, and need
 438         # changing, but stay like this for now to support existing clients.
 439         addElement('marketversion', app.CurrentVersion, doc, apel)
 440         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 441
 442         if app.Provides:
 443             pv = app.Provides.split(',')
 444             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 445         if app.RequiresRoot:
 446             addElement('requirements', 'root', doc, apel)
 447
 448         # Sort the apk list into version order, just so the web site
 449         # doesn't have to do any work by default...
 450         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 451
 452         if 'antiFeatures' in apklist[0]:
 453             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
 454         if app.AntiFeatures:
 455             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
 456
 457         # Check for duplicates - they will make the client unhappy...
 458         for i in range(len(apklist) - 1):
 459             first = apklist[i]
 460             second = apklist[i + 1]
 461             if first['versionCode'] == second['versionCode'] \
 462                and first['sig'] == second['sig']:
 463                 if first['hash'] == second['hash']:
 464                     raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format(
 465                         repodir, first['apkName'], second['apkName']))
 466                 else:
 467                     raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format(
 468                         repodir, first['apkName'], second['apkName']))
 469
 470         current_version_code = 0
 471         current_version_file = None
 472         for apk in apklist:
 473             file_extension = common.get_file_extension(apk['apkName'])
 474             # find the APK for the "Current Version"
 475             if current_version_code < apk['versionCode']:
 476                 current_version_code = apk['versionCode']
 477             if current_version_code < int(app.CurrentVersionCode):
 478                 current_version_file = apk['apkName']
 479
 480             apkel = doc.createElement("package")
 481             apel.appendChild(apkel)
 482             addElement('version', apk['versionName'], doc, apkel)
 483             addElement('versioncode', str(apk['versionCode']), doc, apkel)
 484             addElement('apkname', apk['apkName'], doc, apkel)
 485             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
 486
 487             hashel = doc.createElement("hash")
 488             hashel.setAttribute('type', 'sha256')
 489             hashel.appendChild(doc.createTextNode(apk['hash']))
 490             apkel.appendChild(hashel)
 491
 492             addElement('size', str(apk['size']), doc, apkel)
 493             addElementIfInApk('sdkver', apk,
 494                               'minSdkVersion', doc, apkel)
 495             addElementIfInApk('targetSdkVersion', apk,
 496                               'targetSdkVersion', doc, apkel)
 497             addElementIfInApk('maxsdkver', apk,
 498                               'maxSdkVersion', doc, apkel)
 499             addElementIfInApk('obbMainFile', apk,
 500                               'obbMainFile', doc, apkel)
 501             addElementIfInApk('obbMainFileSha256', apk,
 502                               'obbMainFileSha256', doc, apkel)
 503             addElementIfInApk('obbPatchFile', apk,
 504                               'obbPatchFile', doc, apkel)
 505             addElementIfInApk('obbPatchFileSha256', apk,
 506                               'obbPatchFileSha256', doc, apkel)
 507             if 'added' in apk:
 508                 addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 509
 510             if file_extension == 'apk':  # sig is required for APKs, but only APKs
 511                 addElement('sig', apk['sig'], doc, apkel)
 512
 513                 old_permissions = set()
 514                 sorted_permissions = sorted(apk['uses-permission'])
 515                 for perm in sorted_permissions:
 516                     perm_name = perm.name
 517                     if perm_name.startswith("android.permission."):
 518                         perm_name = perm_name[19:]
 519                     old_permissions.add(perm_name)
 520                 addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 521
 522                 for permission in sorted_permissions:
 523                     permel = doc.createElement('uses-permission')
 524                     permel.setAttribute('name', permission.name)
 525                     if permission.maxSdkVersion is not None:
 526                         permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
 527                         apkel.appendChild(permel)
 528                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
 529                     permel = doc.createElement('uses-permission-sdk-23')
 530                     permel.setAttribute('name', permission_sdk_23.name)
 531                     if permission_sdk_23.maxSdkVersion is not None:
 532                         permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
 533                         apkel.appendChild(permel)
 534                 if 'nativecode' in apk:
 535                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
 536                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
 537
 538         if current_version_file is not None \
 539                 and common.config['make_current_version_link'] \
 540                 and repodir == 'repo':  # only create these
 541             namefield = common.config['current_version_name_source']
 542             sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
 543             apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8')
 544             current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
 545             if os.path.islink(apklinkname):
 546                 os.remove(apklinkname)
 547             os.symlink(current_version_path, apklinkname)
 548             # also symlink gpg signature, if it exists
 549             for extension in (b'.asc', b'.sig'):
 550                 sigfile_path = current_version_path + extension
 551                 if os.path.exists(sigfile_path):
 552                     siglinkname = apklinkname + extension
 553                     if os.path.islink(siglinkname):
 554                         os.remove(siglinkname)
 555                     os.symlink(sigfile_path, siglinkname)
 556
 557     if common.options.pretty:
 558         output = doc.toprettyxml(encoding='utf-8')
 559     else:
 560         output = doc.toxml(encoding='utf-8')
 561
 562     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 563         f.write(output)
 564
 565     if 'repo_keyalias' in common.config:
 566
 567         if common.options.nosign:
 568             logging.info(_("Creating unsigned index in preparation for signing"))
 569         else:
 570             logging.info(_("Creating signed index with this key (SHA256):"))
 571             logging.info("%s" % repo_pubkey_fingerprint)
 572
 573         # Create a jar of the index...
 574         jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar'
 575         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 576         if p.returncode != 0:
 577             raise FDroidException("Failed to create {0}".format(jar_output))
 578
 579         # Sign the index...
 580         signed = os.path.join(repodir, 'index.jar')
 581         if common.options.nosign:
 582             # Remove old signed index if not signing
 583             if os.path.exists(signed):
 584                 os.remove(signed)
 585         else:
 586             signindex.config = common.config
 587             signindex.sign_jar(signed)
 588
 589     # Copy the repo icon into the repo directory...
 590     icon_dir = os.path.join(repodir, 'icons')
 591     iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
 592     shutil.copyfile(common.config['repo_icon'], iconfilename)
 593
 594
 595 def extract_pubkey():
 596     """
 597     Extracts and returns the repository's public key from the keystore.
 598     :return: public key in hex, repository fingerprint
 599     """
 600     if 'repo_pubkey' in common.config:
 601         pubkey = unhexlify(common.config['repo_pubkey'])
 602     else:
 603         env_vars = {'FDROID_KEY_STORE_PASS': common.config['keystorepass']}
 604         p = FDroidPopenBytes([common.config['keytool'], '-exportcert',
 605                               '-alias', common.config['repo_keyalias'],
 606                               '-keystore', common.config['keystore'],
 607                               '-storepass:env', 'FDROID_KEY_STORE_PASS']
 608                              + common.config['smartcardoptions'],
 609                              envs=env_vars, output=False, stderr_to_stdout=False)
 610         if p.returncode != 0 or len(p.output) < 20:
 611             msg = "Failed to get repo pubkey!"
 612             if common.config['keystore'] == 'NONE':
 613                 msg += ' Is your crypto smartcard plugged in?'
 614             raise FDroidException(msg)
 615         pubkey = p.output
 616     repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
 617     return hexlify(pubkey), repo_pubkey_fingerprint
 618
 619
 620 def get_mirror_service_urls(url):
 621     '''Get direct URLs from git service for use by fdroidclient
 622
 623     Via 'servergitmirrors', fdroidserver can create and push a mirror
 624     to certain well known git services like gitlab or github.  This
 625     will always use the 'master' branch since that is the default
 626     branch in git. The files are then accessible via alternate URLs,
 627     where they are served in their raw format via a CDN rather than
 628     from git.
 629     '''
 630
 631     if url.startswith('git@'):
 632         url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
 633
 634     segments = url.split("/")
 635
 636     if segments[4].endswith('.git'):
 637         segments[4] = segments[4][:-4]
 638
 639     hostname = segments[2]
 640     user = segments[3]
 641     repo = segments[4]
 642     branch = "master"
 643     folder = "fdroid"
 644
 645     urls = []
 646     if hostname == "github.com":
 647         # Github-like RAW segments "https://raw.githubusercontent.com/user/repo/branch/folder"
 648         segments[2] = "raw.githubusercontent.com"
 649         segments.extend([branch, folder])
 650         urls.append('/'.join(segments))
 651     elif hostname == "gitlab.com":
 652         # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
 653         gitlab_raw = segments + ['raw', branch, folder]
 654         urls.append('/'.join(gitlab_raw))
 655         # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
 656         gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
 657         urls.append('/'.join(gitlab_pages))
 658         return urls
 659
 660     return urls
 661
 662
 663 def download_repo_index(url_str, etag=None, verify_fingerprint=True):
 664     """
 665     Downloads the repository index from the given :param url_str
 666     and verifies the repository's fingerprint if :param verify_fingerprint is not False.
 667
 668     :raises: VerificationException() if the repository could not be verified
 669
 670     :return: A tuple consisting of:
 671         - The index in JSON format or None if the index did not change
 672         - The new eTag as returned by the HTTP request
 673     """
 674     url = urllib.parse.urlsplit(url_str)
 675
 676     fingerprint = None
 677     if verify_fingerprint:
 678         query = urllib.parse.parse_qs(url.query)
 679         if 'fingerprint' not in query:
 680             raise VerificationException(_("No fingerprint in URL."))
 681         fingerprint = query['fingerprint'][0]
 682
 683     url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
 684     download, new_etag = net.http_get(url.geturl(), etag)
 685
 686     if download is None:
 687         return None, new_etag
 688
 689     with tempfile.NamedTemporaryFile() as fp:
 690         # write and open JAR file
 691         fp.write(download)
 692         jar = zipfile.ZipFile(fp)
 693
 694         # verify that the JAR signature is valid
 695         logging.debug(_('Verifying index signature:'))
 696         common.verify_jar_signature(fp.name)
 697
 698         # get public key and its fingerprint from JAR
 699         public_key, public_key_fingerprint = get_public_key_from_jar(jar)
 700
 701         # compare the fingerprint if verify_fingerprint is True
 702         if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
 703             raise VerificationException(_("The repository's fingerprint does not match."))
 704
 705         # load repository index from JSON
 706         index = json.loads(jar.read('index-v1.json').decode("utf-8"))
 707         index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
 708         index["repo"]["fingerprint"] = public_key_fingerprint
 709
 710         # turn the apps into App objects
 711         index["apps"] = [metadata.App(app) for app in index["apps"]]
 712
 713         return index, new_etag
 714
 715
 716 def get_public_key_from_jar(jar):
 717     """
 718     Get the public key and its fingerprint from a JAR file.
 719
 720     :raises: VerificationException() if the JAR was not signed exactly once
 721
 722     :param jar: a zipfile.ZipFile object
 723     :return: the public key from the jar and its fingerprint
 724     """
 725     # extract certificate from jar
 726     certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
 727     if len(certs) < 1:
 728         raise VerificationException(_("Found no signing certificates for repository."))
 729     if len(certs) > 1:
 730         raise VerificationException(_("Found multiple signing certificates for repository."))
 731
 732     # extract public key from certificate
 733     public_key = common.get_certificate(jar.read(certs[0]))
 734     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 735
 736     return public_key, public_key_fingerprint