fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import urllib.parse
  32 from datetime import datetime, timedelta
  33 from xml.dom.minidom import Document
  34 from argparse import ArgumentParser
  35 import time
  36
  37 import collections
  38 from pyasn1.error import PyAsn1Error
  39 from pyasn1.codec.der import decoder, encoder
  40 from pyasn1_modules import rfc2315
  41 from binascii import hexlify, unhexlify
  42
  43 from PIL import Image
  44 import logging
  45
  46 from . import common
  47 from . import metadata
  48 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  49 from .metadata import MetaDataException
  50
  51 METADATA_VERSION = 17
  52
  53 screen_densities = ['640', '480', '320', '240', '160', '120']
  54
  55 all_screen_densities = ['0'] + screen_densities
  56
  57 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  58 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  59
  60
  61 def dpi_to_px(density):
  62     return (int(density) * 48) / 160
  63
  64
  65 def px_to_dpi(px):
  66     return (int(px) * 160) / 48
  67
  68
  69 def get_icon_dir(repodir, density):
  70     if density == '0':
  71         return os.path.join(repodir, "icons")
  72     return os.path.join(repodir, "icons-%s" % density)
  73
  74
  75 def get_icon_dirs(repodir):
  76     for density in screen_densities:
  77         yield get_icon_dir(repodir, density)
  78
  79
  80 def get_all_icon_dirs(repodir):
  81     for density in all_screen_densities:
  82         yield get_icon_dir(repodir, density)
  83
  84
  85 def update_wiki(apps, sortedids, apks):
  86     """Update the wiki
  87
  88     :param apps: fully populated list of all applications
  89     :param apks: all apks, except...
  90     """
  91     logging.info("Updating wiki")
  92     wikicat = 'Apps'
  93     wikiredircat = 'App Redirects'
  94     import mwclient
  95     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  96                          path=config['wiki_path'])
  97     site.login(config['wiki_user'], config['wiki_password'])
  98     generated_pages = {}
  99     generated_redirects = {}
 100
 101     for appid in sortedids:
 102         app = metadata.App(apps[appid])
 103
 104         wikidata = ''
 105         if app.Disabled:
 106             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 107         if app.AntiFeatures:
 108             for af in app.AntiFeatures:
 109                 wikidata += '{{AntiFeature|' + af + '}}\n'
 110         if app.RequiresRoot:
 111             requiresroot = 'Yes'
 112         else:
 113             requiresroot = 'No'
 114         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 115             appid,
 116             app.Name,
 117             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 118             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 119             app.SourceCode,
 120             app.IssueTracker,
 121             app.WebSite,
 122             app.Changelog,
 123             app.Donate,
 124             app.FlattrID,
 125             app.Bitcoin,
 126             app.Litecoin,
 127             app.License,
 128             requiresroot,
 129             app.AuthorName,
 130             app.AuthorEmail)
 131
 132         if app.Provides:
 133             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 134
 135         wikidata += app.Summary
 136         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 137
 138         wikidata += "=Description=\n"
 139         wikidata += metadata.description_wiki(app.Description) + "\n"
 140
 141         wikidata += "=Maintainer Notes=\n"
 142         if app.MaintainerNotes:
 143             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 144         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 145
 146         # Get a list of all packages for this application...
 147         apklist = []
 148         gotcurrentver = False
 149         cantupdate = False
 150         buildfails = False
 151         for apk in apks:
 152             if apk['id'] == appid:
 153                 if str(apk['versioncode']) == app.CurrentVersionCode:
 154                     gotcurrentver = True
 155                 apklist.append(apk)
 156         # Include ones we can't build, as a special case...
 157         for build in app.builds:
 158             if build.disable:
 159                 if build.vercode == app.CurrentVersionCode:
 160                     cantupdate = True
 161                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 162                 apklist.append({'versioncode': int(build.vercode),
 163                                 'version': build.version,
 164                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 165                                 })
 166             else:
 167                 builtit = False
 168                 for apk in apklist:
 169                     if apk['versioncode'] == int(build.vercode):
 170                         builtit = True
 171                         break
 172                 if not builtit:
 173                     buildfails = True
 174                     apklist.append({'versioncode': int(build.vercode),
 175                                     'version': build.version,
 176                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 177                                     })
 178         if app.CurrentVersionCode == '0':
 179             cantupdate = True
 180         # Sort with most recent first...
 181         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 182
 183         wikidata += "=Versions=\n"
 184         if len(apklist) == 0:
 185             wikidata += "We currently have no versions of this app available."
 186         elif not gotcurrentver:
 187             wikidata += "We don't have the current version of this app."
 188         else:
 189             wikidata += "We have the current version of this app."
 190         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 191         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 192         if len(app.NoSourceSince) > 0:
 193             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 194         if len(app.CurrentVersion) > 0:
 195             wikidata += "The current (recommended) version is " + app.CurrentVersion
 196             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 197         validapks = 0
 198         for apk in apklist:
 199             wikidata += "==" + apk['version'] + "==\n"
 200
 201             if 'buildproblem' in apk:
 202                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 203             else:
 204                 validapks += 1
 205                 wikidata += "This version is built and signed by "
 206                 if 'srcname' in apk:
 207                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 208                 else:
 209                     wikidata += "the original developer.\n\n"
 210             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 211
 212         wikidata += '\n[[Category:' + wikicat + ']]\n'
 213         if len(app.NoSourceSince) > 0:
 214             wikidata += '\n[[Category:Apps missing source code]]\n'
 215         if validapks == 0 and not app.Disabled:
 216             wikidata += '\n[[Category:Apps with no packages]]\n'
 217         if cantupdate and not app.Disabled:
 218             wikidata += "\n[[Category:Apps we cannot update]]\n"
 219         if buildfails and not app.Disabled:
 220             wikidata += "\n[[Category:Apps with failing builds]]\n"
 221         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 222             wikidata += '\n[[Category:Apps to Update]]\n'
 223         if app.Disabled:
 224             wikidata += '\n[[Category:Apps that are disabled]]\n'
 225         if app.UpdateCheckMode == 'None' and not app.Disabled:
 226             wikidata += '\n[[Category:Apps with no update check]]\n'
 227         for appcat in app.Categories:
 228             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 229
 230         # We can't have underscores in the page name, even if they're in
 231         # the package ID, because MediaWiki messes with them...
 232         pagename = appid.replace('_', ' ')
 233
 234         # Drop a trailing newline, because mediawiki is going to drop it anyway
 235         # and it we don't we'll think the page has changed when it hasn't...
 236         if wikidata.endswith('\n'):
 237             wikidata = wikidata[:-1]
 238
 239         generated_pages[pagename] = wikidata
 240
 241         # Make a redirect from the name to the ID too, unless there's
 242         # already an existing page with the name and it isn't a redirect.
 243         noclobber = False
 244         apppagename = app.Name.replace('_', ' ')
 245         apppagename = apppagename.replace('{', '')
 246         apppagename = apppagename.replace('}', ' ')
 247         apppagename = apppagename.replace(':', ' ')
 248         apppagename = apppagename.replace('[', ' ')
 249         apppagename = apppagename.replace(']', ' ')
 250         # Drop double spaces caused mostly by replacing ':' above
 251         apppagename = apppagename.replace('  ', ' ')
 252         for expagename in site.allpages(prefix=apppagename,
 253                                         filterredir='nonredirects',
 254                                         generator=False):
 255             if expagename == apppagename:
 256                 noclobber = True
 257         # Another reason not to make the redirect page is if the app name
 258         # is the same as it's ID, because that will overwrite the real page
 259         # with an redirect to itself! (Although it seems like an odd
 260         # scenario this happens a lot, e.g. where there is metadata but no
 261         # builds or binaries to extract a name from.
 262         if apppagename == pagename:
 263             noclobber = True
 264         if not noclobber:
 265             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 266
 267     for tcat, genp in [(wikicat, generated_pages),
 268                        (wikiredircat, generated_redirects)]:
 269         catpages = site.Pages['Category:' + tcat]
 270         existingpages = []
 271         for page in catpages:
 272             existingpages.append(page.name)
 273             if page.name in genp:
 274                 pagetxt = page.edit()
 275                 if pagetxt != genp[page.name]:
 276                     logging.debug("Updating modified page " + page.name)
 277                     page.save(genp[page.name], summary='Auto-updated')
 278                 else:
 279                     logging.debug("Page " + page.name + " is unchanged")
 280             else:
 281                 logging.warn("Deleting page " + page.name)
 282                 page.delete('No longer published')
 283         for pagename, text in genp.items():
 284             logging.debug("Checking " + pagename)
 285             if pagename not in existingpages:
 286                 logging.debug("Creating page " + pagename)
 287                 try:
 288                     newpage = site.Pages[pagename]
 289                     newpage.save(text, summary='Auto-created')
 290                 except:
 291                     logging.error("...FAILED to create page '{0}'".format(pagename))
 292
 293     # Purge server cache to ensure counts are up to date
 294     site.pages['Repository Maintenance'].purge()
 295
 296
 297 def delete_disabled_builds(apps, apkcache, repodirs):
 298     """Delete disabled build outputs.
 299
 300     :param apps: list of all applications, as per metadata.read_metadata
 301     :param apkcache: current apk cache information
 302     :param repodirs: the repo directories to process
 303     """
 304     for appid, app in apps.items():
 305         for build in app['builds']:
 306             if not build.disable:
 307                 continue
 308             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 309             iconfilename = "%s.%s.png" % (
 310                 appid,
 311                 build.vercode)
 312             for repodir in repodirs:
 313                 files = [
 314                     os.path.join(repodir, apkfilename),
 315                     os.path.join(repodir, apkfilename + '.asc'),
 316                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 317                 ]
 318                 for density in all_screen_densities:
 319                     repo_dir = get_icon_dir(repodir, density)
 320                     files.append(os.path.join(repo_dir, iconfilename))
 321
 322                 for f in files:
 323                     if os.path.exists(f):
 324                         logging.info("Deleting disabled build output " + f)
 325                         os.remove(f)
 326             if apkfilename in apkcache:
 327                 del apkcache[apkfilename]
 328
 329
 330 def resize_icon(iconpath, density):
 331
 332     if not os.path.isfile(iconpath):
 333         return
 334
 335     fp = None
 336     try:
 337         fp = open(iconpath, 'rb')
 338         im = Image.open(fp)
 339         size = dpi_to_px(density)
 340
 341         if any(length > size for length in im.size):
 342             oldsize = im.size
 343             im.thumbnail((size, size), Image.ANTIALIAS)
 344             logging.debug("%s was too large at %s - new size is %s" % (
 345                 iconpath, oldsize, im.size))
 346             im.save(iconpath, "PNG")
 347
 348     except Exception as e:
 349         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 350
 351     finally:
 352         if fp:
 353             fp.close()
 354
 355
 356 def resize_all_icons(repodirs):
 357     """Resize all icons that exceed the max size
 358
 359     :param repodirs: the repo directories to process
 360     """
 361     for repodir in repodirs:
 362         for density in screen_densities:
 363             icon_dir = get_icon_dir(repodir, density)
 364             icon_glob = os.path.join(icon_dir, '*.png')
 365             for iconpath in glob.glob(icon_glob):
 366                 resize_icon(iconpath, density)
 367
 368
 369 # A signature block file with a .DSA, .RSA, or .EC extension
 370 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 371
 372
 373 def getsig(apkpath):
 374     """ Get the signing certificate of an apk. To get the same md5 has that
 375     Android gets, we encode the .RSA certificate in a specific format and pass
 376     it hex-encoded to the md5 digest algorithm.
 377
 378     :param apkpath: path to the apk
 379     :returns: A string containing the md5 of the signature of the apk or None
 380               if an error occurred.
 381     """
 382
 383     cert = None
 384
 385     # verify the jar signature is correct
 386     args = [config['jarsigner'], '-verify', apkpath]
 387     p = FDroidPopen(args)
 388     if p.returncode != 0:
 389         logging.critical(apkpath + " has a bad signature!")
 390         return None
 391
 392     with zipfile.ZipFile(apkpath, 'r') as apk:
 393
 394         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 395
 396         if len(certs) < 1:
 397             logging.error("Found no signing certificates on %s" % apkpath)
 398             return None
 399         if len(certs) > 1:
 400             logging.error("Found multiple signing certificates on %s" % apkpath)
 401             return None
 402
 403         cert = apk.read(certs[0])
 404
 405     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 406     if content.getComponentByName('contentType') != rfc2315.signedData:
 407         logging.error("Unexpected format.")
 408         return None
 409
 410     content = decoder.decode(content.getComponentByName('content'),
 411                              asn1Spec=rfc2315.SignedData())[0]
 412     try:
 413         certificates = content.getComponentByName('certificates')
 414     except PyAsn1Error:
 415         logging.error("Certificates not found.")
 416         return None
 417
 418     cert_encoded = encoder.encode(certificates)[4:]
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_icon_bytes(apkzip, iconsrc):
 424     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 425     try:
 426         return apkzip.read(iconsrc)
 427     except KeyError:
 428         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 429
 430
 431 def sha256sum(filename):
 432     '''Calculate the sha256 of the given file'''
 433     sha = hashlib.sha256()
 434     with open(filename, 'rb') as f:
 435         while True:
 436             t = f.read(16384)
 437             if len(t) == 0:
 438                 break
 439             sha.update(t)
 440     return sha.hexdigest()
 441
 442
 443 def has_old_openssl(filename):
 444     '''checks for known vulnerable openssl versions in the APK'''
 445
 446     # statically load this pattern
 447     if not hasattr(has_old_openssl, "pattern"):
 448         has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 449
 450     with zipfile.ZipFile(filename) as zf:
 451         for name in zf.namelist():
 452             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 453                 lib = zf.open(name)
 454                 while True:
 455                     chunk = lib.read(4096)
 456                     if chunk == b'':
 457                         break
 458                     m = has_old_openssl.pattern.search(chunk)
 459                     if m:
 460                         version = m.group(1).decode('ascii')
 461                         if version.startswith('1.0.1') and version[5] >= 'r' \
 462                            or version.startswith('1.0.2') and version[5] >= 'f':
 463                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 464                         else:
 465                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 466                             return True
 467                         break
 468     return False
 469
 470
 471 def insert_obbs(repodir, apps, apks):
 472     """Scans the .obb files in a given repo directory and adds them to the
 473     relevant APK instances.  OBB files have versionCodes like APK
 474     files, and they are loosely associated.  If there is an OBB file
 475     present, then any APK with the same or higher versionCode will use
 476     that OBB file.  There are two OBB types: main and patch, each APK
 477     can only have only have one of each.
 478
 479     https://developer.android.com/google/play/expansion-files.html
 480
 481     :param repodir: repo directory to scan
 482     :param apps: list of current, valid apps
 483     :param apks: current information on all APKs
 484
 485     """
 486
 487     def obbWarnDelete(f, msg):
 488         logging.warning(msg + f)
 489         if options.delete_unknown:
 490             logging.error("Deleting unknown file: " + f)
 491             os.remove(f)
 492
 493     obbs = []
 494     java_Integer_MIN_VALUE = -pow(2, 31)
 495     for f in glob.glob(os.path.join(repodir, '*.obb')):
 496         obbfile = os.path.basename(f)
 497         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 498         chunks = obbfile.split('.')
 499         if chunks[0] != 'main' and chunks[0] != 'patch':
 500             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 501             continue
 502         if not re.match(r'^-?[0-9]+$', chunks[1]):
 503             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 504             continue
 505         versioncode = int(chunks[1])
 506         packagename = ".".join(chunks[2:-1])
 507
 508         highestVersionCode = java_Integer_MIN_VALUE
 509         if packagename not in apps.keys():
 510             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 511             continue
 512         for apk in apks:
 513             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 514                 highestVersionCode = apk['versioncode']
 515         if versioncode > highestVersionCode:
 516             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 517                           + ') than any APK: ')
 518             continue
 519         obbsha256 = sha256sum(f)
 520         obbs.append((packagename, versioncode, obbfile, obbsha256))
 521
 522     for apk in apks:
 523         for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 524             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 525                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 526                     apk['obbMainFile'] = obbfile
 527                     apk['obbMainFileSha256'] = obbsha256
 528                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 529                     apk['obbPatchFile'] = obbfile
 530                     apk['obbPatchFileSha256'] = obbsha256
 531             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 532                 break
 533
 534
 535 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 536     """Scan a repo for all files with an extension except APK/OBB
 537
 538     :param apkcache: current cached info about all repo files
 539     :param repodir: repo directory to scan
 540     :param knownapks: list of all known files, as per metadata.read_metadata
 541     :param use_date_from_file: use date from file (instead of current date)
 542                                for newly added files
 543     """
 544
 545     cachechanged = False
 546     repo_files = []
 547     for name in os.listdir(repodir):
 548         file_extension = common.get_file_extension(name)
 549         if file_extension == 'apk' or file_extension == 'obb':
 550             continue
 551         filename = os.path.join(repodir, name)
 552         if not common.is_repo_file(name):
 553             continue
 554         stat = os.stat(filename)
 555         if stat.st_size == 0:
 556             logging.error(filename + ' is zero size!')
 557             sys.exit(1)
 558
 559         shasum = sha256sum(filename)
 560         usecache = False
 561         if name in apkcache:
 562             repo_file = apkcache[name]
 563             if repo_file['sha256'] == shasum:
 564                 logging.debug("Reading " + name + " from cache")
 565                 usecache = True
 566             else:
 567                 logging.debug("Ignoring stale cache data for " + name)
 568
 569         if not usecache:
 570             logging.debug("Processing " + name)
 571             repo_file = {}
 572             # TODO rename apkname globally to something more generic
 573             repo_file['name'] = name
 574             repo_file['apkname'] = name
 575             repo_file['sha256'] = shasum
 576             repo_file['versioncode'] = 0
 577             repo_file['version'] = shasum
 578             # the static ID is the SHA256 unless it is set in the metadata
 579             repo_file['id'] = shasum
 580             srcfilename = name + ".src.tar.gz"
 581             if os.path.exists(os.path.join(repodir, srcfilename)):
 582                 repo_file['srcname'] = srcfilename
 583             repo_file['size'] = stat.st_size
 584
 585             apkcache[name] = repo_file
 586             cachechanged = True
 587
 588         if use_date_from_file:
 589             timestamp = stat.st_ctime
 590             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 591         else:
 592             default_date_param = None
 593
 594         # Record in knownapks, getting the added date at the same time..
 595         added = knownapks.recordapk(repo_file['apkname'], repo_file['id'],
 596                                     default_date=default_date_param)
 597         if added:
 598             repo_file['added'] = added
 599
 600         repo_files.append(repo_file)
 601
 602     return repo_files, cachechanged
 603
 604
 605 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
 606     """Scan the apks in the given repo directory.
 607
 608     This also extracts the icons.
 609
 610     :param apkcache: current apk cache information
 611     :param repodir: repo directory to scan
 612     :param knownapks: known apks info
 613     :param use_date_from_apk: use date from APK (instead of current date)
 614                               for newly added APKs
 615     :returns: (apks, cachechanged) where apks is a list of apk information,
 616               and cachechanged is True if the apkcache got changed.
 617     """
 618
 619     cachechanged = False
 620
 621     for icon_dir in get_all_icon_dirs(repodir):
 622         if os.path.exists(icon_dir):
 623             if options.clean:
 624                 shutil.rmtree(icon_dir)
 625                 os.makedirs(icon_dir)
 626         else:
 627             os.makedirs(icon_dir)
 628
 629     apks = []
 630     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 631     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 632     vername_pat = re.compile(".*versionName='([^']*)'.*")
 633     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 634     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 635     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 636     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 637     permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 638     feature_pat = re.compile(".*name='([^']*)'.*")
 639     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 640
 641         apkfilename = apkfile[len(repodir) + 1:]
 642         if ' ' in apkfilename:
 643             logging.critical("Spaces in filenames are not allowed.")
 644             sys.exit(1)
 645
 646         shasum = sha256sum(apkfile)
 647
 648         usecache = False
 649         if apkfilename in apkcache:
 650             apk = apkcache[apkfilename]
 651             if apk['sha256'] == shasum:
 652                 logging.debug("Reading " + apkfilename + " from cache")
 653                 usecache = True
 654             else:
 655                 logging.debug("Ignoring stale cache data for " + apkfilename)
 656
 657         if not usecache:
 658             logging.debug("Processing " + apkfilename)
 659             apk = {}
 660             apk['apkname'] = apkfilename
 661             apk['sha256'] = shasum
 662             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 663             if os.path.exists(os.path.join(repodir, srcfilename)):
 664                 apk['srcname'] = srcfilename
 665             apk['size'] = os.path.getsize(apkfile)
 666             apk['uses-permission'] = set()
 667             apk['uses-permission-sdk-23'] = set()
 668             apk['features'] = set()
 669             apk['icons_src'] = {}
 670             apk['icons'] = {}
 671             apk['antiFeatures'] = set()
 672             if has_old_openssl(apkfile):
 673                 apk['antiFeatures'].add('KnownVuln')
 674             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 675             if p.returncode != 0:
 676                 if options.delete_unknown:
 677                     if os.path.exists(apkfile):
 678                         logging.error("Failed to get apk information, deleting " + apkfile)
 679                         os.remove(apkfile)
 680                     else:
 681                         logging.error("Could not find {0} to remove it".format(apkfile))
 682                 else:
 683                     logging.error("Failed to get apk information, skipping " + apkfile)
 684                 continue
 685             for line in p.output.splitlines():
 686                 if line.startswith("package:"):
 687                     try:
 688                         apk['id'] = re.match(name_pat, line).group(1)
 689                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 690                         apk['version'] = re.match(vername_pat, line).group(1)
 691                     except Exception as e:
 692                         logging.error("Package matching failed: " + str(e))
 693                         logging.info("Line was: " + line)
 694                         sys.exit(1)
 695                 elif line.startswith("application:"):
 696                     apk['name'] = re.match(label_pat, line).group(1)
 697                     # Keep path to non-dpi icon in case we need it
 698                     match = re.match(icon_pat_nodpi, line)
 699                     if match:
 700                         apk['icons_src']['-1'] = match.group(1)
 701                 elif line.startswith("launchable-activity:"):
 702                     # Only use launchable-activity as fallback to application
 703                     if not apk['name']:
 704                         apk['name'] = re.match(label_pat, line).group(1)
 705                     if '-1' not in apk['icons_src']:
 706                         match = re.match(icon_pat_nodpi, line)
 707                         if match:
 708                             apk['icons_src']['-1'] = match.group(1)
 709                 elif line.startswith("application-icon-"):
 710                     match = re.match(icon_pat, line)
 711                     if match:
 712                         density = match.group(1)
 713                         path = match.group(2)
 714                         apk['icons_src'][density] = path
 715                 elif line.startswith("sdkVersion:"):
 716                     m = re.match(sdkversion_pat, line)
 717                     if m is None:
 718                         logging.error(line.replace('sdkVersion:', '')
 719                                       + ' is not a valid minSdkVersion!')
 720                     else:
 721                         apk['minSdkVersion'] = m.group(1)
 722                         # if target not set, default to min
 723                         if 'targetSdkVersion' not in apk:
 724                             apk['targetSdkVersion'] = m.group(1)
 725                 elif line.startswith("targetSdkVersion:"):
 726                     m = re.match(sdkversion_pat, line)
 727                     if m is None:
 728                         logging.error(line.replace('targetSdkVersion:', '')
 729                                       + ' is not a valid targetSdkVersion!')
 730                     else:
 731                         apk['targetSdkVersion'] = m.group(1)
 732                 elif line.startswith("maxSdkVersion:"):
 733                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 734                 elif line.startswith("native-code:"):
 735                     apk['nativecode'] = []
 736                     for arch in line[13:].split(' '):
 737                         apk['nativecode'].append(arch[1:-1])
 738                 elif line.startswith('uses-permission:'):
 739                     perm_match = re.match(permission_pat, line).groupdict()
 740
 741                     permission = UsesPermission(
 742                         perm_match['name'],
 743                         perm_match['maxSdkVersion']
 744                     )
 745
 746                     apk['uses-permission'].add(permission)
 747                 elif line.startswith('uses-permission-sdk-23:'):
 748                     perm_match = re.match(permission_pat, line).groupdict()
 749
 750                     permission_sdk_23 = UsesPermissionSdk23(
 751                         perm_match['name'],
 752                         perm_match['maxSdkVersion']
 753                     )
 754
 755                     apk['uses-permission-sdk-23'].add(permission_sdk_23)
 756
 757                 elif line.startswith('uses-feature:'):
 758                     feature = re.match(feature_pat, line).group(1)
 759                     # Filter out this, it's only added with the latest SDK tools and
 760                     # causes problems for lots of apps.
 761                     if feature != "android.hardware.screen.portrait" \
 762                             and feature != "android.hardware.screen.landscape":
 763                         if feature.startswith("android.feature."):
 764                             feature = feature[16:]
 765                         apk['features'].add(feature)
 766
 767             if 'minSdkVersion' not in apk:
 768                 logging.warn("No SDK version information found in {0}".format(apkfile))
 769                 apk['minSdkVersion'] = 1
 770
 771             # Check for debuggable apks...
 772             if common.isApkAndDebuggable(apkfile, config):
 773                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 774
 775             # Get the signature (or md5 of, to be precise)...
 776             logging.debug('Getting signature of {0}'.format(apkfile))
 777             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 778             if not apk['sig']:
 779                 logging.critical("Failed to get apk signature")
 780                 sys.exit(1)
 781
 782             apkzip = zipfile.ZipFile(apkfile, 'r')
 783
 784             # if an APK has files newer than the system time, suggest updating
 785             # the system clock.  This is useful for offline systems, used for
 786             # signing, which do not have another source of clock sync info. It
 787             # has to be more than 24 hours newer because ZIP/APK files do not
 788             # store timezone info
 789             manifest = apkzip.getinfo('AndroidManifest.xml')
 790             if manifest.date_time[1] == 0:  # month can't be zero
 791                 logging.debug('AndroidManifest.xml has no date')
 792             else:
 793                 dt_obj = datetime(*manifest.date_time)
 794                 checkdt = dt_obj - timedelta(1)
 795                 if datetime.today() < checkdt:
 796                     logging.warn('System clock is older than manifest in: '
 797                                  + apkfilename
 798                                  + '\nSet clock to that time using:\n'
 799                                  + 'sudo date -s "' + str(dt_obj) + '"')
 800
 801             iconfilename = "%s.%s.png" % (
 802                 apk['id'],
 803                 apk['versioncode'])
 804
 805             # Extract the icon file...
 806             empty_densities = []
 807             for density in screen_densities:
 808                 if density not in apk['icons_src']:
 809                     empty_densities.append(density)
 810                     continue
 811                 iconsrc = apk['icons_src'][density]
 812                 icon_dir = get_icon_dir(repodir, density)
 813                 icondest = os.path.join(icon_dir, iconfilename)
 814
 815                 try:
 816                     with open(icondest, 'wb') as f:
 817                         f.write(get_icon_bytes(apkzip, iconsrc))
 818                     apk['icons'][density] = iconfilename
 819
 820                 except:
 821                     logging.warn("Error retrieving icon file")
 822                     del apk['icons'][density]
 823                     del apk['icons_src'][density]
 824                     empty_densities.append(density)
 825
 826             if '-1' in apk['icons_src']:
 827                 iconsrc = apk['icons_src']['-1']
 828                 iconpath = os.path.join(
 829                     get_icon_dir(repodir, '0'), iconfilename)
 830                 with open(iconpath, 'wb') as f:
 831                     f.write(get_icon_bytes(apkzip, iconsrc))
 832                 try:
 833                     im = Image.open(iconpath)
 834                     dpi = px_to_dpi(im.size[0])
 835                     for density in screen_densities:
 836                         if density in apk['icons']:
 837                             break
 838                         if density == screen_densities[-1] or dpi >= int(density):
 839                             apk['icons'][density] = iconfilename
 840                             shutil.move(iconpath,
 841                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 842                             empty_densities.remove(density)
 843                             break
 844                 except Exception as e:
 845                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 846
 847             if apk['icons']:
 848                 apk['icon'] = iconfilename
 849
 850             apkzip.close()
 851
 852             # First try resizing down to not lose quality
 853             last_density = None
 854             for density in screen_densities:
 855                 if density not in empty_densities:
 856                     last_density = density
 857                     continue
 858                 if last_density is None:
 859                     continue
 860                 logging.debug("Density %s not available, resizing down from %s"
 861                               % (density, last_density))
 862
 863                 last_iconpath = os.path.join(
 864                     get_icon_dir(repodir, last_density), iconfilename)
 865                 iconpath = os.path.join(
 866                     get_icon_dir(repodir, density), iconfilename)
 867                 fp = None
 868                 try:
 869                     fp = open(last_iconpath, 'rb')
 870                     im = Image.open(fp)
 871
 872                     size = dpi_to_px(density)
 873
 874                     im.thumbnail((size, size), Image.ANTIALIAS)
 875                     im.save(iconpath, "PNG")
 876                     empty_densities.remove(density)
 877                 except:
 878                     logging.warning("Invalid image file at %s" % last_iconpath)
 879                 finally:
 880                     if fp:
 881                         fp.close()
 882
 883             # Then just copy from the highest resolution available
 884             last_density = None
 885             for density in reversed(screen_densities):
 886                 if density not in empty_densities:
 887                     last_density = density
 888                     continue
 889                 if last_density is None:
 890                     continue
 891                 logging.debug("Density %s not available, copying from lower density %s"
 892                               % (density, last_density))
 893
 894                 shutil.copyfile(
 895                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 896                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 897
 898                 empty_densities.remove(density)
 899
 900             for density in screen_densities:
 901                 icon_dir = get_icon_dir(repodir, density)
 902                 icondest = os.path.join(icon_dir, iconfilename)
 903                 resize_icon(icondest, density)
 904
 905             # Copy from icons-mdpi to icons since mdpi is the baseline density
 906             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 907             if os.path.isfile(baseline):
 908                 apk['icons']['0'] = iconfilename
 909                 shutil.copyfile(baseline,
 910                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 911
 912             if use_date_from_apk and manifest.date_time[1] != 0:
 913                 default_date_param = datetime(*manifest.date_time).utctimetuple()
 914             else:
 915                 default_date_param = None
 916
 917             # Record in known apks, getting the added date at the same time..
 918             added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
 919             if added:
 920                 apk['added'] = added
 921
 922             apkcache[apkfilename] = apk
 923             cachechanged = True
 924
 925         apks.append(apk)
 926
 927     return apks, cachechanged
 928
 929
 930 repo_pubkey_fingerprint = None
 931
 932
 933 # Generate a certificate fingerprint the same way keytool does it
 934 # (but with slightly different formatting)
 935 def cert_fingerprint(data):
 936     digest = hashlib.sha256(data).digest()
 937     ret = []
 938     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 939     return " ".join(ret)
 940
 941
 942 def extract_pubkey():
 943     global repo_pubkey_fingerprint
 944     if 'repo_pubkey' in config:
 945         pubkey = unhexlify(config['repo_pubkey'])
 946     else:
 947         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 948                               '-alias', config['repo_keyalias'],
 949                               '-keystore', config['keystore'],
 950                               '-storepass:file', config['keystorepassfile']]
 951                              + config['smartcardoptions'],
 952                              output=False, stderr_to_stdout=False)
 953         if p.returncode != 0 or len(p.output) < 20:
 954             msg = "Failed to get repo pubkey!"
 955             if config['keystore'] == 'NONE':
 956                 msg += ' Is your crypto smartcard plugged in?'
 957             logging.critical(msg)
 958             sys.exit(1)
 959         pubkey = p.output
 960     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 961     return hexlify(pubkey)
 962
 963
 964 # Get raw URL from git service for mirroring
 965 def get_raw_mirror(url):
 966     # Divide urls in parts
 967     url = url.split("/")
 968
 969     # Get the hostname
 970     hostname = url[2]
 971
 972     # fdroidserver will use always 'master' branch for git-mirroring
 973     branch = "master"
 974     folder = "fdroid"
 975
 976     if hostname == "github.com":
 977         # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid"
 978         url[2] = "raw.githubusercontent.com"
 979         url.extend([branch, folder])
 980     elif hostname == "gitlab.com":
 981         # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid"
 982         url.extend(["raw", branch, folder])
 983     else:
 984         return None
 985
 986     url = "/".join(url)
 987     return url
 988
 989
 990 def make_index(apps, sortedids, apks, repodir, archive):
 991     """Generate the repo index files.
 992
 993     :param apps: fully populated apps list
 994     :param apks: full populated apks list
 995     :param repodir: the repo directory
 996     :param archive: True if this is the archive repo, False if it's the
 997                     main one.
 998     :param categories: list of categories
 999     """
1000
1001     doc = Document()
1002
1003     def addElement(name, value, doc, parent):
1004         el = doc.createElement(name)
1005         el.appendChild(doc.createTextNode(value))
1006         parent.appendChild(el)
1007
1008     def addElementNonEmpty(name, value, doc, parent):
1009         if not value:
1010             return
1011         addElement(name, value, doc, parent)
1012
1013     def addElementIfInApk(name, apk, key, doc, parent):
1014         if key not in apk:
1015             return
1016         value = str(apk[key])
1017         addElement(name, value, doc, parent)
1018
1019     def addElementCDATA(name, value, doc, parent):
1020         el = doc.createElement(name)
1021         el.appendChild(doc.createCDATASection(value))
1022         parent.appendChild(el)
1023
1024     root = doc.createElement("fdroid")
1025     doc.appendChild(root)
1026
1027     repoel = doc.createElement("repo")
1028
1029     mirrorcheckfailed = False
1030     mirrors = []
1031     for mirror in sorted(config.get('mirrors', [])):
1032         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
1033         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
1034             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
1035             mirrorcheckfailed = True
1036         # must end with / or urljoin strips a whole path segment
1037         if mirror.endswith('/'):
1038             mirrors.append(mirror)
1039         else:
1040             mirrors.append(mirror + '/')
1041     for mirror in config.get('servergitmirrors', []):
1042         mirror = get_raw_mirror(mirror)
1043         if mirror is not None:
1044             mirrors.append(mirror + '/')
1045     if mirrorcheckfailed:
1046         sys.exit(1)
1047
1048     if archive:
1049         repoel.setAttribute("name", config['archive_name'])
1050         if config['repo_maxage'] != 0:
1051             repoel.setAttribute("maxage", str(config['repo_maxage']))
1052         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
1053         repoel.setAttribute("url", config['archive_url'])
1054         addElement('description', config['archive_description'], doc, repoel)
1055         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
1056         for mirror in mirrors:
1057             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
1058
1059     else:
1060         repoel.setAttribute("name", config['repo_name'])
1061         if config['repo_maxage'] != 0:
1062             repoel.setAttribute("maxage", str(config['repo_maxage']))
1063         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
1064         repoel.setAttribute("url", config['repo_url'])
1065         addElement('description', config['repo_description'], doc, repoel)
1066         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
1067         for mirror in mirrors:
1068             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
1069
1070     repoel.setAttribute("version", str(METADATA_VERSION))
1071     repoel.setAttribute("timestamp", str(int(time.time())))
1072
1073     nosigningkey = False
1074     if not options.nosign:
1075         if 'repo_keyalias' not in config:
1076             nosigningkey = True
1077             logging.critical("'repo_keyalias' not found in config.py!")
1078         if 'keystore' not in config:
1079             nosigningkey = True
1080             logging.critical("'keystore' not found in config.py!")
1081         if 'keystorepass' not in config and 'keystorepassfile' not in config:
1082             nosigningkey = True
1083             logging.critical("'keystorepass' not found in config.py!")
1084         if 'keypass' not in config and 'keypassfile' not in config:
1085             nosigningkey = True
1086             logging.critical("'keypass' not found in config.py!")
1087         if not os.path.exists(config['keystore']):
1088             nosigningkey = True
1089             logging.critical("'" + config['keystore'] + "' does not exist!")
1090         if nosigningkey:
1091             logging.warning("`fdroid update` requires a signing key, you can create one using:")
1092             logging.warning("\tfdroid update --create-key")
1093             sys.exit(1)
1094
1095     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
1096     root.appendChild(repoel)
1097
1098     for command in ('install', 'uninstall'):
1099         packageNames = []
1100         key = command + '_list'
1101         if key in config:
1102             if isinstance(config[key], str):
1103                 packageNames = [config[key]]
1104             elif all(isinstance(item, str) for item in config[key]):
1105                 packageNames = config[key]
1106             else:
1107                 raise TypeError('only accepts strings, lists, and tuples')
1108         for packageName in packageNames:
1109             element = doc.createElement(command)
1110             root.appendChild(element)
1111             element.setAttribute('packageName', packageName)
1112
1113     for appid in sortedids:
1114         app = metadata.App(apps[appid])
1115
1116         if app.Disabled is not None:
1117             continue
1118
1119         # Get a list of the apks for this app...
1120         apklist = []
1121         for apk in apks:
1122             if apk['id'] == appid:
1123                 apklist.append(apk)
1124
1125         if len(apklist) == 0:
1126             continue
1127
1128         apel = doc.createElement("application")
1129         apel.setAttribute("id", app.id)
1130         root.appendChild(apel)
1131
1132         addElement('id', app.id, doc, apel)
1133         if app.added:
1134             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
1135         if app.lastupdated:
1136             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
1137         addElement('name', app.Name, doc, apel)
1138         addElement('summary', app.Summary, doc, apel)
1139         if app.icon:
1140             addElement('icon', app.icon, doc, apel)
1141
1142         def linkres(appid):
1143             if appid in apps:
1144                 return ("fdroid.app:" + appid, apps[appid].Name)
1145             raise MetaDataException("Cannot resolve app id " + appid)
1146
1147         addElement('desc',
1148                    metadata.description_html(app.Description, linkres),
1149                    doc, apel)
1150         addElement('license', app.License, doc, apel)
1151         if app.Categories:
1152             addElement('categories', ','.join(app.Categories), doc, apel)
1153             # We put the first (primary) category in LAST, which will have
1154             # the desired effect of making clients that only understand one
1155             # category see that one.
1156             addElement('category', app.Categories[0], doc, apel)
1157         addElement('web', app.WebSite, doc, apel)
1158         addElement('source', app.SourceCode, doc, apel)
1159         addElement('tracker', app.IssueTracker, doc, apel)
1160         addElementNonEmpty('changelog', app.Changelog, doc, apel)
1161         addElementNonEmpty('author', app.AuthorName, doc, apel)
1162         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
1163         addElementNonEmpty('donate', app.Donate, doc, apel)
1164         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
1165         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
1166         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
1167
1168         # These elements actually refer to the current version (i.e. which
1169         # one is recommended. They are historically mis-named, and need
1170         # changing, but stay like this for now to support existing clients.
1171         addElement('marketversion', app.CurrentVersion, doc, apel)
1172         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
1173
1174         if app.Provides:
1175             pv = app.Provides.split(',')
1176             addElementNonEmpty('provides', ','.join(pv), doc, apel)
1177         if app.RequiresRoot:
1178             addElement('requirements', 'root', doc, apel)
1179
1180         # Sort the apk list into version order, just so the web site
1181         # doesn't have to do any work by default...
1182         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1183
1184         if 'antiFeatures' in apklist[0]:
1185             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
1186         if app.AntiFeatures:
1187             addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel)
1188
1189         # Check for duplicates - they will make the client unhappy...
1190         for i in range(len(apklist) - 1):
1191             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1192                 logging.critical("duplicate versions: '%s' - '%s'" % (
1193                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
1194                 sys.exit(1)
1195
1196         current_version_code = 0
1197         current_version_file = None
1198         for apk in apklist:
1199             file_extension = common.get_file_extension(apk['apkname'])
1200             # find the APK for the "Current Version"
1201             if current_version_code < apk['versioncode']:
1202                 current_version_code = apk['versioncode']
1203             if current_version_code < int(app.CurrentVersionCode):
1204                 current_version_file = apk['apkname']
1205
1206             apkel = doc.createElement("package")
1207             apel.appendChild(apkel)
1208             addElement('version', apk['version'], doc, apkel)
1209             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1210             addElement('apkname', apk['apkname'], doc, apkel)
1211             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
1212             for hash_type in ['sha256']:
1213                 if hash_type not in apk:
1214                     continue
1215                 hashel = doc.createElement("hash")
1216                 hashel.setAttribute("type", hash_type)
1217                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1218                 apkel.appendChild(hashel)
1219             addElement('size', str(apk['size']), doc, apkel)
1220             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1221             addElementIfInApk('targetSdkVersion', apk,
1222                               'targetSdkVersion', doc, apkel)
1223             addElementIfInApk('maxsdkver', apk,
1224                               'maxSdkVersion', doc, apkel)
1225             addElementIfInApk('obbMainFile', apk,
1226                               'obbMainFile', doc, apkel)
1227             addElementIfInApk('obbMainFileSha256', apk,
1228                               'obbMainFileSha256', doc, apkel)
1229             addElementIfInApk('obbPatchFile', apk,
1230                               'obbPatchFile', doc, apkel)
1231             addElementIfInApk('obbPatchFileSha256', apk,
1232                               'obbPatchFileSha256', doc, apkel)
1233             if 'added' in apk:
1234                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1235
1236             if file_extension == 'apk':  # sig is required for APKs, but only APKs
1237                 addElement('sig', apk['sig'], doc, apkel)
1238
1239                 old_permissions = set()
1240                 sorted_permissions = sorted(apk['uses-permission'])
1241                 for perm in sorted_permissions:
1242                     perm_name = perm.name
1243                     if perm_name.startswith("android.permission."):
1244                         perm_name = perm_name[19:]
1245                     old_permissions.add(perm_name)
1246                 addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
1247
1248                 for permission in sorted_permissions:
1249                     permel = doc.createElement('uses-permission')
1250                     permel.setAttribute('name', permission.name)
1251                     if permission.maxSdkVersion is not None:
1252                         permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
1253                         apkel.appendChild(permel)
1254                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
1255                     permel = doc.createElement('uses-permission-sdk-23')
1256                     permel.setAttribute('name', permission_sdk_23.name)
1257                     if permission_sdk_23.maxSdkVersion is not None:
1258                         permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
1259                         apkel.appendChild(permel)
1260                 if 'nativecode' in apk:
1261                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
1262                 addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel)
1263
1264         if current_version_file is not None \
1265                 and config['make_current_version_link'] \
1266                 and repodir == 'repo':  # only create these
1267             namefield = config['current_version_name_source']
1268             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
1269             apklinkname = sanitized_name + '.apk'
1270             current_version_path = os.path.join(repodir, current_version_file)
1271             if os.path.islink(apklinkname):
1272                 os.remove(apklinkname)
1273             os.symlink(current_version_path, apklinkname)
1274             # also symlink gpg signature, if it exists
1275             for extension in ('.asc', '.sig'):
1276                 sigfile_path = current_version_path + extension
1277                 if os.path.exists(sigfile_path):
1278                     siglinkname = apklinkname + extension
1279                     if os.path.islink(siglinkname):
1280                         os.remove(siglinkname)
1281                     os.symlink(sigfile_path, siglinkname)
1282
1283     if options.pretty:
1284         output = doc.toprettyxml(encoding='utf-8')
1285     else:
1286         output = doc.toxml(encoding='utf-8')
1287
1288     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1289         f.write(output)
1290
1291     if 'repo_keyalias' in config:
1292
1293         if options.nosign:
1294             logging.info("Creating unsigned index in preparation for signing")
1295         else:
1296             logging.info("Creating signed index with this key (SHA256):")
1297             logging.info("%s" % repo_pubkey_fingerprint)
1298
1299         # Create a jar of the index...
1300         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1301         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1302         if p.returncode != 0:
1303             logging.critical("Failed to create {0}".format(jar_output))
1304             sys.exit(1)
1305
1306         # Sign the index...
1307         signed = os.path.join(repodir, 'index.jar')
1308         if options.nosign:
1309             # Remove old signed index if not signing
1310             if os.path.exists(signed):
1311                 os.remove(signed)
1312         else:
1313             args = [config['jarsigner'], '-keystore', config['keystore'],
1314                     '-storepass:file', config['keystorepassfile'],
1315                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1316                     signed, config['repo_keyalias']]
1317             if config['keystore'] == 'NONE':
1318                 args += config['smartcardoptions']
1319             else:  # smardcards never use -keypass
1320                 args += ['-keypass:file', config['keypassfile']]
1321             p = FDroidPopen(args)
1322             if p.returncode != 0:
1323                 logging.critical("Failed to sign index")
1324                 sys.exit(1)
1325
1326     # Copy the repo icon into the repo directory...
1327     icon_dir = os.path.join(repodir, 'icons')
1328     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1329     shutil.copyfile(config['repo_icon'], iconfilename)
1330
1331
1332 def make_categories_txt(repodir, categories):
1333     '''Write a category list in the repo to allow quick access'''
1334     catdata = ''
1335     for cat in sorted(categories):
1336         catdata += cat + '\n'
1337     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1338         f.write(catdata)
1339
1340
1341 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1342
1343     for appid, app in apps.items():
1344
1345         if app.ArchivePolicy:
1346             keepversions = int(app.ArchivePolicy[:-9])
1347         else:
1348             keepversions = defaultkeepversions
1349
1350         def filter_apk_list_sorted(apk_list):
1351             res = []
1352             for apk in apk_list:
1353                 if apk['id'] == appid:
1354                     res.append(apk)
1355
1356             # Sort the apk list by version code. First is highest/newest.
1357             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1358
1359         def move_file(from_dir, to_dir, filename, ignore_missing):
1360             from_path = os.path.join(from_dir, filename)
1361             if ignore_missing and not os.path.exists(from_path):
1362                 return
1363             to_path = os.path.join(to_dir, filename)
1364             shutil.move(from_path, to_path)
1365
1366         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1367                       .format(appid, len(apks), keepversions, len(archapks)))
1368
1369         if len(apks) > keepversions:
1370             apklist = filter_apk_list_sorted(apks)
1371             # Move back the ones we don't want.
1372             for apk in apklist[keepversions:]:
1373                 logging.info("Moving " + apk['apkname'] + " to archive")
1374                 move_file(repodir, archivedir, apk['apkname'], False)
1375                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1376                 for density in all_screen_densities:
1377                     repo_icon_dir = get_icon_dir(repodir, density)
1378                     archive_icon_dir = get_icon_dir(archivedir, density)
1379                     if density not in apk['icons']:
1380                         continue
1381                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1382                 if 'srcname' in apk:
1383                     move_file(repodir, archivedir, apk['srcname'], False)
1384                 archapks.append(apk)
1385                 apks.remove(apk)
1386         elif len(apks) < keepversions and len(archapks) > 0:
1387             required = keepversions - len(apks)
1388             archapklist = filter_apk_list_sorted(archapks)
1389             # Move forward the ones we want again.
1390             for apk in archapklist[:required]:
1391                 logging.info("Moving " + apk['apkname'] + " from archive")
1392                 move_file(archivedir, repodir, apk['apkname'], False)
1393                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1394                 for density in all_screen_densities:
1395                     repo_icon_dir = get_icon_dir(repodir, density)
1396                     archive_icon_dir = get_icon_dir(archivedir, density)
1397                     if density not in apk['icons']:
1398                         continue
1399                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1400                 if 'srcname' in apk:
1401                     move_file(archivedir, repodir, apk['srcname'], False)
1402                 archapks.remove(apk)
1403                 apks.append(apk)
1404
1405
1406 def add_apks_to_per_app_repos(repodir, apks):
1407     apks_per_app = dict()
1408     for apk in apks:
1409         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1410         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1411         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1412         apks_per_app[apk['id']] = apk
1413
1414         if not os.path.exists(apk['per_app_icons']):
1415             logging.info('Adding new repo for only ' + apk['id'])
1416             os.makedirs(apk['per_app_icons'])
1417
1418         apkpath = os.path.join(repodir, apk['apkname'])
1419         shutil.copy(apkpath, apk['per_app_repo'])
1420         apksigpath = apkpath + '.sig'
1421         if os.path.exists(apksigpath):
1422             shutil.copy(apksigpath, apk['per_app_repo'])
1423         apkascpath = apkpath + '.asc'
1424         if os.path.exists(apkascpath):
1425             shutil.copy(apkascpath, apk['per_app_repo'])
1426
1427
1428 config = None
1429 options = None
1430
1431
1432 def main():
1433
1434     global config, options
1435
1436     # Parse command line...
1437     parser = ArgumentParser()
1438     common.setup_global_opts(parser)
1439     parser.add_argument("--create-key", action="store_true", default=False,
1440                         help="Create a repo signing key in a keystore")
1441     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1442                         help="Create skeleton metadata files that are missing")
1443     parser.add_argument("--delete-unknown", action="store_true", default=False,
1444                         help="Delete APKs and/or OBBs without metadata from the repo")
1445     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1446                         help="Report on build data status")
1447     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1448                         help="Interactively ask about things that need updating.")
1449     parser.add_argument("-I", "--icons", action="store_true", default=False,
1450                         help="Resize all the icons exceeding the max pixel size and exit")
1451     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1452                         help="Specify editor to use in interactive mode. Default " +
1453                         "is /etc/alternatives/editor")
1454     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1455                         help="Update the wiki")
1456     parser.add_argument("--pretty", action="store_true", default=False,
1457                         help="Produce human-readable index.xml")
1458     parser.add_argument("--clean", action="store_true", default=False,
1459                         help="Clean update - don't uses caches, reprocess all apks")
1460     parser.add_argument("--nosign", action="store_true", default=False,
1461                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1462     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1463                         help="Use date from apk instead of current time for newly added apks")
1464     metadata.add_metadata_arguments(parser)
1465     options = parser.parse_args()
1466     metadata.warnings_action = options.W
1467
1468     config = common.read_config(options)
1469
1470     if not ('jarsigner' in config and 'keytool' in config):
1471         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1472         sys.exit(1)
1473
1474     repodirs = ['repo']
1475     if config['archive_older'] != 0:
1476         repodirs.append('archive')
1477         if not os.path.exists('archive'):
1478             os.mkdir('archive')
1479
1480     if options.icons:
1481         resize_all_icons(repodirs)
1482         sys.exit(0)
1483
1484     # check that icons exist now, rather than fail at the end of `fdroid update`
1485     for k in ['repo_icon', 'archive_icon']:
1486         if k in config:
1487             if not os.path.exists(config[k]):
1488                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1489                 sys.exit(1)
1490
1491     # if the user asks to create a keystore, do it now, reusing whatever it can
1492     if options.create_key:
1493         if os.path.exists(config['keystore']):
1494             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1495             logging.critical("\t'" + config['keystore'] + "'")
1496             sys.exit(1)
1497
1498         if 'repo_keyalias' not in config:
1499             config['repo_keyalias'] = socket.getfqdn()
1500             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1501         if 'keydname' not in config:
1502             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1503             common.write_to_config(config, 'keydname', config['keydname'])
1504         if 'keystore' not in config:
1505             config['keystore'] = common.default_config.keystore
1506             common.write_to_config(config, 'keystore', config['keystore'])
1507
1508         password = common.genpassword()
1509         if 'keystorepass' not in config:
1510             config['keystorepass'] = password
1511             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1512         if 'keypass' not in config:
1513             config['keypass'] = password
1514             common.write_to_config(config, 'keypass', config['keypass'])
1515         common.genkeystore(config)
1516
1517     # Get all apps...
1518     apps = metadata.read_metadata()
1519
1520     # Generate a list of categories...
1521     categories = set()
1522     for app in apps.values():
1523         categories.update(app.Categories)
1524
1525     # Read known apks data (will be updated and written back when we've finished)
1526     knownapks = common.KnownApks()
1527
1528     # Gather information about all the apk files in the repo directory, using
1529     # cached data if possible.
1530     apkcachefile = os.path.join('tmp', 'apkcache')
1531     if not options.clean and os.path.exists(apkcachefile):
1532         with open(apkcachefile, 'rb') as cf:
1533             apkcache = pickle.load(cf, encoding='utf-8')
1534         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1535             apkcache = {}
1536     else:
1537         apkcache = {}
1538
1539     delete_disabled_builds(apps, apkcache, repodirs)
1540
1541     # Scan all apks in the main repo
1542     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1543
1544     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1545                                            options.use_date_from_apk)
1546     cachechanged = cachechanged or fcachechanged
1547     apks += files
1548     # Generate warnings for apk's with no metadata (or create skeleton
1549     # metadata files, if requested on the command line)
1550     newmetadata = False
1551     for apk in apks:
1552         if apk['id'] not in apps:
1553             if options.create_metadata:
1554                 if 'name' not in apk:
1555                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1556                     continue
1557                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1558                 f.write("License:Unknown\n")
1559                 f.write("Web Site:\n")
1560                 f.write("Source Code:\n")
1561                 f.write("Issue Tracker:\n")
1562                 f.write("Changelog:\n")
1563                 f.write("Summary:" + apk['name'] + "\n")
1564                 f.write("Description:\n")
1565                 f.write(apk['name'] + "\n")
1566                 f.write(".\n")
1567                 f.write("Name:" + apk['name'] + "\n")
1568                 f.close()
1569                 logging.info("Generated skeleton metadata for " + apk['id'])
1570                 newmetadata = True
1571             else:
1572                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1573                 if options.delete_unknown:
1574                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1575                     rmf = os.path.join(repodirs[0], apk['apkname'])
1576                     if not os.path.exists(rmf):
1577                         logging.error("Could not find {0} to remove it".format(rmf))
1578                     else:
1579                         os.remove(rmf)
1580                 else:
1581                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1582
1583     # update the metadata with the newly created ones included
1584     if newmetadata:
1585         apps = metadata.read_metadata()
1586
1587     insert_obbs(repodirs[0], apps, apks)
1588
1589     # Scan the archive repo for apks as well
1590     if len(repodirs) > 1:
1591         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1592         if cc:
1593             cachechanged = True
1594     else:
1595         archapks = []
1596
1597     # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
1598     UNSET_VERSION_CODE = -0x100000000
1599
1600     # Some information from the apks needs to be applied up to the application
1601     # level. When doing this, we use the info from the most recent version's apk.
1602     # We deal with figuring out when the app was added and last updated at the
1603     # same time.
1604     for appid, app in apps.items():
1605         bestver = UNSET_VERSION_CODE
1606         for apk in apks + archapks:
1607             if apk['id'] == appid:
1608                 if apk['versioncode'] > bestver:
1609                     bestver = apk['versioncode']
1610                     bestapk = apk
1611
1612                 if 'added' in apk:
1613                     if not app.added or apk['added'] < app.added:
1614                         app.added = apk['added']
1615                     if not app.lastupdated or apk['added'] > app.lastupdated:
1616                         app.lastupdated = apk['added']
1617
1618         if not app.added:
1619             logging.debug("Don't know when " + appid + " was added")
1620         if not app.lastupdated:
1621             logging.debug("Don't know when " + appid + " was last updated")
1622
1623         if bestver == UNSET_VERSION_CODE:
1624
1625             if app.Name is None:
1626                 app.Name = app.AutoName or appid
1627             app.icon = None
1628             logging.debug("Application " + appid + " has no packages")
1629         else:
1630             if app.Name is None:
1631                 app.Name = bestapk['name']
1632             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1633             if app.CurrentVersionCode is None:
1634                 app.CurrentVersionCode = str(bestver)
1635
1636     # Sort the app list by name, then the web site doesn't have to by default.
1637     # (we had to wait until we'd scanned the apks to do this, because mostly the
1638     # name comes from there!)
1639     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1640
1641     # APKs are placed into multiple repos based on the app package, providing
1642     # per-app subscription feeds for nightly builds and things like it
1643     if config['per_app_repos']:
1644         add_apks_to_per_app_repos(repodirs[0], apks)
1645         for appid, app in apps.items():
1646             repodir = os.path.join(appid, 'fdroid', 'repo')
1647             appdict = dict()
1648             appdict[appid] = app
1649             if os.path.isdir(repodir):
1650                 make_index(appdict, [appid], apks, repodir, False)
1651             else:
1652                 logging.info('Skipping index generation for ' + appid)
1653         return
1654
1655     if len(repodirs) > 1:
1656         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1657
1658     # Make the index for the main repo...
1659     make_index(apps, sortedids, apks, repodirs[0], False)
1660     make_categories_txt(repodirs[0], categories)
1661
1662     # If there's an archive repo,  make the index for it. We already scanned it
1663     # earlier on.
1664     if len(repodirs) > 1:
1665         make_index(apps, sortedids, archapks, repodirs[1], True)
1666
1667     if config['update_stats']:
1668
1669         # Update known apks info...
1670         knownapks.writeifchanged()
1671
1672         # Generate latest apps data for widget
1673         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1674             data = ''
1675             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1676                 for line in f:
1677                     appid = line.rstrip()
1678                     data += appid + "\t"
1679                     app = apps[appid]
1680                     data += app.Name + "\t"
1681                     if app.icon is not None:
1682                         data += app.icon + "\t"
1683                     data += app.License + "\n"
1684             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1685                 f.write(data)
1686
1687     if cachechanged:
1688         apkcache["METADATA_VERSION"] = METADATA_VERSION
1689         with open(apkcachefile, 'wb') as cf:
1690             pickle.dump(apkcache, cf)
1691
1692     # Update the wiki...
1693     if options.wiki:
1694         update_wiki(apps, sortedids, apks + archapks)
1695
1696     logging.info("Finished.")
1697
1698
1699 if __name__ == "__main__":
1700     main()