fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import common
  41 from . import index
  42 from . import metadata
  43 from .common import SdkToolsPopen
  44 from .exception import BuildException, FDroidException
  45
  46 METADATA_VERSION = 18
  47
  48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  49 UNSET_VERSION_CODE = -0x100000000
  50
  51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85
  86 def dpi_to_px(density):
  87     return (int(density) * 48) / 160
  88
  89
  90 def px_to_dpi(px):
  91     return (int(px) * 160) / 48
  92
  93
  94 def get_icon_dir(repodir, density):
  95     if density == '0':
  96         return os.path.join(repodir, "icons")
  97     return os.path.join(repodir, "icons-%s" % density)
  98
  99
 100 def get_icon_dirs(repodir):
 101     for density in screen_densities:
 102         yield get_icon_dir(repodir, density)
 103
 104
 105 def get_all_icon_dirs(repodir):
 106     for density in all_screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def update_wiki(apps, sortedids, apks):
 111     """Update the wiki
 112
 113     :param apps: fully populated list of all applications
 114     :param apks: all apks, except...
 115     """
 116     logging.info("Updating wiki")
 117     wikicat = 'Apps'
 118     wikiredircat = 'App Redirects'
 119     import mwclient
 120     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 121                          path=config['wiki_path'])
 122     site.login(config['wiki_user'], config['wiki_password'])
 123     generated_pages = {}
 124     generated_redirects = {}
 125
 126     for appid in sortedids:
 127         app = metadata.App(apps[appid])
 128
 129         wikidata = ''
 130         if app.Disabled:
 131             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 132         if app.AntiFeatures:
 133             for af in app.AntiFeatures:
 134                 wikidata += '{{AntiFeature|' + af + '}}\n'
 135         if app.RequiresRoot:
 136             requiresroot = 'Yes'
 137         else:
 138             requiresroot = 'No'
 139         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 140             appid,
 141             app.Name,
 142             app.added.strftime('%Y-%m-%d') if app.added else '',
 143             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 144             app.SourceCode,
 145             app.IssueTracker,
 146             app.WebSite,
 147             app.Changelog,
 148             app.Donate,
 149             app.FlattrID,
 150             app.Bitcoin,
 151             app.Litecoin,
 152             app.License,
 153             requiresroot,
 154             app.AuthorName,
 155             app.AuthorEmail)
 156
 157         if app.Provides:
 158             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 159
 160         wikidata += app.Summary
 161         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 162
 163         wikidata += "=Description=\n"
 164         wikidata += metadata.description_wiki(app.Description) + "\n"
 165
 166         wikidata += "=Maintainer Notes=\n"
 167         if app.MaintainerNotes:
 168             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 169         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 170
 171         # Get a list of all packages for this application...
 172         apklist = []
 173         gotcurrentver = False
 174         cantupdate = False
 175         buildfails = False
 176         for apk in apks:
 177             if apk['packageName'] == appid:
 178                 if str(apk['versionCode']) == app.CurrentVersionCode:
 179                     gotcurrentver = True
 180                 apklist.append(apk)
 181         # Include ones we can't build, as a special case...
 182         for build in app.builds:
 183             if build.disable:
 184                 if build.versionCode == app.CurrentVersionCode:
 185                     cantupdate = True
 186                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 187                 apklist.append({'versionCode': int(build.versionCode),
 188                                 'versionName': build.versionName,
 189                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 190                                 })
 191             else:
 192                 builtit = False
 193                 for apk in apklist:
 194                     if apk['versionCode'] == int(build.versionCode):
 195                         builtit = True
 196                         break
 197                 if not builtit:
 198                     buildfails = True
 199                     apklist.append({'versionCode': int(build.versionCode),
 200                                     'versionName': build.versionName,
 201                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 202                                     })
 203         if app.CurrentVersionCode == '0':
 204             cantupdate = True
 205         # Sort with most recent first...
 206         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 207
 208         wikidata += "=Versions=\n"
 209         if len(apklist) == 0:
 210             wikidata += "We currently have no versions of this app available."
 211         elif not gotcurrentver:
 212             wikidata += "We don't have the current version of this app."
 213         else:
 214             wikidata += "We have the current version of this app."
 215         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 216         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 217         if len(app.NoSourceSince) > 0:
 218             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 219         if len(app.CurrentVersion) > 0:
 220             wikidata += "The current (recommended) version is " + app.CurrentVersion
 221             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 222         validapks = 0
 223         for apk in apklist:
 224             wikidata += "==" + apk['versionName'] + "==\n"
 225
 226             if 'buildproblem' in apk:
 227                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 228             else:
 229                 validapks += 1
 230                 wikidata += "This version is built and signed by "
 231                 if 'srcname' in apk:
 232                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 233                 else:
 234                     wikidata += "the original developer.\n\n"
 235             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 236
 237         wikidata += '\n[[Category:' + wikicat + ']]\n'
 238         if len(app.NoSourceSince) > 0:
 239             wikidata += '\n[[Category:Apps missing source code]]\n'
 240         if validapks == 0 and not app.Disabled:
 241             wikidata += '\n[[Category:Apps with no packages]]\n'
 242         if cantupdate and not app.Disabled:
 243             wikidata += "\n[[Category:Apps we cannot update]]\n"
 244         if buildfails and not app.Disabled:
 245             wikidata += "\n[[Category:Apps with failing builds]]\n"
 246         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 247             wikidata += '\n[[Category:Apps to Update]]\n'
 248         if app.Disabled:
 249             wikidata += '\n[[Category:Apps that are disabled]]\n'
 250         if app.UpdateCheckMode == 'None' and not app.Disabled:
 251             wikidata += '\n[[Category:Apps with no update check]]\n'
 252         for appcat in app.Categories:
 253             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 254
 255         # We can't have underscores in the page name, even if they're in
 256         # the package ID, because MediaWiki messes with them...
 257         pagename = appid.replace('_', ' ')
 258
 259         # Drop a trailing newline, because mediawiki is going to drop it anyway
 260         # and it we don't we'll think the page has changed when it hasn't...
 261         if wikidata.endswith('\n'):
 262             wikidata = wikidata[:-1]
 263
 264         generated_pages[pagename] = wikidata
 265
 266         # Make a redirect from the name to the ID too, unless there's
 267         # already an existing page with the name and it isn't a redirect.
 268         noclobber = False
 269         apppagename = app.Name.replace('_', ' ')
 270         apppagename = apppagename.replace('{', '')
 271         apppagename = apppagename.replace('}', ' ')
 272         apppagename = apppagename.replace(':', ' ')
 273         apppagename = apppagename.replace('[', ' ')
 274         apppagename = apppagename.replace(']', ' ')
 275         # Drop double spaces caused mostly by replacing ':' above
 276         apppagename = apppagename.replace('  ', ' ')
 277         for expagename in site.allpages(prefix=apppagename,
 278                                         filterredir='nonredirects',
 279                                         generator=False):
 280             if expagename == apppagename:
 281                 noclobber = True
 282         # Another reason not to make the redirect page is if the app name
 283         # is the same as it's ID, because that will overwrite the real page
 284         # with an redirect to itself! (Although it seems like an odd
 285         # scenario this happens a lot, e.g. where there is metadata but no
 286         # builds or binaries to extract a name from.
 287         if apppagename == pagename:
 288             noclobber = True
 289         if not noclobber:
 290             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 291
 292     for tcat, genp in [(wikicat, generated_pages),
 293                        (wikiredircat, generated_redirects)]:
 294         catpages = site.Pages['Category:' + tcat]
 295         existingpages = []
 296         for page in catpages:
 297             existingpages.append(page.name)
 298             if page.name in genp:
 299                 pagetxt = page.edit()
 300                 if pagetxt != genp[page.name]:
 301                     logging.debug("Updating modified page " + page.name)
 302                     page.save(genp[page.name], summary='Auto-updated')
 303                 else:
 304                     logging.debug("Page " + page.name + " is unchanged")
 305             else:
 306                 logging.warn("Deleting page " + page.name)
 307                 page.delete('No longer published')
 308         for pagename, text in genp.items():
 309             logging.debug("Checking " + pagename)
 310             if pagename not in existingpages:
 311                 logging.debug("Creating page " + pagename)
 312                 try:
 313                     newpage = site.Pages[pagename]
 314                     newpage.save(text, summary='Auto-created')
 315                 except Exception as e:
 316                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 317
 318     # Purge server cache to ensure counts are up to date
 319     site.pages['Repository Maintenance'].purge()
 320
 321
 322 def delete_disabled_builds(apps, apkcache, repodirs):
 323     """Delete disabled build outputs.
 324
 325     :param apps: list of all applications, as per metadata.read_metadata
 326     :param apkcache: current apk cache information
 327     :param repodirs: the repo directories to process
 328     """
 329     for appid, app in apps.items():
 330         for build in app['builds']:
 331             if not build.disable:
 332                 continue
 333             apkfilename = common.get_release_filename(app, build)
 334             iconfilename = "%s.%s.png" % (
 335                 appid,
 336                 build.versionCode)
 337             for repodir in repodirs:
 338                 files = [
 339                     os.path.join(repodir, apkfilename),
 340                     os.path.join(repodir, apkfilename + '.asc'),
 341                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 342                 ]
 343                 for density in all_screen_densities:
 344                     repo_dir = get_icon_dir(repodir, density)
 345                     files.append(os.path.join(repo_dir, iconfilename))
 346
 347                 for f in files:
 348                     if os.path.exists(f):
 349                         logging.info("Deleting disabled build output " + f)
 350                         os.remove(f)
 351             if apkfilename in apkcache:
 352                 del apkcache[apkfilename]
 353
 354
 355 def resize_icon(iconpath, density):
 356
 357     if not os.path.isfile(iconpath):
 358         return
 359
 360     fp = None
 361     try:
 362         fp = open(iconpath, 'rb')
 363         im = Image.open(fp)
 364         size = dpi_to_px(density)
 365
 366         if any(length > size for length in im.size):
 367             oldsize = im.size
 368             im.thumbnail((size, size), Image.ANTIALIAS)
 369             logging.debug("%s was too large at %s - new size is %s" % (
 370                 iconpath, oldsize, im.size))
 371             im.save(iconpath, "PNG")
 372
 373     except Exception as e:
 374         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 375
 376     finally:
 377         if fp:
 378             fp.close()
 379
 380
 381 def resize_all_icons(repodirs):
 382     """Resize all icons that exceed the max size
 383
 384     :param repodirs: the repo directories to process
 385     """
 386     for repodir in repodirs:
 387         for density in screen_densities:
 388             icon_dir = get_icon_dir(repodir, density)
 389             icon_glob = os.path.join(icon_dir, '*.png')
 390             for iconpath in glob.glob(icon_glob):
 391                 resize_icon(iconpath, density)
 392
 393
 394 def getsig(apkpath):
 395     """ Get the signing certificate of an apk. To get the same md5 has that
 396     Android gets, we encode the .RSA certificate in a specific format and pass
 397     it hex-encoded to the md5 digest algorithm.
 398
 399     :param apkpath: path to the apk
 400     :returns: A string containing the md5 of the signature of the apk or None
 401               if an error occurred.
 402     """
 403
 404     with zipfile.ZipFile(apkpath, 'r') as apk:
 405         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 406
 407         if len(certs) < 1:
 408             logging.error("Found no signing certificates on %s" % apkpath)
 409             return None
 410         if len(certs) > 1:
 411             logging.error("Found multiple signing certificates on %s" % apkpath)
 412             return None
 413
 414         cert = apk.read(certs[0])
 415
 416     cert_encoded = common.get_certificate(cert)
 417
 418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 419
 420
 421 def get_cache_file():
 422     return os.path.join('tmp', 'apkcache')
 423
 424
 425 def get_cache():
 426     """
 427     Gather information about all the apk files in the repo directory,
 428     using cached data if possible.
 429     :return: apkcache
 430     """
 431     apkcachefile = get_cache_file()
 432     if not options.clean and os.path.exists(apkcachefile):
 433         with open(apkcachefile, 'rb') as cf:
 434             apkcache = pickle.load(cf, encoding='utf-8')
 435         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
 436             apkcache = {}
 437     else:
 438         apkcache = {}
 439
 440     return apkcache
 441
 442
 443 def write_cache(apkcache):
 444     apkcachefile = get_cache_file()
 445     cache_path = os.path.dirname(apkcachefile)
 446     if not os.path.exists(cache_path):
 447         os.makedirs(cache_path)
 448     apkcache["METADATA_VERSION"] = METADATA_VERSION
 449     with open(apkcachefile, 'wb') as cf:
 450         pickle.dump(apkcache, cf)
 451
 452
 453 def get_icon_bytes(apkzip, iconsrc):
 454     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 455     try:
 456         return apkzip.read(iconsrc)
 457     except KeyError:
 458         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 459
 460
 461 def sha256sum(filename):
 462     '''Calculate the sha256 of the given file'''
 463     sha = hashlib.sha256()
 464     with open(filename, 'rb') as f:
 465         while True:
 466             t = f.read(16384)
 467             if len(t) == 0:
 468                 break
 469             sha.update(t)
 470     return sha.hexdigest()
 471
 472
 473 def has_known_vulnerability(filename):
 474     """checks for known vulnerabilities in the APK
 475
 476     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 477     version.  Google also enforces this:
 478     https://support.google.com/faqs/answer/6376725?hl=en
 479
 480     Checks whether there are more than one classes.dex or AndroidManifest.xml
 481     files, which is invalid and an essential part of the "Master Key" attack.
 482
 483     http://www.saurik.com/id/17
 484     """
 485
 486     # statically load this pattern
 487     if not hasattr(has_known_vulnerability, "pattern"):
 488         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 489
 490     files_in_apk = set()
 491     with zipfile.ZipFile(filename) as zf:
 492         for name in zf.namelist():
 493             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 494                 lib = zf.open(name)
 495                 while True:
 496                     chunk = lib.read(4096)
 497                     if chunk == b'':
 498                         break
 499                     m = has_known_vulnerability.pattern.search(chunk)
 500                     if m:
 501                         version = m.group(1).decode('ascii')
 502                         if version.startswith('1.0.1') and version[5] >= 'r' \
 503                            or version.startswith('1.0.2') and version[5] >= 'f':
 504                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 505                         else:
 506                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 507                             return True
 508                         break
 509             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 510                 if name in files_in_apk:
 511                     return True
 512                 files_in_apk.add(name)
 513
 514     return False
 515
 516
 517 def insert_obbs(repodir, apps, apks):
 518     """Scans the .obb files in a given repo directory and adds them to the
 519     relevant APK instances.  OBB files have versionCodes like APK
 520     files, and they are loosely associated.  If there is an OBB file
 521     present, then any APK with the same or higher versionCode will use
 522     that OBB file.  There are two OBB types: main and patch, each APK
 523     can only have only have one of each.
 524
 525     https://developer.android.com/google/play/expansion-files.html
 526
 527     :param repodir: repo directory to scan
 528     :param apps: list of current, valid apps
 529     :param apks: current information on all APKs
 530
 531     """
 532
 533     def obbWarnDelete(f, msg):
 534         logging.warning(msg + f)
 535         if options.delete_unknown:
 536             logging.error("Deleting unknown file: " + f)
 537             os.remove(f)
 538
 539     obbs = []
 540     java_Integer_MIN_VALUE = -pow(2, 31)
 541     currentPackageNames = apps.keys()
 542     for f in glob.glob(os.path.join(repodir, '*.obb')):
 543         obbfile = os.path.basename(f)
 544         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 545         chunks = obbfile.split('.')
 546         if chunks[0] != 'main' and chunks[0] != 'patch':
 547             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 548             continue
 549         if not re.match(r'^-?[0-9]+$', chunks[1]):
 550             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 551             continue
 552         versionCode = int(chunks[1])
 553         packagename = ".".join(chunks[2:-1])
 554
 555         highestVersionCode = java_Integer_MIN_VALUE
 556         if packagename not in currentPackageNames:
 557             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 558             continue
 559         for apk in apks:
 560             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 561                 highestVersionCode = apk['versionCode']
 562         if versionCode > highestVersionCode:
 563             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 564                           + ') than any APK: ')
 565             continue
 566         obbsha256 = sha256sum(f)
 567         obbs.append((packagename, versionCode, obbfile, obbsha256))
 568
 569     for apk in apks:
 570         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 571             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 572                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 573                     apk['obbMainFile'] = obbfile
 574                     apk['obbMainFileSha256'] = obbsha256
 575                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 576                     apk['obbPatchFile'] = obbfile
 577                     apk['obbPatchFileSha256'] = obbsha256
 578             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 579                 break
 580
 581
 582 def _get_localized_dict(app, locale):
 583     '''get the dict to add localized store metadata to'''
 584     if 'localized' not in app:
 585         app['localized'] = collections.OrderedDict()
 586     if locale not in app['localized']:
 587         app['localized'][locale] = collections.OrderedDict()
 588     return app['localized'][locale]
 589
 590
 591 def _set_localized_text_entry(app, locale, key, f):
 592     limit = config['char_limits'][key]
 593     localized = _get_localized_dict(app, locale)
 594     with open(f) as fp:
 595         text = fp.read()[:limit]
 596         if len(text) > 0:
 597             localized[key] = text
 598
 599
 600 def _set_author_entry(app, key, f):
 601     limit = config['char_limits']['author']
 602     with open(f) as fp:
 603         text = fp.read()[:limit]
 604         if len(text) > 0:
 605             app[key] = text
 606
 607
 608 def copy_triple_t_store_metadata(apps):
 609     """Include store metadata from the app's source repo
 610
 611     The Triple-T Gradle Play Publisher is a plugin that has a standard
 612     file layout for all of the metadata and graphics that the Google
 613     Play Store accepts.  Since F-Droid has the git repo, it can just
 614     pluck those files directly.  This method reads any text files into
 615     the app dict, then copies any graphics into the fdroid repo
 616     directory structure.
 617
 618     This needs to be run before insert_localized_app_metadata() so that
 619     the graphics files that are copied into the fdroid repo get
 620     properly indexed.
 621
 622     https://github.com/Triple-T/gradle-play-publisher#upload-images
 623     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 624
 625     """
 626
 627     if not os.path.isdir('build'):
 628         return  # nothing to do
 629
 630     for packageName, app in apps.items():
 631         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 632             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 633             for root, dirs, files in os.walk(d):
 634                 segments = root.split('/')
 635                 locale = segments[-2]
 636                 for f in files:
 637                     if f == 'fulldescription':
 638                         _set_localized_text_entry(app, locale, 'description',
 639                                                   os.path.join(root, f))
 640                         continue
 641                     elif f == 'shortdescription':
 642                         _set_localized_text_entry(app, locale, 'summary',
 643                                                   os.path.join(root, f))
 644                         continue
 645                     elif f == 'title':
 646                         _set_localized_text_entry(app, locale, 'name',
 647                                                   os.path.join(root, f))
 648                         continue
 649                     elif f == 'video':
 650                         _set_localized_text_entry(app, locale, 'video',
 651                                                   os.path.join(root, f))
 652                         continue
 653                     elif f == 'whatsnew':
 654                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 655                                                   os.path.join(root, f))
 656                         continue
 657                     elif f == 'contactEmail':
 658                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 659                         continue
 660                     elif f == 'contactPhone':
 661                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 662                         continue
 663                     elif f == 'contactWebsite':
 664                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 665                         continue
 666
 667                     base, extension = common.get_extension(f)
 668                     dirname = os.path.basename(root)
 669                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 670                         if segments[-2] == 'listing':
 671                             locale = segments[-3]
 672                         else:
 673                             locale = segments[-2]
 674                         destdir = os.path.join('repo', packageName, locale)
 675                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 676                         sourcefile = os.path.join(root, f)
 677                         destfile = os.path.join(destdir, dirname + '.' + extension)
 678                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 679                         shutil.copy(sourcefile, destfile)
 680
 681
 682 def insert_localized_app_metadata(apps):
 683     """scans standard locations for graphics and localized text
 684
 685     Scans for localized description files, store graphics, and
 686     screenshot PNG files in statically defined screenshots directory
 687     and adds them to the app metadata.  The screenshots and graphic
 688     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 689     and must be in the following layout:
 690     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 691
 692     repo/packageName/locale/featureGraphic.png
 693     repo/packageName/locale/phoneScreenshots/1.png
 694     repo/packageName/locale/phoneScreenshots/2.png
 695
 696     The changelog files must be text files named with the versionCode
 697     ending with ".txt" and must be in the following layout:
 698     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 699
 700     repo/packageName/locale/changelogs/12345.txt
 701
 702     This will scan the each app's source repo then the metadata/ dir
 703     for these standard locations of changelog files.  If it finds
 704     them, they will be added to the dict of all packages, with the
 705     versions in the metadata/ folder taking precendence over the what
 706     is in the app's source repo.
 707
 708     Where "packageName" is the app's packageName and "locale" is the locale
 709     of the graphics, e.g. what language they are in, using the IETF RFC5646
 710     format (en-US, fr-CA, es-MX, etc).
 711
 712     This will also scan the app's git for a fastlane folder, and the
 713     metadata/ folder and the apps' source repos for standard locations
 714     of graphic and screenshot files.  If it finds them, it will copy
 715     them into the repo.  The fastlane files follow this pattern:
 716     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 717
 718     """
 719
 720     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 721     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 722     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 723
 724     for srcd in sorted(sourcedirs):
 725         if not os.path.isdir(srcd):
 726             continue
 727         for root, dirs, files in os.walk(srcd):
 728             segments = root.split('/')
 729             packageName = segments[1]
 730             if packageName not in apps:
 731                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 732                 continue
 733             locale = segments[-1]
 734             destdir = os.path.join('repo', packageName, locale)
 735             for f in files:
 736                 if f in ('description.txt', 'full_description.txt'):
 737                     _set_localized_text_entry(apps[packageName], locale, 'description',
 738                                               os.path.join(root, f))
 739                     continue
 740                 elif f in ('summary.txt', 'short_description.txt'):
 741                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 742                                               os.path.join(root, f))
 743                     continue
 744                 elif f in ('name.txt', 'title.txt'):
 745                     _set_localized_text_entry(apps[packageName], locale, 'name',
 746                                               os.path.join(root, f))
 747                     continue
 748                 elif f == 'video.txt':
 749                     _set_localized_text_entry(apps[packageName], locale, 'video',
 750                                               os.path.join(root, f))
 751                     continue
 752                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 753                     locale = segments[-2]
 754                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 755                                               os.path.join(root, f))
 756                     continue
 757
 758                 base, extension = common.get_extension(f)
 759                 if locale == 'images':
 760                     locale = segments[-2]
 761                     destdir = os.path.join('repo', packageName, locale)
 762                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 763                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 764                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 765                     shutil.copy(os.path.join(root, f), destdir)
 766             for d in dirs:
 767                 if d in SCREENSHOT_DIRS:
 768                     for f in glob.glob(os.path.join(root, d, '*.*')):
 769                         _, extension = common.get_extension(f)
 770                         if extension in ALLOWED_EXTENSIONS:
 771                             screenshotdestdir = os.path.join(destdir, d)
 772                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 773                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 774                             shutil.copy(f, screenshotdestdir)
 775
 776     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 777     for d in repofiles:
 778         if not os.path.isdir(d):
 779             continue
 780         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 781             if not os.path.isfile(f):
 782                 continue
 783             segments = f.split('/')
 784             packageName = segments[1]
 785             locale = segments[2]
 786             screenshotdir = segments[3]
 787             filename = os.path.basename(f)
 788             base, extension = common.get_extension(filename)
 789
 790             if packageName not in apps:
 791                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 792                                 % (filename, packageName))
 793                 continue
 794             graphics = _get_localized_dict(apps[packageName], locale)
 795
 796             if extension not in ALLOWED_EXTENSIONS:
 797                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 798             elif base in GRAPHIC_NAMES:
 799                 # there can only be zero or one of these per locale
 800                 graphics[base] = filename
 801             elif screenshotdir in SCREENSHOT_DIRS:
 802                 # there can any number of these per locale
 803                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 804                 if screenshotdir not in graphics:
 805                     graphics[screenshotdir] = []
 806                 graphics[screenshotdir].append(filename)
 807             else:
 808                 logging.warning('Unsupported graphics file found: ' + f)
 809
 810
 811 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 812     """Scan a repo for all files with an extension except APK/OBB
 813
 814     :param apkcache: current cached info about all repo files
 815     :param repodir: repo directory to scan
 816     :param knownapks: list of all known files, as per metadata.read_metadata
 817     :param use_date_from_file: use date from file (instead of current date)
 818                                for newly added files
 819     """
 820
 821     cachechanged = False
 822     repo_files = []
 823     repodir = repodir.encode('utf-8')
 824     for name in os.listdir(repodir):
 825         file_extension = common.get_file_extension(name)
 826         if file_extension == 'apk' or file_extension == 'obb':
 827             continue
 828         filename = os.path.join(repodir, name)
 829         name_utf8 = name.decode('utf-8')
 830         if filename.endswith(b'_src.tar.gz'):
 831             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 832             continue
 833         if not common.is_repo_file(filename):
 834             continue
 835         stat = os.stat(filename)
 836         if stat.st_size == 0:
 837             raise FDroidException(filename + ' is zero size!')
 838
 839         shasum = sha256sum(filename)
 840         usecache = False
 841         if name in apkcache:
 842             repo_file = apkcache[name]
 843             # added time is cached as tuple but used here as datetime instance
 844             if 'added' in repo_file:
 845                 a = repo_file['added']
 846                 if isinstance(a, datetime):
 847                     repo_file['added'] = a
 848                 else:
 849                     repo_file['added'] = datetime(*a[:6])
 850             if repo_file.get('hash') == shasum:
 851                 logging.debug("Reading " + name_utf8 + " from cache")
 852                 usecache = True
 853             else:
 854                 logging.debug("Ignoring stale cache data for " + name)
 855
 856         if not usecache:
 857             logging.debug("Processing " + name_utf8)
 858             repo_file = collections.OrderedDict()
 859             repo_file['name'] = os.path.splitext(name_utf8)[0]
 860             # TODO rename apkname globally to something more generic
 861             repo_file['apkName'] = name_utf8
 862             repo_file['hash'] = shasum
 863             repo_file['hashType'] = 'sha256'
 864             repo_file['versionCode'] = 0
 865             repo_file['versionName'] = shasum
 866             # the static ID is the SHA256 unless it is set in the metadata
 867             repo_file['packageName'] = shasum
 868
 869             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 870             if m:
 871                 repo_file['packageName'] = m.group(1)
 872                 repo_file['versionCode'] = int(m.group(2))
 873             srcfilename = name + b'_src.tar.gz'
 874             if os.path.exists(os.path.join(repodir, srcfilename)):
 875                 repo_file['srcname'] = srcfilename.decode('utf-8')
 876             repo_file['size'] = stat.st_size
 877
 878             apkcache[name] = repo_file
 879             cachechanged = True
 880
 881         if use_date_from_file:
 882             timestamp = stat.st_ctime
 883             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 884         else:
 885             default_date_param = None
 886
 887         # Record in knownapks, getting the added date at the same time..
 888         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 889                                     default_date=default_date_param)
 890         if added:
 891             repo_file['added'] = added
 892
 893         repo_files.append(repo_file)
 894
 895     return repo_files, cachechanged
 896
 897
 898 def scan_apk_aapt(apk, apkfile):
 899     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 900     if p.returncode != 0:
 901         if options.delete_unknown:
 902             if os.path.exists(apkfile):
 903                 logging.error("Failed to get apk information, deleting " + apkfile)
 904                 os.remove(apkfile)
 905             else:
 906                 logging.error("Could not find {0} to remove it".format(apkfile))
 907         else:
 908             logging.error("Failed to get apk information, skipping " + apkfile)
 909         raise BuildException("Invalid APK")
 910     for line in p.output.splitlines():
 911         if line.startswith("package:"):
 912             try:
 913                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 914                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 915                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 916             except Exception as e:
 917                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
 918         elif line.startswith("application:"):
 919             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 920             # Keep path to non-dpi icon in case we need it
 921             match = re.match(APK_ICON_PAT_NODPI, line)
 922             if match:
 923                 apk['icons_src']['-1'] = match.group(1)
 924         elif line.startswith("launchable-activity:"):
 925             # Only use launchable-activity as fallback to application
 926             if not apk['name']:
 927                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 928             if '-1' not in apk['icons_src']:
 929                 match = re.match(APK_ICON_PAT_NODPI, line)
 930                 if match:
 931                     apk['icons_src']['-1'] = match.group(1)
 932         elif line.startswith("application-icon-"):
 933             match = re.match(APK_ICON_PAT, line)
 934             if match:
 935                 density = match.group(1)
 936                 path = match.group(2)
 937                 apk['icons_src'][density] = path
 938         elif line.startswith("sdkVersion:"):
 939             m = re.match(APK_SDK_VERSION_PAT, line)
 940             if m is None:
 941                 logging.error(line.replace('sdkVersion:', '')
 942                               + ' is not a valid minSdkVersion!')
 943             else:
 944                 apk['minSdkVersion'] = m.group(1)
 945                 # if target not set, default to min
 946                 if 'targetSdkVersion' not in apk:
 947                     apk['targetSdkVersion'] = m.group(1)
 948         elif line.startswith("targetSdkVersion:"):
 949             m = re.match(APK_SDK_VERSION_PAT, line)
 950             if m is None:
 951                 logging.error(line.replace('targetSdkVersion:', '')
 952                               + ' is not a valid targetSdkVersion!')
 953             else:
 954                 apk['targetSdkVersion'] = m.group(1)
 955         elif line.startswith("maxSdkVersion:"):
 956             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 957         elif line.startswith("native-code:"):
 958             apk['nativecode'] = []
 959             for arch in line[13:].split(' '):
 960                 apk['nativecode'].append(arch[1:-1])
 961         elif line.startswith('uses-permission:'):
 962             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 963             if perm_match['maxSdkVersion']:
 964                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 965             permission = UsesPermission(
 966                 perm_match['name'],
 967                 perm_match['maxSdkVersion']
 968             )
 969
 970             apk['uses-permission'].append(permission)
 971         elif line.startswith('uses-permission-sdk-23:'):
 972             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 973             if perm_match['maxSdkVersion']:
 974                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 975             permission_sdk_23 = UsesPermissionSdk23(
 976                 perm_match['name'],
 977                 perm_match['maxSdkVersion']
 978             )
 979
 980             apk['uses-permission-sdk-23'].append(permission_sdk_23)
 981
 982         elif line.startswith('uses-feature:'):
 983             feature = re.match(APK_FEATURE_PAT, line).group(1)
 984             # Filter out this, it's only added with the latest SDK tools and
 985             # causes problems for lots of apps.
 986             if feature != "android.hardware.screen.portrait" \
 987                     and feature != "android.hardware.screen.landscape":
 988                 if feature.startswith("android.feature."):
 989                     feature = feature[16:]
 990                 apk['features'].add(feature)
 991
 992
 993 def scan_apk_androguard(apk, apkfile):
 994     try:
 995         from androguard.core.bytecodes.apk import APK
 996         apkobject = APK(apkfile)
 997         if apkobject.is_valid_APK():
 998             arsc = apkobject.get_android_resources()
 999         else:
1000             if options.delete_unknown:
1001                 if os.path.exists(apkfile):
1002                     logging.error("Failed to get apk information, deleting " + apkfile)
1003                     os.remove(apkfile)
1004                 else:
1005                     logging.error("Could not find {0} to remove it".format(apkfile))
1006             else:
1007                 logging.error("Failed to get apk information, skipping " + apkfile)
1008             raise BuildException("Invaild APK")
1009     except ImportError:
1010         raise FDroidException("androguard library is not installed and aapt not present")
1011     except FileNotFoundError:
1012         logging.error("Could not open apk file for analysis")
1013         raise BuildException("Invalid APK")
1014
1015     apk['packageName'] = apkobject.get_package()
1016     apk['versionCode'] = int(apkobject.get_androidversion_code())
1017     apk['versionName'] = apkobject.get_androidversion_name()
1018     if apk['versionName'][0] == "@":
1019         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1020         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1021         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1022     apk['name'] = apkobject.get_app_name()
1023
1024     if apkobject.get_max_sdk_version() is not None:
1025         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1026     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1027     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1028
1029     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1030     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1031
1032     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1033
1034     for file in apkobject.get_files():
1035         d_re = density_re.match(file)
1036         if d_re:
1037             folder = d_re.group(1).split('-')
1038             if len(folder) > 1:
1039                 resolution = folder[1]
1040             else:
1041                 resolution = 'mdpi'
1042             density = screen_resolutions[resolution]
1043             apk['icons_src'][density] = d_re.group(0)
1044
1045     if apk['icons_src'].get('-1') is None:
1046         apk['icons_src']['-1'] = apk['icons_src']['160']
1047
1048     arch_re = re.compile("^lib/(.*)/.*$")
1049     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1050     if len(arch) >= 1:
1051         apk['nativecode'] = []
1052         apk['nativecode'].extend(sorted(list(arch)))
1053
1054     xml = apkobject.get_android_manifest_xml()
1055
1056     for item in xml.getElementsByTagName('uses-permission'):
1057         name = str(item.getAttribute("android:name"))
1058         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1059         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1060         permission = UsesPermission(
1061             name,
1062             maxSdkVersion
1063         )
1064         apk['uses-permission'].append(permission)
1065
1066     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1067         name = str(item.getAttribute("android:name"))
1068         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1069         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1070         permission_sdk_23 = UsesPermissionSdk23(
1071             name,
1072             maxSdkVersion
1073         )
1074         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1075
1076     for item in xml.getElementsByTagName('uses-feature'):
1077         feature = str(item.getAttribute("android:name"))
1078         if feature != "android.hardware.screen.portrait" \
1079                 and feature != "android.hardware.screen.landscape":
1080             if feature.startswith("android.feature."):
1081                 feature = feature[16:]
1082         apk['features'].append(feature)
1083
1084
1085 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1086              allow_disabled_algorithms=False, archive_bad_sig=False):
1087     """Scan the apk with the given filename in the given repo directory.
1088
1089     This also extracts the icons.
1090
1091     :param apkcache: current apk cache information
1092     :param apkfilename: the filename of the apk to scan
1093     :param repodir: repo directory to scan
1094     :param knownapks: known apks info
1095     :param use_date_from_apk: use date from APK (instead of current date)
1096                               for newly added APKs
1097     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1098                                       disabled algorithms in the signature (e.g. MD5)
1099     :param archive_bad_sig: move APKs with a bad signature to the archive
1100     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1101      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1102     """
1103
1104     if ' ' in apkfilename:
1105         if options.rename_apks:
1106             newfilename = apkfilename.replace(' ', '_')
1107             os.rename(os.path.join(repodir, apkfilename),
1108                       os.path.join(repodir, newfilename))
1109             apkfilename = newfilename
1110         else:
1111             logging.critical("Spaces in filenames are not allowed.")
1112             return True, None, False
1113
1114     apkfile = os.path.join(repodir, apkfilename)
1115     shasum = sha256sum(apkfile)
1116
1117     cachechanged = False
1118     usecache = False
1119     if apkfilename in apkcache:
1120         apk = apkcache[apkfilename]
1121         if apk.get('hash') == shasum:
1122             logging.debug("Reading " + apkfilename + " from cache")
1123             usecache = True
1124         else:
1125             logging.debug("Ignoring stale cache data for " + apkfilename)
1126
1127     if not usecache:
1128         logging.debug("Processing " + apkfilename)
1129         apk = {}
1130         apk['hash'] = shasum
1131         apk['hashType'] = 'sha256'
1132         apk['uses-permission'] = []
1133         apk['uses-permission-sdk-23'] = []
1134         apk['features'] = []
1135         apk['icons_src'] = {}
1136         apk['icons'] = {}
1137         apk['antiFeatures'] = set()
1138
1139         try:
1140             if SdkToolsPopen(['aapt', 'version'], output=False):
1141                 scan_apk_aapt(apk, apkfile)
1142             else:
1143                 scan_apk_androguard(apk, apkfile)
1144         except BuildException:
1145             return True, None, False
1146
1147         if 'minSdkVersion' not in apk:
1148             logging.warn("No SDK version information found in {0}".format(apkfile))
1149             apk['minSdkVersion'] = 1
1150
1151         # Check for debuggable apks...
1152         if common.isApkAndDebuggable(apkfile):
1153             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1154
1155         # Get the signature (or md5 of, to be precise)...
1156         logging.debug('Getting signature of {0}'.format(apkfile))
1157         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1158         if not apk['sig']:
1159             logging.critical("Failed to get apk signature")
1160             return True, None, False
1161
1162         if options.rename_apks:
1163             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1164             std_short_name = os.path.join(repodir, n)
1165             if apkfile != std_short_name:
1166                 if os.path.exists(std_short_name):
1167                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1168                     if apkfile != std_long_name:
1169                         if os.path.exists(std_long_name):
1170                             dupdir = os.path.join('duplicates', repodir)
1171                             if not os.path.isdir(dupdir):
1172                                 os.makedirs(dupdir, exist_ok=True)
1173                             dupfile = os.path.join('duplicates', std_long_name)
1174                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1175                             os.rename(apkfile, dupfile)
1176                             return True, None, False
1177                         else:
1178                             os.rename(apkfile, std_long_name)
1179                     apkfile = std_long_name
1180                 else:
1181                     os.rename(apkfile, std_short_name)
1182                     apkfile = std_short_name
1183                 apkfilename = apkfile[len(repodir) + 1:]
1184
1185         apk['apkName'] = apkfilename
1186         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1187         if os.path.exists(os.path.join(repodir, srcfilename)):
1188             apk['srcname'] = srcfilename
1189         apk['size'] = os.path.getsize(apkfile)
1190
1191         # verify the jar signature is correct, allow deprecated
1192         # algorithms only if the APK is in the archive.
1193         skipapk = False
1194         if not common.verify_apk_signature(apkfile):
1195             if repodir == 'archive' or allow_disabled_algorithms:
1196                 if common.verify_old_apk_signature(apkfile):
1197                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1198                 else:
1199                     skipapk = True
1200             else:
1201                 skipapk = True
1202
1203         if skipapk:
1204             if archive_bad_sig:
1205                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1206                 move_apk_between_sections(repodir, 'archive', apk)
1207             else:
1208                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1209             return True, None, False
1210
1211         if 'KnownVuln' not in apk['antiFeatures']:
1212             if has_known_vulnerability(apkfile):
1213                 apk['antiFeatures'].add('KnownVuln')
1214
1215         apkzip = zipfile.ZipFile(apkfile, 'r')
1216
1217         # if an APK has files newer than the system time, suggest updating
1218         # the system clock.  This is useful for offline systems, used for
1219         # signing, which do not have another source of clock sync info. It
1220         # has to be more than 24 hours newer because ZIP/APK files do not
1221         # store timezone info
1222         manifest = apkzip.getinfo('AndroidManifest.xml')
1223         if manifest.date_time[1] == 0:  # month can't be zero
1224             logging.debug('AndroidManifest.xml has no date')
1225         else:
1226             dt_obj = datetime(*manifest.date_time)
1227             checkdt = dt_obj - timedelta(1)
1228             if datetime.today() < checkdt:
1229                 logging.warn('System clock is older than manifest in: '
1230                              + apkfilename
1231                              + '\nSet clock to that time using:\n'
1232                              + 'sudo date -s "' + str(dt_obj) + '"')
1233
1234         iconfilename = "%s.%s.png" % (
1235             apk['packageName'],
1236             apk['versionCode'])
1237
1238         # Extract the icon file...
1239         empty_densities = []
1240         for density in screen_densities:
1241             if density not in apk['icons_src']:
1242                 empty_densities.append(density)
1243                 continue
1244             iconsrc = apk['icons_src'][density]
1245             icon_dir = get_icon_dir(repodir, density)
1246             icondest = os.path.join(icon_dir, iconfilename)
1247
1248             try:
1249                 with open(icondest, 'wb') as f:
1250                     f.write(get_icon_bytes(apkzip, iconsrc))
1251                 apk['icons'][density] = iconfilename
1252             except (zipfile.BadZipFile, ValueError, KeyError) as e:
1253                 logging.warning("Error retrieving icon file: %s" % (icondest))
1254                 del apk['icons_src'][density]
1255                 empty_densities.append(density)
1256
1257         if '-1' in apk['icons_src']:
1258             iconsrc = apk['icons_src']['-1']
1259             iconpath = os.path.join(
1260                 get_icon_dir(repodir, '0'), iconfilename)
1261             with open(iconpath, 'wb') as f:
1262                 f.write(get_icon_bytes(apkzip, iconsrc))
1263             try:
1264                 im = Image.open(iconpath)
1265                 dpi = px_to_dpi(im.size[0])
1266                 for density in screen_densities:
1267                     if density in apk['icons']:
1268                         break
1269                     if density == screen_densities[-1] or dpi >= int(density):
1270                         apk['icons'][density] = iconfilename
1271                         shutil.move(iconpath,
1272                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
1273                         empty_densities.remove(density)
1274                         break
1275             except Exception as e:
1276                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1277
1278         if apk['icons']:
1279             apk['icon'] = iconfilename
1280
1281         apkzip.close()
1282
1283         # First try resizing down to not lose quality
1284         last_density = None
1285         for density in screen_densities:
1286             if density not in empty_densities:
1287                 last_density = density
1288                 continue
1289             if last_density is None:
1290                 continue
1291             logging.debug("Density %s not available, resizing down from %s"
1292                           % (density, last_density))
1293
1294             last_iconpath = os.path.join(
1295                 get_icon_dir(repodir, last_density), iconfilename)
1296             iconpath = os.path.join(
1297                 get_icon_dir(repodir, density), iconfilename)
1298             fp = None
1299             try:
1300                 fp = open(last_iconpath, 'rb')
1301                 im = Image.open(fp)
1302
1303                 size = dpi_to_px(density)
1304
1305                 im.thumbnail((size, size), Image.ANTIALIAS)
1306                 im.save(iconpath, "PNG")
1307                 empty_densities.remove(density)
1308             except Exception as e:
1309                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1310             finally:
1311                 if fp:
1312                     fp.close()
1313
1314         # Then just copy from the highest resolution available
1315         last_density = None
1316         for density in reversed(screen_densities):
1317             if density not in empty_densities:
1318                 last_density = density
1319                 continue
1320             if last_density is None:
1321                 continue
1322             logging.debug("Density %s not available, copying from lower density %s"
1323                           % (density, last_density))
1324
1325             shutil.copyfile(
1326                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1327                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1328
1329             empty_densities.remove(density)
1330
1331         for density in screen_densities:
1332             icon_dir = get_icon_dir(repodir, density)
1333             icondest = os.path.join(icon_dir, iconfilename)
1334             resize_icon(icondest, density)
1335
1336         # Copy from icons-mdpi to icons since mdpi is the baseline density
1337         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1338         if os.path.isfile(baseline):
1339             apk['icons']['0'] = iconfilename
1340             shutil.copyfile(baseline,
1341                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1342
1343         if use_date_from_apk and manifest.date_time[1] != 0:
1344             default_date_param = datetime(*manifest.date_time)
1345         else:
1346             default_date_param = None
1347
1348         # Record in known apks, getting the added date at the same time..
1349         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1350                                     default_date=default_date_param)
1351         if added:
1352             apk['added'] = added
1353
1354         apkcache[apkfilename] = apk
1355         cachechanged = True
1356
1357     return False, apk, cachechanged
1358
1359
1360 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1361     """Scan the apks in the given repo directory.
1362
1363     This also extracts the icons.
1364
1365     :param apkcache: current apk cache information
1366     :param repodir: repo directory to scan
1367     :param knownapks: known apks info
1368     :param use_date_from_apk: use date from APK (instead of current date)
1369                               for newly added APKs
1370     :returns: (apks, cachechanged) where apks is a list of apk information,
1371               and cachechanged is True if the apkcache got changed.
1372     """
1373
1374     cachechanged = False
1375
1376     for icon_dir in get_all_icon_dirs(repodir):
1377         if os.path.exists(icon_dir):
1378             if options.clean:
1379                 shutil.rmtree(icon_dir)
1380                 os.makedirs(icon_dir)
1381         else:
1382             os.makedirs(icon_dir)
1383
1384     apks = []
1385     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1386         apkfilename = apkfile[len(repodir) + 1:]
1387         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1388         (skip, apk, cachethis) = scan_apk(apkcache, apkfilename, repodir, knownapks,
1389                                           use_date_from_apk, ada, True)
1390         if skip:
1391             continue
1392         apks.append(apk)
1393         cachechanged = cachechanged or cachethis
1394
1395     return apks, cachechanged
1396
1397
1398 def apply_info_from_latest_apk(apps, apks):
1399     """
1400     Some information from the apks needs to be applied up to the application level.
1401     When doing this, we use the info from the most recent version's apk.
1402     We deal with figuring out when the app was added and last updated at the same time.
1403     """
1404     for appid, app in apps.items():
1405         bestver = UNSET_VERSION_CODE
1406         for apk in apks:
1407             if apk['packageName'] == appid:
1408                 if apk['versionCode'] > bestver:
1409                     bestver = apk['versionCode']
1410                     bestapk = apk
1411
1412                 if 'added' in apk:
1413                     if not app.added or apk['added'] < app.added:
1414                         app.added = apk['added']
1415                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1416                         app.lastUpdated = apk['added']
1417
1418         if not app.added:
1419             logging.debug("Don't know when " + appid + " was added")
1420         if not app.lastUpdated:
1421             logging.debug("Don't know when " + appid + " was last updated")
1422
1423         if bestver == UNSET_VERSION_CODE:
1424
1425             if app.Name is None:
1426                 app.Name = app.AutoName or appid
1427             app.icon = None
1428             logging.debug("Application " + appid + " has no packages")
1429         else:
1430             if app.Name is None:
1431                 app.Name = bestapk['name']
1432             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1433             if app.CurrentVersionCode is None:
1434                 app.CurrentVersionCode = str(bestver)
1435
1436
1437 def make_categories_txt(repodir, categories):
1438     '''Write a category list in the repo to allow quick access'''
1439     catdata = ''
1440     for cat in sorted(categories):
1441         catdata += cat + '\n'
1442     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1443         f.write(catdata)
1444
1445
1446 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1447
1448     def filter_apk_list_sorted(apk_list):
1449         res = []
1450         for apk in apk_list:
1451             if apk['packageName'] == appid:
1452                 res.append(apk)
1453
1454         # Sort the apk list by version code. First is highest/newest.
1455         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1456
1457     for appid, app in apps.items():
1458
1459         if app.ArchivePolicy:
1460             keepversions = int(app.ArchivePolicy[:-9])
1461         else:
1462             keepversions = defaultkeepversions
1463
1464         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1465                       .format(appid, len(apks), keepversions, len(archapks)))
1466
1467         current_app_apks = filter_apk_list_sorted(apks)
1468         if len(current_app_apks) > keepversions:
1469             # Move back the ones we don't want.
1470             for apk in current_app_apks[keepversions:]:
1471                 move_apk_between_sections(repodir, archivedir, apk)
1472                 archapks.append(apk)
1473                 apks.remove(apk)
1474
1475         current_app_archapks = filter_apk_list_sorted(archapks)
1476         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1477             kept = 0
1478             # Move forward the ones we want again, except DisableAlgorithm
1479             for apk in current_app_archapks:
1480                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1481                     move_apk_between_sections(archivedir, repodir, apk)
1482                     archapks.remove(apk)
1483                     apks.append(apk)
1484                     kept += 1
1485                 if kept == keepversions:
1486                     break
1487
1488
1489 def move_apk_between_sections(from_dir, to_dir, apk):
1490     """move an APK from repo to archive or vice versa"""
1491
1492     def _move_file(from_dir, to_dir, filename, ignore_missing):
1493         from_path = os.path.join(from_dir, filename)
1494         if ignore_missing and not os.path.exists(from_path):
1495             return
1496         to_path = os.path.join(to_dir, filename)
1497         if not os.path.exists(to_dir):
1498             os.mkdir(to_dir)
1499         shutil.move(from_path, to_path)
1500
1501     if from_dir == to_dir:
1502         return
1503
1504     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1505     _move_file(from_dir, to_dir, apk['apkName'], False)
1506     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1507     for density in all_screen_densities:
1508         from_icon_dir = get_icon_dir(from_dir, density)
1509         to_icon_dir = get_icon_dir(to_dir, density)
1510         if density not in apk['icons']:
1511             continue
1512         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1513     if 'srcname' in apk:
1514         _move_file(from_dir, to_dir, apk['srcname'], False)
1515
1516
1517 def add_apks_to_per_app_repos(repodir, apks):
1518     apks_per_app = dict()
1519     for apk in apks:
1520         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1521         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1522         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1523         apks_per_app[apk['packageName']] = apk
1524
1525         if not os.path.exists(apk['per_app_icons']):
1526             logging.info('Adding new repo for only ' + apk['packageName'])
1527             os.makedirs(apk['per_app_icons'])
1528
1529         apkpath = os.path.join(repodir, apk['apkName'])
1530         shutil.copy(apkpath, apk['per_app_repo'])
1531         apksigpath = apkpath + '.sig'
1532         if os.path.exists(apksigpath):
1533             shutil.copy(apksigpath, apk['per_app_repo'])
1534         apkascpath = apkpath + '.asc'
1535         if os.path.exists(apkascpath):
1536             shutil.copy(apkascpath, apk['per_app_repo'])
1537
1538
1539 config = None
1540 options = None
1541
1542
1543 def main():
1544
1545     global config, options
1546
1547     # Parse command line...
1548     parser = ArgumentParser()
1549     common.setup_global_opts(parser)
1550     parser.add_argument("--create-key", action="store_true", default=False,
1551                         help="Create a repo signing key in a keystore")
1552     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1553                         help="Create skeleton metadata files that are missing")
1554     parser.add_argument("--delete-unknown", action="store_true", default=False,
1555                         help="Delete APKs and/or OBBs without metadata from the repo")
1556     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1557                         help="Report on build data status")
1558     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1559                         help="Interactively ask about things that need updating.")
1560     parser.add_argument("-I", "--icons", action="store_true", default=False,
1561                         help="Resize all the icons exceeding the max pixel size and exit")
1562     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1563                         help="Specify editor to use in interactive mode. Default " +
1564                         "is /etc/alternatives/editor")
1565     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1566                         help="Update the wiki")
1567     parser.add_argument("--pretty", action="store_true", default=False,
1568                         help="Produce human-readable index.xml")
1569     parser.add_argument("--clean", action="store_true", default=False,
1570                         help="Clean update - don't uses caches, reprocess all apks")
1571     parser.add_argument("--nosign", action="store_true", default=False,
1572                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1573     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1574                         help="Use date from apk instead of current time for newly added apks")
1575     parser.add_argument("--rename-apks", action="store_true", default=False,
1576                         help="Rename APK files that do not match package.name_123.apk")
1577     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1578                         help="Include APKs that are signed with disabled algorithms like MD5")
1579     metadata.add_metadata_arguments(parser)
1580     options = parser.parse_args()
1581     metadata.warnings_action = options.W
1582
1583     config = common.read_config(options)
1584
1585     if not ('jarsigner' in config and 'keytool' in config):
1586         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1587
1588     repodirs = ['repo']
1589     if config['archive_older'] != 0:
1590         repodirs.append('archive')
1591         if not os.path.exists('archive'):
1592             os.mkdir('archive')
1593
1594     if options.icons:
1595         resize_all_icons(repodirs)
1596         sys.exit(0)
1597
1598     if options.rename_apks:
1599         options.clean = True
1600
1601     # check that icons exist now, rather than fail at the end of `fdroid update`
1602     for k in ['repo_icon', 'archive_icon']:
1603         if k in config:
1604             if not os.path.exists(config[k]):
1605                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1606                 sys.exit(1)
1607
1608     # if the user asks to create a keystore, do it now, reusing whatever it can
1609     if options.create_key:
1610         if os.path.exists(config['keystore']):
1611             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1612             logging.critical("\t'" + config['keystore'] + "'")
1613             sys.exit(1)
1614
1615         if 'repo_keyalias' not in config:
1616             config['repo_keyalias'] = socket.getfqdn()
1617             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1618         if 'keydname' not in config:
1619             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1620             common.write_to_config(config, 'keydname', config['keydname'])
1621         if 'keystore' not in config:
1622             config['keystore'] = common.default_config['keystore']
1623             common.write_to_config(config, 'keystore', config['keystore'])
1624
1625         password = common.genpassword()
1626         if 'keystorepass' not in config:
1627             config['keystorepass'] = password
1628             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1629         if 'keypass' not in config:
1630             config['keypass'] = password
1631             common.write_to_config(config, 'keypass', config['keypass'])
1632         common.genkeystore(config)
1633
1634     # Get all apps...
1635     apps = metadata.read_metadata()
1636
1637     # Generate a list of categories...
1638     categories = set()
1639     for app in apps.values():
1640         categories.update(app.Categories)
1641
1642     # Read known apks data (will be updated and written back when we've finished)
1643     knownapks = common.KnownApks()
1644
1645     # Get APK cache
1646     apkcache = get_cache()
1647
1648     # Delete builds for disabled apps
1649     delete_disabled_builds(apps, apkcache, repodirs)
1650
1651     # Scan all apks in the main repo
1652     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1653
1654     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1655                                            options.use_date_from_apk)
1656     cachechanged = cachechanged or fcachechanged
1657     apks += files
1658     # Generate warnings for apk's with no metadata (or create skeleton
1659     # metadata files, if requested on the command line)
1660     newmetadata = False
1661     for apk in apks:
1662         if apk['packageName'] not in apps:
1663             if options.create_metadata:
1664                 if 'name' not in apk:
1665                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1666                     continue
1667                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1668                 f.write("License:Unknown\n")
1669                 f.write("Web Site:\n")
1670                 f.write("Source Code:\n")
1671                 f.write("Issue Tracker:\n")
1672                 f.write("Changelog:\n")
1673                 f.write("Summary:" + apk['name'] + "\n")
1674                 f.write("Description:\n")
1675                 f.write(apk['name'] + "\n")
1676                 f.write(".\n")
1677                 f.write("Name:" + apk['name'] + "\n")
1678                 f.close()
1679                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1680                 newmetadata = True
1681             else:
1682                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1683                 if options.delete_unknown:
1684                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1685                     rmf = os.path.join(repodirs[0], apk['apkName'])
1686                     if not os.path.exists(rmf):
1687                         logging.error("Could not find {0} to remove it".format(rmf))
1688                     else:
1689                         os.remove(rmf)
1690                 else:
1691                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1692
1693     # update the metadata with the newly created ones included
1694     if newmetadata:
1695         apps = metadata.read_metadata()
1696
1697     copy_triple_t_store_metadata(apps)
1698     insert_obbs(repodirs[0], apps, apks)
1699     insert_localized_app_metadata(apps)
1700
1701     # Scan the archive repo for apks as well
1702     if len(repodirs) > 1:
1703         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1704         if cc:
1705             cachechanged = True
1706     else:
1707         archapks = []
1708
1709     # Apply information from latest apks to the application and update dates
1710     apply_info_from_latest_apk(apps, apks + archapks)
1711
1712     # Sort the app list by name, then the web site doesn't have to by default.
1713     # (we had to wait until we'd scanned the apks to do this, because mostly the
1714     # name comes from there!)
1715     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1716
1717     # APKs are placed into multiple repos based on the app package, providing
1718     # per-app subscription feeds for nightly builds and things like it
1719     if config['per_app_repos']:
1720         add_apks_to_per_app_repos(repodirs[0], apks)
1721         for appid, app in apps.items():
1722             repodir = os.path.join(appid, 'fdroid', 'repo')
1723             appdict = dict()
1724             appdict[appid] = app
1725             if os.path.isdir(repodir):
1726                 index.make(appdict, [appid], apks, repodir, False)
1727             else:
1728                 logging.info('Skipping index generation for ' + appid)
1729         return
1730
1731     if len(repodirs) > 1:
1732         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1733
1734     # Make the index for the main repo...
1735     index.make(apps, sortedids, apks, repodirs[0], False)
1736     make_categories_txt(repodirs[0], categories)
1737
1738     # If there's an archive repo,  make the index for it. We already scanned it
1739     # earlier on.
1740     if len(repodirs) > 1:
1741         index.make(apps, sortedids, archapks, repodirs[1], True)
1742
1743     git_remote = config.get('binary_transparency_remote')
1744     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1745         from . import btlog
1746         btlog.make_binary_transparency_log(repodirs)
1747
1748     if config['update_stats']:
1749         # Update known apks info...
1750         knownapks.writeifchanged()
1751
1752         # Generate latest apps data for widget
1753         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1754             data = ''
1755             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1756                 for line in f:
1757                     appid = line.rstrip()
1758                     data += appid + "\t"
1759                     app = apps[appid]
1760                     data += app.Name + "\t"
1761                     if app.icon is not None:
1762                         data += app.icon + "\t"
1763                     data += app.License + "\n"
1764             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1765                 f.write(data)
1766
1767     if cachechanged:
1768         write_cache(apkcache)
1769
1770     # Update the wiki...
1771     if options.wiki:
1772         update_wiki(apps, sortedids, apks + archapks)
1773
1774     logging.info("Finished.")
1775
1776
1777 if __name__ == "__main__":
1778     main()