fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import common
  41 from . import index
  42 from . import metadata
  43 from .common import SdkToolsPopen
  44 from .exception import BuildException, FDroidException
  45
  46 METADATA_VERSION = 18
  47
  48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  49 UNSET_VERSION_CODE = -0x100000000
  50
  51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85
  86 def dpi_to_px(density):
  87     return (int(density) * 48) / 160
  88
  89
  90 def px_to_dpi(px):
  91     return (int(px) * 160) / 48
  92
  93
  94 def get_icon_dir(repodir, density):
  95     if density == '0':
  96         return os.path.join(repodir, "icons")
  97     return os.path.join(repodir, "icons-%s" % density)
  98
  99
 100 def get_icon_dirs(repodir):
 101     for density in screen_densities:
 102         yield get_icon_dir(repodir, density)
 103
 104
 105 def get_all_icon_dirs(repodir):
 106     for density in all_screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def update_wiki(apps, sortedids, apks):
 111     """Update the wiki
 112
 113     :param apps: fully populated list of all applications
 114     :param apks: all apks, except...
 115     """
 116     logging.info("Updating wiki")
 117     wikicat = 'Apps'
 118     wikiredircat = 'App Redirects'
 119     import mwclient
 120     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 121                          path=config['wiki_path'])
 122     site.login(config['wiki_user'], config['wiki_password'])
 123     generated_pages = {}
 124     generated_redirects = {}
 125
 126     for appid in sortedids:
 127         app = metadata.App(apps[appid])
 128
 129         wikidata = ''
 130         if app.Disabled:
 131             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 132         if app.AntiFeatures:
 133             for af in app.AntiFeatures:
 134                 wikidata += '{{AntiFeature|' + af + '}}\n'
 135         if app.RequiresRoot:
 136             requiresroot = 'Yes'
 137         else:
 138             requiresroot = 'No'
 139         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 140             appid,
 141             app.Name,
 142             app.added.strftime('%Y-%m-%d') if app.added else '',
 143             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 144             app.SourceCode,
 145             app.IssueTracker,
 146             app.WebSite,
 147             app.Changelog,
 148             app.Donate,
 149             app.FlattrID,
 150             app.Bitcoin,
 151             app.Litecoin,
 152             app.License,
 153             requiresroot,
 154             app.AuthorName,
 155             app.AuthorEmail)
 156
 157         if app.Provides:
 158             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 159
 160         wikidata += app.Summary
 161         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 162
 163         wikidata += "=Description=\n"
 164         wikidata += metadata.description_wiki(app.Description) + "\n"
 165
 166         wikidata += "=Maintainer Notes=\n"
 167         if app.MaintainerNotes:
 168             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 169         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 170
 171         # Get a list of all packages for this application...
 172         apklist = []
 173         gotcurrentver = False
 174         cantupdate = False
 175         buildfails = False
 176         for apk in apks:
 177             if apk['packageName'] == appid:
 178                 if str(apk['versionCode']) == app.CurrentVersionCode:
 179                     gotcurrentver = True
 180                 apklist.append(apk)
 181         # Include ones we can't build, as a special case...
 182         for build in app.builds:
 183             if build.disable:
 184                 if build.versionCode == app.CurrentVersionCode:
 185                     cantupdate = True
 186                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 187                 apklist.append({'versionCode': int(build.versionCode),
 188                                 'versionName': build.versionName,
 189                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 190                                 })
 191             else:
 192                 builtit = False
 193                 for apk in apklist:
 194                     if apk['versionCode'] == int(build.versionCode):
 195                         builtit = True
 196                         break
 197                 if not builtit:
 198                     buildfails = True
 199                     apklist.append({'versionCode': int(build.versionCode),
 200                                     'versionName': build.versionName,
 201                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 202                                     })
 203         if app.CurrentVersionCode == '0':
 204             cantupdate = True
 205         # Sort with most recent first...
 206         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 207
 208         wikidata += "=Versions=\n"
 209         if len(apklist) == 0:
 210             wikidata += "We currently have no versions of this app available."
 211         elif not gotcurrentver:
 212             wikidata += "We don't have the current version of this app."
 213         else:
 214             wikidata += "We have the current version of this app."
 215         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 216         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 217         if len(app.NoSourceSince) > 0:
 218             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 219         if len(app.CurrentVersion) > 0:
 220             wikidata += "The current (recommended) version is " + app.CurrentVersion
 221             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 222         validapks = 0
 223         for apk in apklist:
 224             wikidata += "==" + apk['versionName'] + "==\n"
 225
 226             if 'buildproblem' in apk:
 227                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 228             else:
 229                 validapks += 1
 230                 wikidata += "This version is built and signed by "
 231                 if 'srcname' in apk:
 232                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 233                 else:
 234                     wikidata += "the original developer.\n\n"
 235             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 236
 237         wikidata += '\n[[Category:' + wikicat + ']]\n'
 238         if len(app.NoSourceSince) > 0:
 239             wikidata += '\n[[Category:Apps missing source code]]\n'
 240         if validapks == 0 and not app.Disabled:
 241             wikidata += '\n[[Category:Apps with no packages]]\n'
 242         if cantupdate and not app.Disabled:
 243             wikidata += "\n[[Category:Apps we cannot update]]\n"
 244         if buildfails and not app.Disabled:
 245             wikidata += "\n[[Category:Apps with failing builds]]\n"
 246         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 247             wikidata += '\n[[Category:Apps to Update]]\n'
 248         if app.Disabled:
 249             wikidata += '\n[[Category:Apps that are disabled]]\n'
 250         if app.UpdateCheckMode == 'None' and not app.Disabled:
 251             wikidata += '\n[[Category:Apps with no update check]]\n'
 252         for appcat in app.Categories:
 253             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 254
 255         # We can't have underscores in the page name, even if they're in
 256         # the package ID, because MediaWiki messes with them...
 257         pagename = appid.replace('_', ' ')
 258
 259         # Drop a trailing newline, because mediawiki is going to drop it anyway
 260         # and it we don't we'll think the page has changed when it hasn't...
 261         if wikidata.endswith('\n'):
 262             wikidata = wikidata[:-1]
 263
 264         generated_pages[pagename] = wikidata
 265
 266         # Make a redirect from the name to the ID too, unless there's
 267         # already an existing page with the name and it isn't a redirect.
 268         noclobber = False
 269         apppagename = app.Name.replace('_', ' ')
 270         apppagename = apppagename.replace('{', '')
 271         apppagename = apppagename.replace('}', ' ')
 272         apppagename = apppagename.replace(':', ' ')
 273         apppagename = apppagename.replace('[', ' ')
 274         apppagename = apppagename.replace(']', ' ')
 275         # Drop double spaces caused mostly by replacing ':' above
 276         apppagename = apppagename.replace('  ', ' ')
 277         for expagename in site.allpages(prefix=apppagename,
 278                                         filterredir='nonredirects',
 279                                         generator=False):
 280             if expagename == apppagename:
 281                 noclobber = True
 282         # Another reason not to make the redirect page is if the app name
 283         # is the same as it's ID, because that will overwrite the real page
 284         # with an redirect to itself! (Although it seems like an odd
 285         # scenario this happens a lot, e.g. where there is metadata but no
 286         # builds or binaries to extract a name from.
 287         if apppagename == pagename:
 288             noclobber = True
 289         if not noclobber:
 290             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 291
 292     for tcat, genp in [(wikicat, generated_pages),
 293                        (wikiredircat, generated_redirects)]:
 294         catpages = site.Pages['Category:' + tcat]
 295         existingpages = []
 296         for page in catpages:
 297             existingpages.append(page.name)
 298             if page.name in genp:
 299                 pagetxt = page.edit()
 300                 if pagetxt != genp[page.name]:
 301                     logging.debug("Updating modified page " + page.name)
 302                     page.save(genp[page.name], summary='Auto-updated')
 303                 else:
 304                     logging.debug("Page " + page.name + " is unchanged")
 305             else:
 306                 logging.warn("Deleting page " + page.name)
 307                 page.delete('No longer published')
 308         for pagename, text in genp.items():
 309             logging.debug("Checking " + pagename)
 310             if pagename not in existingpages:
 311                 logging.debug("Creating page " + pagename)
 312                 try:
 313                     newpage = site.Pages[pagename]
 314                     newpage.save(text, summary='Auto-created')
 315                 except Exception as e:
 316                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 317
 318     # Purge server cache to ensure counts are up to date
 319     site.pages['Repository Maintenance'].purge()
 320
 321
 322 def delete_disabled_builds(apps, apkcache, repodirs):
 323     """Delete disabled build outputs.
 324
 325     :param apps: list of all applications, as per metadata.read_metadata
 326     :param apkcache: current apk cache information
 327     :param repodirs: the repo directories to process
 328     """
 329     for appid, app in apps.items():
 330         for build in app['builds']:
 331             if not build.disable:
 332                 continue
 333             apkfilename = common.get_release_filename(app, build)
 334             iconfilename = "%s.%s.png" % (
 335                 appid,
 336                 build.versionCode)
 337             for repodir in repodirs:
 338                 files = [
 339                     os.path.join(repodir, apkfilename),
 340                     os.path.join(repodir, apkfilename + '.asc'),
 341                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 342                 ]
 343                 for density in all_screen_densities:
 344                     repo_dir = get_icon_dir(repodir, density)
 345                     files.append(os.path.join(repo_dir, iconfilename))
 346
 347                 for f in files:
 348                     if os.path.exists(f):
 349                         logging.info("Deleting disabled build output " + f)
 350                         os.remove(f)
 351             if apkfilename in apkcache:
 352                 del apkcache[apkfilename]
 353
 354
 355 def resize_icon(iconpath, density):
 356
 357     if not os.path.isfile(iconpath):
 358         return
 359
 360     fp = None
 361     try:
 362         fp = open(iconpath, 'rb')
 363         im = Image.open(fp)
 364         size = dpi_to_px(density)
 365
 366         if any(length > size for length in im.size):
 367             oldsize = im.size
 368             im.thumbnail((size, size), Image.ANTIALIAS)
 369             logging.debug("%s was too large at %s - new size is %s" % (
 370                 iconpath, oldsize, im.size))
 371             im.save(iconpath, "PNG")
 372
 373     except Exception as e:
 374         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 375
 376     finally:
 377         if fp:
 378             fp.close()
 379
 380
 381 def resize_all_icons(repodirs):
 382     """Resize all icons that exceed the max size
 383
 384     :param repodirs: the repo directories to process
 385     """
 386     for repodir in repodirs:
 387         for density in screen_densities:
 388             icon_dir = get_icon_dir(repodir, density)
 389             icon_glob = os.path.join(icon_dir, '*.png')
 390             for iconpath in glob.glob(icon_glob):
 391                 resize_icon(iconpath, density)
 392
 393
 394 def getsig(apkpath):
 395     """ Get the signing certificate of an apk. To get the same md5 has that
 396     Android gets, we encode the .RSA certificate in a specific format and pass
 397     it hex-encoded to the md5 digest algorithm.
 398
 399     :param apkpath: path to the apk
 400     :returns: A string containing the md5 of the signature of the apk or None
 401               if an error occurred.
 402     """
 403
 404     with zipfile.ZipFile(apkpath, 'r') as apk:
 405         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 406
 407         if len(certs) < 1:
 408             logging.error("Found no signing certificates on %s" % apkpath)
 409             return None
 410         if len(certs) > 1:
 411             logging.error("Found multiple signing certificates on %s" % apkpath)
 412             return None
 413
 414         cert = apk.read(certs[0])
 415
 416     cert_encoded = common.get_certificate(cert)
 417
 418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 419
 420
 421 def get_cache_file():
 422     return os.path.join('tmp', 'apkcache')
 423
 424
 425 def get_cache():
 426     """
 427     Gather information about all the apk files in the repo directory,
 428     using cached data if possible.
 429     :return: apkcache
 430     """
 431     apkcachefile = get_cache_file()
 432     if not options.clean and os.path.exists(apkcachefile):
 433         with open(apkcachefile, 'rb') as cf:
 434             apkcache = pickle.load(cf, encoding='utf-8')
 435         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
 436             apkcache = {}
 437     else:
 438         apkcache = {}
 439
 440     return apkcache
 441
 442
 443 def write_cache(apkcache):
 444     apkcachefile = get_cache_file()
 445     cache_path = os.path.dirname(apkcachefile)
 446     if not os.path.exists(cache_path):
 447         os.makedirs(cache_path)
 448     apkcache["METADATA_VERSION"] = METADATA_VERSION
 449     with open(apkcachefile, 'wb') as cf:
 450         pickle.dump(apkcache, cf)
 451
 452
 453 def get_icon_bytes(apkzip, iconsrc):
 454     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 455     try:
 456         return apkzip.read(iconsrc)
 457     except KeyError:
 458         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 459
 460
 461 def sha256sum(filename):
 462     '''Calculate the sha256 of the given file'''
 463     sha = hashlib.sha256()
 464     with open(filename, 'rb') as f:
 465         while True:
 466             t = f.read(16384)
 467             if len(t) == 0:
 468                 break
 469             sha.update(t)
 470     return sha.hexdigest()
 471
 472
 473 def has_known_vulnerability(filename):
 474     """checks for known vulnerabilities in the APK
 475
 476     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 477     version.  Google also enforces this:
 478     https://support.google.com/faqs/answer/6376725?hl=en
 479
 480     Checks whether there are more than one classes.dex or AndroidManifest.xml
 481     files, which is invalid and an essential part of the "Master Key" attack.
 482
 483     http://www.saurik.com/id/17
 484     """
 485
 486     # statically load this pattern
 487     if not hasattr(has_known_vulnerability, "pattern"):
 488         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 489
 490     files_in_apk = set()
 491     with zipfile.ZipFile(filename) as zf:
 492         for name in zf.namelist():
 493             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 494                 lib = zf.open(name)
 495                 while True:
 496                     chunk = lib.read(4096)
 497                     if chunk == b'':
 498                         break
 499                     m = has_known_vulnerability.pattern.search(chunk)
 500                     if m:
 501                         version = m.group(1).decode('ascii')
 502                         if version.startswith('1.0.1') and version[5] >= 'r' \
 503                            or version.startswith('1.0.2') and version[5] >= 'f':
 504                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 505                         else:
 506                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 507                             return True
 508                         break
 509             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 510                 if name in files_in_apk:
 511                     return True
 512                 files_in_apk.add(name)
 513
 514     return False
 515
 516
 517 def insert_obbs(repodir, apps, apks):
 518     """Scans the .obb files in a given repo directory and adds them to the
 519     relevant APK instances.  OBB files have versionCodes like APK
 520     files, and they are loosely associated.  If there is an OBB file
 521     present, then any APK with the same or higher versionCode will use
 522     that OBB file.  There are two OBB types: main and patch, each APK
 523     can only have only have one of each.
 524
 525     https://developer.android.com/google/play/expansion-files.html
 526
 527     :param repodir: repo directory to scan
 528     :param apps: list of current, valid apps
 529     :param apks: current information on all APKs
 530
 531     """
 532
 533     def obbWarnDelete(f, msg):
 534         logging.warning(msg + f)
 535         if options.delete_unknown:
 536             logging.error("Deleting unknown file: " + f)
 537             os.remove(f)
 538
 539     obbs = []
 540     java_Integer_MIN_VALUE = -pow(2, 31)
 541     currentPackageNames = apps.keys()
 542     for f in glob.glob(os.path.join(repodir, '*.obb')):
 543         obbfile = os.path.basename(f)
 544         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 545         chunks = obbfile.split('.')
 546         if chunks[0] != 'main' and chunks[0] != 'patch':
 547             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 548             continue
 549         if not re.match(r'^-?[0-9]+$', chunks[1]):
 550             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 551             continue
 552         versionCode = int(chunks[1])
 553         packagename = ".".join(chunks[2:-1])
 554
 555         highestVersionCode = java_Integer_MIN_VALUE
 556         if packagename not in currentPackageNames:
 557             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 558             continue
 559         for apk in apks:
 560             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 561                 highestVersionCode = apk['versionCode']
 562         if versionCode > highestVersionCode:
 563             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 564                           + ') than any APK: ')
 565             continue
 566         obbsha256 = sha256sum(f)
 567         obbs.append((packagename, versionCode, obbfile, obbsha256))
 568
 569     for apk in apks:
 570         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 571             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 572                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 573                     apk['obbMainFile'] = obbfile
 574                     apk['obbMainFileSha256'] = obbsha256
 575                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 576                     apk['obbPatchFile'] = obbfile
 577                     apk['obbPatchFileSha256'] = obbsha256
 578             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 579                 break
 580
 581
 582 def _get_localized_dict(app, locale):
 583     '''get the dict to add localized store metadata to'''
 584     if 'localized' not in app:
 585         app['localized'] = collections.OrderedDict()
 586     if locale not in app['localized']:
 587         app['localized'][locale] = collections.OrderedDict()
 588     return app['localized'][locale]
 589
 590
 591 def _set_localized_text_entry(app, locale, key, f):
 592     limit = config['char_limits'][key]
 593     localized = _get_localized_dict(app, locale)
 594     with open(f) as fp:
 595         text = fp.read()[:limit]
 596         if len(text) > 0:
 597             localized[key] = text
 598
 599
 600 def _set_author_entry(app, key, f):
 601     limit = config['char_limits']['author']
 602     with open(f) as fp:
 603         text = fp.read()[:limit]
 604         if len(text) > 0:
 605             app[key] = text
 606
 607
 608 def copy_triple_t_store_metadata(apps):
 609     """Include store metadata from the app's source repo
 610
 611     The Triple-T Gradle Play Publisher is a plugin that has a standard
 612     file layout for all of the metadata and graphics that the Google
 613     Play Store accepts.  Since F-Droid has the git repo, it can just
 614     pluck those files directly.  This method reads any text files into
 615     the app dict, then copies any graphics into the fdroid repo
 616     directory structure.
 617
 618     This needs to be run before insert_localized_app_metadata() so that
 619     the graphics files that are copied into the fdroid repo get
 620     properly indexed.
 621
 622     https://github.com/Triple-T/gradle-play-publisher#upload-images
 623     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 624
 625     """
 626
 627     if not os.path.isdir('build'):
 628         return  # nothing to do
 629
 630     for packageName, app in apps.items():
 631         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 632             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 633             for root, dirs, files in os.walk(d):
 634                 segments = root.split('/')
 635                 locale = segments[-2]
 636                 for f in files:
 637                     if f == 'fulldescription':
 638                         _set_localized_text_entry(app, locale, 'description',
 639                                                   os.path.join(root, f))
 640                         continue
 641                     elif f == 'shortdescription':
 642                         _set_localized_text_entry(app, locale, 'summary',
 643                                                   os.path.join(root, f))
 644                         continue
 645                     elif f == 'title':
 646                         _set_localized_text_entry(app, locale, 'name',
 647                                                   os.path.join(root, f))
 648                         continue
 649                     elif f == 'video':
 650                         _set_localized_text_entry(app, locale, 'video',
 651                                                   os.path.join(root, f))
 652                         continue
 653                     elif f == 'whatsnew':
 654                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 655                                                   os.path.join(root, f))
 656                         continue
 657                     elif f == 'contactEmail':
 658                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 659                         continue
 660                     elif f == 'contactPhone':
 661                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 662                         continue
 663                     elif f == 'contactWebsite':
 664                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 665                         continue
 666
 667                     base, extension = common.get_extension(f)
 668                     dirname = os.path.basename(root)
 669                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 670                         if segments[-2] == 'listing':
 671                             locale = segments[-3]
 672                         else:
 673                             locale = segments[-2]
 674                         destdir = os.path.join('repo', packageName, locale)
 675                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 676                         sourcefile = os.path.join(root, f)
 677                         destfile = os.path.join(destdir, dirname + '.' + extension)
 678                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 679                         shutil.copy(sourcefile, destfile)
 680
 681
 682 def insert_localized_app_metadata(apps):
 683     """scans standard locations for graphics and localized text
 684
 685     Scans for localized description files, store graphics, and
 686     screenshot PNG files in statically defined screenshots directory
 687     and adds them to the app metadata.  The screenshots and graphic
 688     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 689     and must be in the following layout:
 690     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 691
 692     repo/packageName/locale/featureGraphic.png
 693     repo/packageName/locale/phoneScreenshots/1.png
 694     repo/packageName/locale/phoneScreenshots/2.png
 695
 696     The changelog files must be text files named with the versionCode
 697     ending with ".txt" and must be in the following layout:
 698     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 699
 700     repo/packageName/locale/changelogs/12345.txt
 701
 702     This will scan the each app's source repo then the metadata/ dir
 703     for these standard locations of changelog files.  If it finds
 704     them, they will be added to the dict of all packages, with the
 705     versions in the metadata/ folder taking precendence over the what
 706     is in the app's source repo.
 707
 708     Where "packageName" is the app's packageName and "locale" is the locale
 709     of the graphics, e.g. what language they are in, using the IETF RFC5646
 710     format (en-US, fr-CA, es-MX, etc).
 711
 712     This will also scan the app's git for a fastlane folder, and the
 713     metadata/ folder and the apps' source repos for standard locations
 714     of graphic and screenshot files.  If it finds them, it will copy
 715     them into the repo.  The fastlane files follow this pattern:
 716     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 717
 718     """
 719
 720     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 721     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 722     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 723
 724     for srcd in sorted(sourcedirs):
 725         if not os.path.isdir(srcd):
 726             continue
 727         for root, dirs, files in os.walk(srcd):
 728             segments = root.split('/')
 729             packageName = segments[1]
 730             if packageName not in apps:
 731                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 732                 continue
 733             locale = segments[-1]
 734             destdir = os.path.join('repo', packageName, locale)
 735             for f in files:
 736                 if f in ('description.txt', 'full_description.txt'):
 737                     _set_localized_text_entry(apps[packageName], locale, 'description',
 738                                               os.path.join(root, f))
 739                     continue
 740                 elif f in ('summary.txt', 'short_description.txt'):
 741                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 742                                               os.path.join(root, f))
 743                     continue
 744                 elif f in ('name.txt', 'title.txt'):
 745                     _set_localized_text_entry(apps[packageName], locale, 'name',
 746                                               os.path.join(root, f))
 747                     continue
 748                 elif f == 'video.txt':
 749                     _set_localized_text_entry(apps[packageName], locale, 'video',
 750                                               os.path.join(root, f))
 751                     continue
 752                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 753                     locale = segments[-2]
 754                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 755                                               os.path.join(root, f))
 756                     continue
 757
 758                 base, extension = common.get_extension(f)
 759                 if locale == 'images':
 760                     locale = segments[-2]
 761                     destdir = os.path.join('repo', packageName, locale)
 762                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 763                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 764                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 765                     shutil.copy(os.path.join(root, f), destdir)
 766             for d in dirs:
 767                 if d in SCREENSHOT_DIRS:
 768                     for f in glob.glob(os.path.join(root, d, '*.*')):
 769                         _, extension = common.get_extension(f)
 770                         if extension in ALLOWED_EXTENSIONS:
 771                             screenshotdestdir = os.path.join(destdir, d)
 772                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 773                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 774                             shutil.copy(f, screenshotdestdir)
 775
 776     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 777     for d in repofiles:
 778         if not os.path.isdir(d):
 779             continue
 780         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 781             if not os.path.isfile(f):
 782                 continue
 783             segments = f.split('/')
 784             packageName = segments[1]
 785             locale = segments[2]
 786             screenshotdir = segments[3]
 787             filename = os.path.basename(f)
 788             base, extension = common.get_extension(filename)
 789
 790             if packageName not in apps:
 791                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 792                                 % (filename, packageName))
 793                 continue
 794             graphics = _get_localized_dict(apps[packageName], locale)
 795
 796             if extension not in ALLOWED_EXTENSIONS:
 797                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 798             elif base in GRAPHIC_NAMES:
 799                 # there can only be zero or one of these per locale
 800                 graphics[base] = filename
 801             elif screenshotdir in SCREENSHOT_DIRS:
 802                 # there can any number of these per locale
 803                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 804                 if screenshotdir not in graphics:
 805                     graphics[screenshotdir] = []
 806                 graphics[screenshotdir].append(filename)
 807             else:
 808                 logging.warning('Unsupported graphics file found: ' + f)
 809
 810
 811 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 812     """Scan a repo for all files with an extension except APK/OBB
 813
 814     :param apkcache: current cached info about all repo files
 815     :param repodir: repo directory to scan
 816     :param knownapks: list of all known files, as per metadata.read_metadata
 817     :param use_date_from_file: use date from file (instead of current date)
 818                                for newly added files
 819     """
 820
 821     cachechanged = False
 822     repo_files = []
 823     repodir = repodir.encode('utf-8')
 824     for name in os.listdir(repodir):
 825         file_extension = common.get_file_extension(name)
 826         if file_extension == 'apk' or file_extension == 'obb':
 827             continue
 828         filename = os.path.join(repodir, name)
 829         name_utf8 = name.decode('utf-8')
 830         if filename.endswith(b'_src.tar.gz'):
 831             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 832             continue
 833         if not common.is_repo_file(filename):
 834             continue
 835         stat = os.stat(filename)
 836         if stat.st_size == 0:
 837             raise FDroidException(filename + ' is zero size!')
 838
 839         shasum = sha256sum(filename)
 840         usecache = False
 841         if name in apkcache:
 842             repo_file = apkcache[name]
 843             # added time is cached as tuple but used here as datetime instance
 844             if 'added' in repo_file:
 845                 a = repo_file['added']
 846                 if isinstance(a, datetime):
 847                     repo_file['added'] = a
 848                 else:
 849                     repo_file['added'] = datetime(*a[:6])
 850             if repo_file.get('hash') == shasum:
 851                 logging.debug("Reading " + name_utf8 + " from cache")
 852                 usecache = True
 853             else:
 854                 logging.debug("Ignoring stale cache data for " + name)
 855
 856         if not usecache:
 857             logging.debug("Processing " + name_utf8)
 858             repo_file = collections.OrderedDict()
 859             repo_file['name'] = os.path.splitext(name_utf8)[0]
 860             # TODO rename apkname globally to something more generic
 861             repo_file['apkName'] = name_utf8
 862             repo_file['hash'] = shasum
 863             repo_file['hashType'] = 'sha256'
 864             repo_file['versionCode'] = 0
 865             repo_file['versionName'] = shasum
 866             # the static ID is the SHA256 unless it is set in the metadata
 867             repo_file['packageName'] = shasum
 868
 869             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 870             if m:
 871                 repo_file['packageName'] = m.group(1)
 872                 repo_file['versionCode'] = int(m.group(2))
 873             srcfilename = name + b'_src.tar.gz'
 874             if os.path.exists(os.path.join(repodir, srcfilename)):
 875                 repo_file['srcname'] = srcfilename.decode('utf-8')
 876             repo_file['size'] = stat.st_size
 877
 878             apkcache[name] = repo_file
 879             cachechanged = True
 880
 881         if use_date_from_file:
 882             timestamp = stat.st_ctime
 883             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 884         else:
 885             default_date_param = None
 886
 887         # Record in knownapks, getting the added date at the same time..
 888         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 889                                     default_date=default_date_param)
 890         if added:
 891             repo_file['added'] = added
 892
 893         repo_files.append(repo_file)
 894
 895     return repo_files, cachechanged
 896
 897
 898 def scan_apk_aapt(apk, apkfile):
 899     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 900     if p.returncode != 0:
 901         if options.delete_unknown:
 902             if os.path.exists(apkfile):
 903                 logging.error("Failed to get apk information, deleting " + apkfile)
 904                 os.remove(apkfile)
 905             else:
 906                 logging.error("Could not find {0} to remove it".format(apkfile))
 907         else:
 908             logging.error("Failed to get apk information, skipping " + apkfile)
 909         raise BuildException("Invalid APK")
 910     for line in p.output.splitlines():
 911         if line.startswith("package:"):
 912             try:
 913                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 914                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 915                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 916             except Exception as e:
 917                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
 918         elif line.startswith("application:"):
 919             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 920             # Keep path to non-dpi icon in case we need it
 921             match = re.match(APK_ICON_PAT_NODPI, line)
 922             if match:
 923                 apk['icons_src']['-1'] = match.group(1)
 924         elif line.startswith("launchable-activity:"):
 925             # Only use launchable-activity as fallback to application
 926             if not apk['name']:
 927                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 928             if '-1' not in apk['icons_src']:
 929                 match = re.match(APK_ICON_PAT_NODPI, line)
 930                 if match:
 931                     apk['icons_src']['-1'] = match.group(1)
 932         elif line.startswith("application-icon-"):
 933             match = re.match(APK_ICON_PAT, line)
 934             if match:
 935                 density = match.group(1)
 936                 path = match.group(2)
 937                 apk['icons_src'][density] = path
 938         elif line.startswith("sdkVersion:"):
 939             m = re.match(APK_SDK_VERSION_PAT, line)
 940             if m is None:
 941                 logging.error(line.replace('sdkVersion:', '')
 942                               + ' is not a valid minSdkVersion!')
 943             else:
 944                 apk['minSdkVersion'] = m.group(1)
 945                 # if target not set, default to min
 946                 if 'targetSdkVersion' not in apk:
 947                     apk['targetSdkVersion'] = m.group(1)
 948         elif line.startswith("targetSdkVersion:"):
 949             m = re.match(APK_SDK_VERSION_PAT, line)
 950             if m is None:
 951                 logging.error(line.replace('targetSdkVersion:', '')
 952                               + ' is not a valid targetSdkVersion!')
 953             else:
 954                 apk['targetSdkVersion'] = m.group(1)
 955         elif line.startswith("maxSdkVersion:"):
 956             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 957         elif line.startswith("native-code:"):
 958             apk['nativecode'] = []
 959             for arch in line[13:].split(' '):
 960                 apk['nativecode'].append(arch[1:-1])
 961         elif line.startswith('uses-permission:'):
 962             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 963             if perm_match['maxSdkVersion']:
 964                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 965             permission = UsesPermission(
 966                 perm_match['name'],
 967                 perm_match['maxSdkVersion']
 968             )
 969
 970             apk['uses-permission'].append(permission)
 971         elif line.startswith('uses-permission-sdk-23:'):
 972             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 973             if perm_match['maxSdkVersion']:
 974                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 975             permission_sdk_23 = UsesPermissionSdk23(
 976                 perm_match['name'],
 977                 perm_match['maxSdkVersion']
 978             )
 979
 980             apk['uses-permission-sdk-23'].append(permission_sdk_23)
 981
 982         elif line.startswith('uses-feature:'):
 983             feature = re.match(APK_FEATURE_PAT, line).group(1)
 984             # Filter out this, it's only added with the latest SDK tools and
 985             # causes problems for lots of apps.
 986             if feature != "android.hardware.screen.portrait" \
 987                     and feature != "android.hardware.screen.landscape":
 988                 if feature.startswith("android.feature."):
 989                     feature = feature[16:]
 990                 apk['features'].add(feature)
 991
 992
 993 def scan_apk_androguard(apk, apkfile):
 994     try:
 995         from androguard.core.bytecodes.apk import APK
 996         apkobject = APK(apkfile)
 997         if apkobject.is_valid_APK():
 998             arsc = apkobject.get_android_resources()
 999         else:
1000             if options.delete_unknown:
1001                 if os.path.exists(apkfile):
1002                     logging.error("Failed to get apk information, deleting " + apkfile)
1003                     os.remove(apkfile)
1004                 else:
1005                     logging.error("Could not find {0} to remove it".format(apkfile))
1006             else:
1007                 logging.error("Failed to get apk information, skipping " + apkfile)
1008             raise BuildException("Invaild APK")
1009     except ImportError:
1010         raise FDroidException("androguard library is not installed and aapt not present")
1011     except FileNotFoundError:
1012         logging.error("Could not open apk file for analysis")
1013         raise BuildException("Invalid APK")
1014
1015     apk['packageName'] = apkobject.get_package()
1016     apk['versionCode'] = int(apkobject.get_androidversion_code())
1017     apk['versionName'] = apkobject.get_androidversion_name()
1018     if apk['versionName'][0] == "@":
1019         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1020         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1021         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1022     apk['name'] = apkobject.get_app_name()
1023
1024     if apkobject.get_max_sdk_version() is not None:
1025         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1026     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1027     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1028
1029     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1030     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1031
1032     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1033
1034     for file in apkobject.get_files():
1035         d_re = density_re.match(file)
1036         if d_re:
1037             folder = d_re.group(1).split('-')
1038             if len(folder) > 1:
1039                 resolution = folder[1]
1040             else:
1041                 resolution = 'mdpi'
1042             density = screen_resolutions[resolution]
1043             apk['icons_src'][density] = d_re.group(0)
1044
1045     if apk['icons_src'].get('-1') is None:
1046         apk['icons_src']['-1'] = apk['icons_src']['160']
1047
1048     arch_re = re.compile("^lib/(.*)/.*$")
1049     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1050     if len(arch) >= 1:
1051         apk['nativecode'] = []
1052         apk['nativecode'].extend(sorted(list(arch)))
1053
1054     xml = apkobject.get_android_manifest_xml()
1055
1056     for item in xml.getElementsByTagName('uses-permission'):
1057         name = str(item.getAttribute("android:name"))
1058         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1059         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1060         permission = UsesPermission(
1061             name,
1062             maxSdkVersion
1063         )
1064         apk['uses-permission'].append(permission)
1065
1066     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1067         name = str(item.getAttribute("android:name"))
1068         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1069         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1070         permission_sdk_23 = UsesPermissionSdk23(
1071             name,
1072             maxSdkVersion
1073         )
1074         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1075
1076     for item in xml.getElementsByTagName('uses-feature'):
1077         feature = str(item.getAttribute("android:name"))
1078         if feature != "android.hardware.screen.portrait" \
1079                 and feature != "android.hardware.screen.landscape":
1080             if feature.startswith("android.feature."):
1081                 feature = feature[16:]
1082         apk['features'].append(feature)
1083
1084
1085 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1086              allow_disabled_algorithms=False, archive_bad_sig=False):
1087     """Scan the apk with the given filename in the given repo directory.
1088
1089     This also extracts the icons.
1090
1091     :param apkcache: current apk cache information
1092     :param apkfilename: the filename of the apk to scan
1093     :param repodir: repo directory to scan
1094     :param knownapks: known apks info
1095     :param use_date_from_apk: use date from APK (instead of current date)
1096                               for newly added APKs
1097     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1098                                       disabled algorithms in the signature (e.g. MD5)
1099     :param archive_bad_sig: move APKs with a bad signature to the archive
1100     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1101      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1102     """
1103
1104     if ' ' in apkfilename:
1105         if options.rename_apks:
1106             newfilename = apkfilename.replace(' ', '_')
1107             os.rename(os.path.join(repodir, apkfilename),
1108                       os.path.join(repodir, newfilename))
1109             apkfilename = newfilename
1110         else:
1111             logging.critical("Spaces in filenames are not allowed.")
1112             return True, None, False
1113
1114     apkfile = os.path.join(repodir, apkfilename)
1115     shasum = sha256sum(apkfile)
1116
1117     cachechanged = False
1118     usecache = False
1119     if apkfilename in apkcache:
1120         apk = apkcache[apkfilename]
1121         if apk.get('hash') == shasum:
1122             logging.debug("Reading " + apkfilename + " from cache")
1123             usecache = True
1124         else:
1125             logging.debug("Ignoring stale cache data for " + apkfilename)
1126
1127     if not usecache:
1128         logging.debug("Processing " + apkfilename)
1129         apk = {}
1130         apk['hash'] = shasum
1131         apk['hashType'] = 'sha256'
1132         apk['uses-permission'] = []
1133         apk['uses-permission-sdk-23'] = []
1134         apk['features'] = []
1135         apk['icons_src'] = {}
1136         apk['icons'] = {}
1137         apk['antiFeatures'] = set()
1138
1139         try:
1140             if SdkToolsPopen(['aapt', 'version'], output=False):
1141                 scan_apk_aapt(apk, apkfile)
1142             else:
1143                 scan_apk_androguard(apk, apkfile)
1144         except BuildException:
1145             return True, None, False
1146
1147         if 'minSdkVersion' not in apk:
1148             logging.warn("No SDK version information found in {0}".format(apkfile))
1149             apk['minSdkVersion'] = 1
1150
1151         # Check for debuggable apks...
1152         if common.isApkAndDebuggable(apkfile):
1153             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1154
1155         # Get the signature (or md5 of, to be precise)...
1156         logging.debug('Getting signature of {0}'.format(apkfile))
1157         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1158         if not apk['sig']:
1159             logging.critical("Failed to get apk signature")
1160             return True, None, False
1161
1162         if options.rename_apks:
1163             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1164             std_short_name = os.path.join(repodir, n)
1165             if apkfile != std_short_name:
1166                 if os.path.exists(std_short_name):
1167                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1168                     if apkfile != std_long_name:
1169                         if os.path.exists(std_long_name):
1170                             dupdir = os.path.join('duplicates', repodir)
1171                             if not os.path.isdir(dupdir):
1172                                 os.makedirs(dupdir, exist_ok=True)
1173                             dupfile = os.path.join('duplicates', std_long_name)
1174                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1175                             os.rename(apkfile, dupfile)
1176                             return True, None, False
1177                         else:
1178                             os.rename(apkfile, std_long_name)
1179                     apkfile = std_long_name
1180                 else:
1181                     os.rename(apkfile, std_short_name)
1182                     apkfile = std_short_name
1183                 apkfilename = apkfile[len(repodir) + 1:]
1184
1185         apk['apkName'] = apkfilename
1186         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1187         if os.path.exists(os.path.join(repodir, srcfilename)):
1188             apk['srcname'] = srcfilename
1189         apk['size'] = os.path.getsize(apkfile)
1190
1191         # verify the jar signature is correct, allow deprecated
1192         # algorithms only if the APK is in the archive.
1193         skipapk = False
1194         if not common.verify_apk_signature(apkfile):
1195             if repodir == 'archive' or allow_disabled_algorithms:
1196                 if common.verify_old_apk_signature(apkfile):
1197                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1198                 else:
1199                     skipapk = True
1200             else:
1201                 skipapk = True
1202
1203         if skipapk:
1204             if archive_bad_sig:
1205                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1206                 move_apk_between_sections(repodir, 'archive', apk)
1207             else:
1208                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1209             return True, None, False
1210
1211         if 'KnownVuln' not in apk['antiFeatures']:
1212             if has_known_vulnerability(apkfile):
1213                 apk['antiFeatures'].add('KnownVuln')
1214
1215         apkzip = zipfile.ZipFile(apkfile, 'r')
1216
1217         # if an APK has files newer than the system time, suggest updating
1218         # the system clock.  This is useful for offline systems, used for
1219         # signing, which do not have another source of clock sync info. It
1220         # has to be more than 24 hours newer because ZIP/APK files do not
1221         # store timezone info
1222         manifest = apkzip.getinfo('AndroidManifest.xml')
1223         if manifest.date_time[1] == 0:  # month can't be zero
1224             logging.debug('AndroidManifest.xml has no date')
1225         else:
1226             dt_obj = datetime(*manifest.date_time)
1227             checkdt = dt_obj - timedelta(1)
1228             if datetime.today() < checkdt:
1229                 logging.warn('System clock is older than manifest in: '
1230                              + apkfilename
1231                              + '\nSet clock to that time using:\n'
1232                              + 'sudo date -s "' + str(dt_obj) + '"')
1233
1234         iconfilename = "%s.%s.png" % (
1235             apk['packageName'],
1236             apk['versionCode'])
1237
1238         # Extract the icon file...
1239         empty_densities = []
1240         for density in screen_densities:
1241             if density not in apk['icons_src']:
1242                 empty_densities.append(density)
1243                 continue
1244             iconsrc = apk['icons_src'][density]
1245             icon_dir = get_icon_dir(repodir, density)
1246             icondest = os.path.join(icon_dir, iconfilename)
1247
1248             try:
1249                 with open(icondest, 'wb') as f:
1250                     f.write(get_icon_bytes(apkzip, iconsrc))
1251                 apk['icons'][density] = iconfilename
1252             except (zipfile.BadZipFile, ValueError, KeyError) as e:
1253                 logging.warning("Error retrieving icon file: %s" % (icondest))
1254                 del apk['icons_src'][density]
1255                 empty_densities.append(density)
1256
1257         if '-1' in apk['icons_src']:
1258             iconsrc = apk['icons_src']['-1']
1259             iconpath = os.path.join(
1260                 get_icon_dir(repodir, '0'), iconfilename)
1261             with open(iconpath, 'wb') as f:
1262                 f.write(get_icon_bytes(apkzip, iconsrc))
1263             try:
1264                 im = Image.open(iconpath)
1265                 dpi = px_to_dpi(im.size[0])
1266                 for density in screen_densities:
1267                     if density in apk['icons']:
1268                         break
1269                     if density == screen_densities[-1] or dpi >= int(density):
1270                         apk['icons'][density] = iconfilename
1271                         shutil.move(iconpath,
1272                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
1273                         empty_densities.remove(density)
1274                         break
1275             except Exception as e:
1276                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1277
1278         if apk['icons']:
1279             apk['icon'] = iconfilename
1280
1281         apkzip.close()
1282
1283         # First try resizing down to not lose quality
1284         last_density = None
1285         for density in screen_densities:
1286             if density not in empty_densities:
1287                 last_density = density
1288                 continue
1289             if last_density is None:
1290                 continue
1291             logging.debug("Density %s not available, resizing down from %s"
1292                           % (density, last_density))
1293
1294             last_iconpath = os.path.join(
1295                 get_icon_dir(repodir, last_density), iconfilename)
1296             iconpath = os.path.join(
1297                 get_icon_dir(repodir, density), iconfilename)
1298             fp = None
1299             try:
1300                 fp = open(last_iconpath, 'rb')
1301                 im = Image.open(fp)
1302
1303                 size = dpi_to_px(density)
1304
1305                 im.thumbnail((size, size), Image.ANTIALIAS)
1306                 im.save(iconpath, "PNG")
1307                 empty_densities.remove(density)
1308             except Exception as e:
1309                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1310             finally:
1311                 if fp:
1312                     fp.close()
1313
1314         # Then just copy from the highest resolution available
1315         last_density = None
1316         for density in reversed(screen_densities):
1317             if density not in empty_densities:
1318                 last_density = density
1319                 continue
1320             if last_density is None:
1321                 continue
1322             logging.debug("Density %s not available, copying from lower density %s"
1323                           % (density, last_density))
1324
1325             shutil.copyfile(
1326                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1327                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1328
1329             empty_densities.remove(density)
1330
1331         for density in screen_densities:
1332             icon_dir = get_icon_dir(repodir, density)
1333             icondest = os.path.join(icon_dir, iconfilename)
1334             resize_icon(icondest, density)
1335
1336         # Copy from icons-mdpi to icons since mdpi is the baseline density
1337         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1338         if os.path.isfile(baseline):
1339             apk['icons']['0'] = iconfilename
1340             shutil.copyfile(baseline,
1341                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1342
1343         if use_date_from_apk and manifest.date_time[1] != 0:
1344             default_date_param = datetime(*manifest.date_time)
1345         else:
1346             default_date_param = None
1347
1348         # Record in known apks, getting the added date at the same time..
1349         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1350                                     default_date=default_date_param)
1351         if added:
1352             apk['added'] = added
1353
1354         apkcache[apkfilename] = apk
1355         cachechanged = True
1356
1357     return False, apk, cachechanged
1358
1359
1360 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1361     """Scan the apks in the given repo directory.
1362
1363     This also extracts the icons.
1364
1365     :param apkcache: current apk cache information
1366     :param repodir: repo directory to scan
1367     :param knownapks: known apks info
1368     :param use_date_from_apk: use date from APK (instead of current date)
1369                               for newly added APKs
1370     :returns: (apks, cachechanged) where apks is a list of apk information,
1371               and cachechanged is True if the apkcache got changed.
1372     """
1373
1374     cachechanged = False
1375
1376     for icon_dir in get_all_icon_dirs(repodir):
1377         if os.path.exists(icon_dir):
1378             if options.clean:
1379                 shutil.rmtree(icon_dir)
1380                 os.makedirs(icon_dir)
1381         else:
1382             os.makedirs(icon_dir)
1383
1384     apks = []
1385     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1386         apkfilename = apkfile[len(repodir) + 1:]
1387         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1388         (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks,
1389                                              use_date_from_apk, ada, True)
1390         if skip:
1391             continue
1392         apks.append(apk)
1393
1394     return apks, cachechanged
1395
1396
1397 def apply_info_from_latest_apk(apps, apks):
1398     """
1399     Some information from the apks needs to be applied up to the application level.
1400     When doing this, we use the info from the most recent version's apk.
1401     We deal with figuring out when the app was added and last updated at the same time.
1402     """
1403     for appid, app in apps.items():
1404         bestver = UNSET_VERSION_CODE
1405         for apk in apks:
1406             if apk['packageName'] == appid:
1407                 if apk['versionCode'] > bestver:
1408                     bestver = apk['versionCode']
1409                     bestapk = apk
1410
1411                 if 'added' in apk:
1412                     if not app.added or apk['added'] < app.added:
1413                         app.added = apk['added']
1414                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1415                         app.lastUpdated = apk['added']
1416
1417         if not app.added:
1418             logging.debug("Don't know when " + appid + " was added")
1419         if not app.lastUpdated:
1420             logging.debug("Don't know when " + appid + " was last updated")
1421
1422         if bestver == UNSET_VERSION_CODE:
1423
1424             if app.Name is None:
1425                 app.Name = app.AutoName or appid
1426             app.icon = None
1427             logging.debug("Application " + appid + " has no packages")
1428         else:
1429             if app.Name is None:
1430                 app.Name = bestapk['name']
1431             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1432             if app.CurrentVersionCode is None:
1433                 app.CurrentVersionCode = str(bestver)
1434
1435
1436 def make_categories_txt(repodir, categories):
1437     '''Write a category list in the repo to allow quick access'''
1438     catdata = ''
1439     for cat in sorted(categories):
1440         catdata += cat + '\n'
1441     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1442         f.write(catdata)
1443
1444
1445 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1446
1447     def filter_apk_list_sorted(apk_list):
1448         res = []
1449         for apk in apk_list:
1450             if apk['packageName'] == appid:
1451                 res.append(apk)
1452
1453         # Sort the apk list by version code. First is highest/newest.
1454         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1455
1456     for appid, app in apps.items():
1457
1458         if app.ArchivePolicy:
1459             keepversions = int(app.ArchivePolicy[:-9])
1460         else:
1461             keepversions = defaultkeepversions
1462
1463         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1464                       .format(appid, len(apks), keepversions, len(archapks)))
1465
1466         current_app_apks = filter_apk_list_sorted(apks)
1467         if len(current_app_apks) > keepversions:
1468             # Move back the ones we don't want.
1469             for apk in current_app_apks[keepversions:]:
1470                 move_apk_between_sections(repodir, archivedir, apk)
1471                 archapks.append(apk)
1472                 apks.remove(apk)
1473
1474         current_app_archapks = filter_apk_list_sorted(archapks)
1475         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1476             kept = 0
1477             # Move forward the ones we want again, except DisableAlgorithm
1478             for apk in current_app_archapks:
1479                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1480                     move_apk_between_sections(archivedir, repodir, apk)
1481                     archapks.remove(apk)
1482                     apks.append(apk)
1483                     kept += 1
1484                 if kept == keepversions:
1485                     break
1486
1487
1488 def move_apk_between_sections(from_dir, to_dir, apk):
1489     """move an APK from repo to archive or vice versa"""
1490
1491     def _move_file(from_dir, to_dir, filename, ignore_missing):
1492         from_path = os.path.join(from_dir, filename)
1493         if ignore_missing and not os.path.exists(from_path):
1494             return
1495         to_path = os.path.join(to_dir, filename)
1496         shutil.move(from_path, to_path)
1497
1498     if from_dir == to_dir:
1499         return
1500
1501     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1502     _move_file(from_dir, to_dir, apk['apkName'], False)
1503     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1504     for density in all_screen_densities:
1505         from_icon_dir = get_icon_dir(from_dir, density)
1506         to_icon_dir = get_icon_dir(to_dir, density)
1507         if density not in apk['icons']:
1508             continue
1509         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1510     if 'srcname' in apk:
1511         _move_file(from_dir, to_dir, apk['srcname'], False)
1512
1513
1514 def add_apks_to_per_app_repos(repodir, apks):
1515     apks_per_app = dict()
1516     for apk in apks:
1517         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1518         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1519         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1520         apks_per_app[apk['packageName']] = apk
1521
1522         if not os.path.exists(apk['per_app_icons']):
1523             logging.info('Adding new repo for only ' + apk['packageName'])
1524             os.makedirs(apk['per_app_icons'])
1525
1526         apkpath = os.path.join(repodir, apk['apkName'])
1527         shutil.copy(apkpath, apk['per_app_repo'])
1528         apksigpath = apkpath + '.sig'
1529         if os.path.exists(apksigpath):
1530             shutil.copy(apksigpath, apk['per_app_repo'])
1531         apkascpath = apkpath + '.asc'
1532         if os.path.exists(apkascpath):
1533             shutil.copy(apkascpath, apk['per_app_repo'])
1534
1535
1536 config = None
1537 options = None
1538
1539
1540 def main():
1541
1542     global config, options
1543
1544     # Parse command line...
1545     parser = ArgumentParser()
1546     common.setup_global_opts(parser)
1547     parser.add_argument("--create-key", action="store_true", default=False,
1548                         help="Create a repo signing key in a keystore")
1549     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1550                         help="Create skeleton metadata files that are missing")
1551     parser.add_argument("--delete-unknown", action="store_true", default=False,
1552                         help="Delete APKs and/or OBBs without metadata from the repo")
1553     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1554                         help="Report on build data status")
1555     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1556                         help="Interactively ask about things that need updating.")
1557     parser.add_argument("-I", "--icons", action="store_true", default=False,
1558                         help="Resize all the icons exceeding the max pixel size and exit")
1559     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1560                         help="Specify editor to use in interactive mode. Default " +
1561                         "is /etc/alternatives/editor")
1562     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1563                         help="Update the wiki")
1564     parser.add_argument("--pretty", action="store_true", default=False,
1565                         help="Produce human-readable index.xml")
1566     parser.add_argument("--clean", action="store_true", default=False,
1567                         help="Clean update - don't uses caches, reprocess all apks")
1568     parser.add_argument("--nosign", action="store_true", default=False,
1569                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1570     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1571                         help="Use date from apk instead of current time for newly added apks")
1572     parser.add_argument("--rename-apks", action="store_true", default=False,
1573                         help="Rename APK files that do not match package.name_123.apk")
1574     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1575                         help="Include APKs that are signed with disabled algorithms like MD5")
1576     metadata.add_metadata_arguments(parser)
1577     options = parser.parse_args()
1578     metadata.warnings_action = options.W
1579
1580     config = common.read_config(options)
1581
1582     if not ('jarsigner' in config and 'keytool' in config):
1583         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1584
1585     repodirs = ['repo']
1586     if config['archive_older'] != 0:
1587         repodirs.append('archive')
1588         if not os.path.exists('archive'):
1589             os.mkdir('archive')
1590
1591     if options.icons:
1592         resize_all_icons(repodirs)
1593         sys.exit(0)
1594
1595     if options.rename_apks:
1596         options.clean = True
1597
1598     # check that icons exist now, rather than fail at the end of `fdroid update`
1599     for k in ['repo_icon', 'archive_icon']:
1600         if k in config:
1601             if not os.path.exists(config[k]):
1602                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1603                 sys.exit(1)
1604
1605     # if the user asks to create a keystore, do it now, reusing whatever it can
1606     if options.create_key:
1607         if os.path.exists(config['keystore']):
1608             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1609             logging.critical("\t'" + config['keystore'] + "'")
1610             sys.exit(1)
1611
1612         if 'repo_keyalias' not in config:
1613             config['repo_keyalias'] = socket.getfqdn()
1614             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1615         if 'keydname' not in config:
1616             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1617             common.write_to_config(config, 'keydname', config['keydname'])
1618         if 'keystore' not in config:
1619             config['keystore'] = common.default_config['keystore']
1620             common.write_to_config(config, 'keystore', config['keystore'])
1621
1622         password = common.genpassword()
1623         if 'keystorepass' not in config:
1624             config['keystorepass'] = password
1625             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1626         if 'keypass' not in config:
1627             config['keypass'] = password
1628             common.write_to_config(config, 'keypass', config['keypass'])
1629         common.genkeystore(config)
1630
1631     # Get all apps...
1632     apps = metadata.read_metadata()
1633
1634     # Generate a list of categories...
1635     categories = set()
1636     for app in apps.values():
1637         categories.update(app.Categories)
1638
1639     # Read known apks data (will be updated and written back when we've finished)
1640     knownapks = common.KnownApks()
1641
1642     # Get APK cache
1643     apkcache = get_cache()
1644
1645     # Delete builds for disabled apps
1646     delete_disabled_builds(apps, apkcache, repodirs)
1647
1648     # Scan all apks in the main repo
1649     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1650
1651     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1652                                            options.use_date_from_apk)
1653     cachechanged = cachechanged or fcachechanged
1654     apks += files
1655     # Generate warnings for apk's with no metadata (or create skeleton
1656     # metadata files, if requested on the command line)
1657     newmetadata = False
1658     for apk in apks:
1659         if apk['packageName'] not in apps:
1660             if options.create_metadata:
1661                 if 'name' not in apk:
1662                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1663                     continue
1664                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1665                 f.write("License:Unknown\n")
1666                 f.write("Web Site:\n")
1667                 f.write("Source Code:\n")
1668                 f.write("Issue Tracker:\n")
1669                 f.write("Changelog:\n")
1670                 f.write("Summary:" + apk['name'] + "\n")
1671                 f.write("Description:\n")
1672                 f.write(apk['name'] + "\n")
1673                 f.write(".\n")
1674                 f.write("Name:" + apk['name'] + "\n")
1675                 f.close()
1676                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1677                 newmetadata = True
1678             else:
1679                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1680                 if options.delete_unknown:
1681                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1682                     rmf = os.path.join(repodirs[0], apk['apkName'])
1683                     if not os.path.exists(rmf):
1684                         logging.error("Could not find {0} to remove it".format(rmf))
1685                     else:
1686                         os.remove(rmf)
1687                 else:
1688                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1689
1690     # update the metadata with the newly created ones included
1691     if newmetadata:
1692         apps = metadata.read_metadata()
1693
1694     copy_triple_t_store_metadata(apps)
1695     insert_obbs(repodirs[0], apps, apks)
1696     insert_localized_app_metadata(apps)
1697
1698     # Scan the archive repo for apks as well
1699     if len(repodirs) > 1:
1700         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1701         if cc:
1702             cachechanged = True
1703     else:
1704         archapks = []
1705
1706     # Apply information from latest apks to the application and update dates
1707     apply_info_from_latest_apk(apps, apks + archapks)
1708
1709     # Sort the app list by name, then the web site doesn't have to by default.
1710     # (we had to wait until we'd scanned the apks to do this, because mostly the
1711     # name comes from there!)
1712     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1713
1714     # APKs are placed into multiple repos based on the app package, providing
1715     # per-app subscription feeds for nightly builds and things like it
1716     if config['per_app_repos']:
1717         add_apks_to_per_app_repos(repodirs[0], apks)
1718         for appid, app in apps.items():
1719             repodir = os.path.join(appid, 'fdroid', 'repo')
1720             appdict = dict()
1721             appdict[appid] = app
1722             if os.path.isdir(repodir):
1723                 index.make(appdict, [appid], apks, repodir, False)
1724             else:
1725                 logging.info('Skipping index generation for ' + appid)
1726         return
1727
1728     if len(repodirs) > 1:
1729         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1730
1731     # Make the index for the main repo...
1732     index.make(apps, sortedids, apks, repodirs[0], False)
1733     make_categories_txt(repodirs[0], categories)
1734
1735     # If there's an archive repo,  make the index for it. We already scanned it
1736     # earlier on.
1737     if len(repodirs) > 1:
1738         index.make(apps, sortedids, archapks, repodirs[1], True)
1739
1740     git_remote = config.get('binary_transparency_remote')
1741     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1742         from . import btlog
1743         btlog.make_binary_transparency_log(repodirs)
1744
1745     if config['update_stats']:
1746         # Update known apks info...
1747         knownapks.writeifchanged()
1748
1749         # Generate latest apps data for widget
1750         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1751             data = ''
1752             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1753                 for line in f:
1754                     appid = line.rstrip()
1755                     data += appid + "\t"
1756                     app = apps[appid]
1757                     data += app.Name + "\t"
1758                     if app.icon is not None:
1759                         data += app.icon + "\t"
1760                     data += app.License + "\n"
1761             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1762                 f.write(data)
1763
1764     if cachechanged:
1765         write_cache(apkcache)
1766
1767     # Update the wiki...
1768     if options.wiki:
1769         update_wiki(apps, sortedids, apks + archapks)
1770
1771     logging.info("Finished.")
1772
1773
1774 if __name__ == "__main__":
1775     main()