fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import common
  41 from . import index
  42 from . import metadata
  43 from .common import SdkToolsPopen
  44 from .exception import BuildException, FDroidException
  45
  46 METADATA_VERSION = 18
  47
  48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  49 UNSET_VERSION_CODE = -0x100000000
  50
  51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85
  86 def dpi_to_px(density):
  87     return (int(density) * 48) / 160
  88
  89
  90 def px_to_dpi(px):
  91     return (int(px) * 160) / 48
  92
  93
  94 def get_icon_dir(repodir, density):
  95     if density == '0':
  96         return os.path.join(repodir, "icons")
  97     return os.path.join(repodir, "icons-%s" % density)
  98
  99
 100 def get_icon_dirs(repodir):
 101     for density in screen_densities:
 102         yield get_icon_dir(repodir, density)
 103
 104
 105 def get_all_icon_dirs(repodir):
 106     for density in all_screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def update_wiki(apps, sortedids, apks):
 111     """Update the wiki
 112
 113     :param apps: fully populated list of all applications
 114     :param apks: all apks, except...
 115     """
 116     logging.info("Updating wiki")
 117     wikicat = 'Apps'
 118     wikiredircat = 'App Redirects'
 119     import mwclient
 120     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 121                          path=config['wiki_path'])
 122     site.login(config['wiki_user'], config['wiki_password'])
 123     generated_pages = {}
 124     generated_redirects = {}
 125
 126     for appid in sortedids:
 127         app = metadata.App(apps[appid])
 128
 129         wikidata = ''
 130         if app.Disabled:
 131             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 132         if app.AntiFeatures:
 133             for af in app.AntiFeatures:
 134                 wikidata += '{{AntiFeature|' + af + '}}\n'
 135         if app.RequiresRoot:
 136             requiresroot = 'Yes'
 137         else:
 138             requiresroot = 'No'
 139         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 140             appid,
 141             app.Name,
 142             app.added.strftime('%Y-%m-%d') if app.added else '',
 143             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 144             app.SourceCode,
 145             app.IssueTracker,
 146             app.WebSite,
 147             app.Changelog,
 148             app.Donate,
 149             app.FlattrID,
 150             app.Bitcoin,
 151             app.Litecoin,
 152             app.License,
 153             requiresroot,
 154             app.AuthorName,
 155             app.AuthorEmail)
 156
 157         if app.Provides:
 158             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 159
 160         wikidata += app.Summary
 161         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 162
 163         wikidata += "=Description=\n"
 164         wikidata += metadata.description_wiki(app.Description) + "\n"
 165
 166         wikidata += "=Maintainer Notes=\n"
 167         if app.MaintainerNotes:
 168             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 169         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 170
 171         # Get a list of all packages for this application...
 172         apklist = []
 173         gotcurrentver = False
 174         cantupdate = False
 175         buildfails = False
 176         for apk in apks:
 177             if apk['packageName'] == appid:
 178                 if str(apk['versionCode']) == app.CurrentVersionCode:
 179                     gotcurrentver = True
 180                 apklist.append(apk)
 181         # Include ones we can't build, as a special case...
 182         for build in app.builds:
 183             if build.disable:
 184                 if build.versionCode == app.CurrentVersionCode:
 185                     cantupdate = True
 186                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 187                 apklist.append({'versionCode': int(build.versionCode),
 188                                 'versionName': build.versionName,
 189                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 190                                 })
 191             else:
 192                 builtit = False
 193                 for apk in apklist:
 194                     if apk['versionCode'] == int(build.versionCode):
 195                         builtit = True
 196                         break
 197                 if not builtit:
 198                     buildfails = True
 199                     apklist.append({'versionCode': int(build.versionCode),
 200                                     'versionName': build.versionName,
 201                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 202                                     })
 203         if app.CurrentVersionCode == '0':
 204             cantupdate = True
 205         # Sort with most recent first...
 206         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 207
 208         wikidata += "=Versions=\n"
 209         if len(apklist) == 0:
 210             wikidata += "We currently have no versions of this app available."
 211         elif not gotcurrentver:
 212             wikidata += "We don't have the current version of this app."
 213         else:
 214             wikidata += "We have the current version of this app."
 215         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 216         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 217         if len(app.NoSourceSince) > 0:
 218             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 219         if len(app.CurrentVersion) > 0:
 220             wikidata += "The current (recommended) version is " + app.CurrentVersion
 221             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 222         validapks = 0
 223         for apk in apklist:
 224             wikidata += "==" + apk['versionName'] + "==\n"
 225
 226             if 'buildproblem' in apk:
 227                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 228             else:
 229                 validapks += 1
 230                 wikidata += "This version is built and signed by "
 231                 if 'srcname' in apk:
 232                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 233                 else:
 234                     wikidata += "the original developer.\n\n"
 235             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 236
 237         wikidata += '\n[[Category:' + wikicat + ']]\n'
 238         if len(app.NoSourceSince) > 0:
 239             wikidata += '\n[[Category:Apps missing source code]]\n'
 240         if validapks == 0 and not app.Disabled:
 241             wikidata += '\n[[Category:Apps with no packages]]\n'
 242         if cantupdate and not app.Disabled:
 243             wikidata += "\n[[Category:Apps we cannot update]]\n"
 244         if buildfails and not app.Disabled:
 245             wikidata += "\n[[Category:Apps with failing builds]]\n"
 246         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 247             wikidata += '\n[[Category:Apps to Update]]\n'
 248         if app.Disabled:
 249             wikidata += '\n[[Category:Apps that are disabled]]\n'
 250         if app.UpdateCheckMode == 'None' and not app.Disabled:
 251             wikidata += '\n[[Category:Apps with no update check]]\n'
 252         for appcat in app.Categories:
 253             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 254
 255         # We can't have underscores in the page name, even if they're in
 256         # the package ID, because MediaWiki messes with them...
 257         pagename = appid.replace('_', ' ')
 258
 259         # Drop a trailing newline, because mediawiki is going to drop it anyway
 260         # and it we don't we'll think the page has changed when it hasn't...
 261         if wikidata.endswith('\n'):
 262             wikidata = wikidata[:-1]
 263
 264         generated_pages[pagename] = wikidata
 265
 266         # Make a redirect from the name to the ID too, unless there's
 267         # already an existing page with the name and it isn't a redirect.
 268         noclobber = False
 269         apppagename = app.Name.replace('_', ' ')
 270         apppagename = apppagename.replace('{', '')
 271         apppagename = apppagename.replace('}', ' ')
 272         apppagename = apppagename.replace(':', ' ')
 273         apppagename = apppagename.replace('[', ' ')
 274         apppagename = apppagename.replace(']', ' ')
 275         # Drop double spaces caused mostly by replacing ':' above
 276         apppagename = apppagename.replace('  ', ' ')
 277         for expagename in site.allpages(prefix=apppagename,
 278                                         filterredir='nonredirects',
 279                                         generator=False):
 280             if expagename == apppagename:
 281                 noclobber = True
 282         # Another reason not to make the redirect page is if the app name
 283         # is the same as it's ID, because that will overwrite the real page
 284         # with an redirect to itself! (Although it seems like an odd
 285         # scenario this happens a lot, e.g. where there is metadata but no
 286         # builds or binaries to extract a name from.
 287         if apppagename == pagename:
 288             noclobber = True
 289         if not noclobber:
 290             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 291
 292     for tcat, genp in [(wikicat, generated_pages),
 293                        (wikiredircat, generated_redirects)]:
 294         catpages = site.Pages['Category:' + tcat]
 295         existingpages = []
 296         for page in catpages:
 297             existingpages.append(page.name)
 298             if page.name in genp:
 299                 pagetxt = page.edit()
 300                 if pagetxt != genp[page.name]:
 301                     logging.debug("Updating modified page " + page.name)
 302                     page.save(genp[page.name], summary='Auto-updated')
 303                 else:
 304                     logging.debug("Page " + page.name + " is unchanged")
 305             else:
 306                 logging.warn("Deleting page " + page.name)
 307                 page.delete('No longer published')
 308         for pagename, text in genp.items():
 309             logging.debug("Checking " + pagename)
 310             if pagename not in existingpages:
 311                 logging.debug("Creating page " + pagename)
 312                 try:
 313                     newpage = site.Pages[pagename]
 314                     newpage.save(text, summary='Auto-created')
 315                 except Exception as e:
 316                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 317
 318     # Purge server cache to ensure counts are up to date
 319     site.pages['Repository Maintenance'].purge()
 320
 321
 322 def delete_disabled_builds(apps, apkcache, repodirs):
 323     """Delete disabled build outputs.
 324
 325     :param apps: list of all applications, as per metadata.read_metadata
 326     :param apkcache: current apk cache information
 327     :param repodirs: the repo directories to process
 328     """
 329     for appid, app in apps.items():
 330         for build in app['builds']:
 331             if not build.disable:
 332                 continue
 333             apkfilename = common.get_release_filename(app, build)
 334             iconfilename = "%s.%s.png" % (
 335                 appid,
 336                 build.versionCode)
 337             for repodir in repodirs:
 338                 files = [
 339                     os.path.join(repodir, apkfilename),
 340                     os.path.join(repodir, apkfilename + '.asc'),
 341                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 342                 ]
 343                 for density in all_screen_densities:
 344                     repo_dir = get_icon_dir(repodir, density)
 345                     files.append(os.path.join(repo_dir, iconfilename))
 346
 347                 for f in files:
 348                     if os.path.exists(f):
 349                         logging.info("Deleting disabled build output " + f)
 350                         os.remove(f)
 351             if apkfilename in apkcache:
 352                 del apkcache[apkfilename]
 353
 354
 355 def resize_icon(iconpath, density):
 356
 357     if not os.path.isfile(iconpath):
 358         return
 359
 360     fp = None
 361     try:
 362         fp = open(iconpath, 'rb')
 363         im = Image.open(fp)
 364         size = dpi_to_px(density)
 365
 366         if any(length > size for length in im.size):
 367             oldsize = im.size
 368             im.thumbnail((size, size), Image.ANTIALIAS)
 369             logging.debug("%s was too large at %s - new size is %s" % (
 370                 iconpath, oldsize, im.size))
 371             im.save(iconpath, "PNG")
 372
 373     except Exception as e:
 374         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 375
 376     finally:
 377         if fp:
 378             fp.close()
 379
 380
 381 def resize_all_icons(repodirs):
 382     """Resize all icons that exceed the max size
 383
 384     :param repodirs: the repo directories to process
 385     """
 386     for repodir in repodirs:
 387         for density in screen_densities:
 388             icon_dir = get_icon_dir(repodir, density)
 389             icon_glob = os.path.join(icon_dir, '*.png')
 390             for iconpath in glob.glob(icon_glob):
 391                 resize_icon(iconpath, density)
 392
 393
 394 def getsig(apkpath):
 395     """ Get the signing certificate of an apk. To get the same md5 has that
 396     Android gets, we encode the .RSA certificate in a specific format and pass
 397     it hex-encoded to the md5 digest algorithm.
 398
 399     :param apkpath: path to the apk
 400     :returns: A string containing the md5 of the signature of the apk or None
 401               if an error occurred.
 402     """
 403
 404     with zipfile.ZipFile(apkpath, 'r') as apk:
 405         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 406
 407         if len(certs) < 1:
 408             logging.error("Found no signing certificates on %s" % apkpath)
 409             return None
 410         if len(certs) > 1:
 411             logging.error("Found multiple signing certificates on %s" % apkpath)
 412             return None
 413
 414         cert = apk.read(certs[0])
 415
 416     cert_encoded = common.get_certificate(cert)
 417
 418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 419
 420
 421 def get_cache_file():
 422     return os.path.join('tmp', 'apkcache')
 423
 424
 425 def get_cache():
 426     """Get the cached dict of the APK index
 427
 428     Gather information about all the apk files in the repo directory,
 429     using cached data if possible. Some of the index operations take a
 430     long time, like calculating the SHA-256 and verifying the APK
 431     signature.
 432
 433     The cache is invalidated if the metadata version is different, or
 434     the 'allow_disabled_algorithms' config/option is different.  In
 435     those cases, there is no easy way to know what has changed from
 436     the cache, so just rerun the whole thing.
 437
 438     :return: apkcache
 439
 440     """
 441     apkcachefile = get_cache_file()
 442     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 443     if not options.clean and os.path.exists(apkcachefile):
 444         with open(apkcachefile, 'rb') as cf:
 445             apkcache = pickle.load(cf, encoding='utf-8')
 446         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 447            or apkcache.get('allow_disabled_algorithms') != ada:
 448             apkcache = {}
 449     else:
 450         apkcache = {}
 451
 452     apkcache["METADATA_VERSION"] = METADATA_VERSION
 453     apkcache['allow_disabled_algorithms'] = ada
 454
 455     return apkcache
 456
 457
 458 def write_cache(apkcache):
 459     apkcachefile = get_cache_file()
 460     cache_path = os.path.dirname(apkcachefile)
 461     if not os.path.exists(cache_path):
 462         os.makedirs(cache_path)
 463     with open(apkcachefile, 'wb') as cf:
 464         pickle.dump(apkcache, cf)
 465
 466
 467 def get_icon_bytes(apkzip, iconsrc):
 468     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 469     try:
 470         return apkzip.read(iconsrc)
 471     except KeyError:
 472         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 473
 474
 475 def sha256sum(filename):
 476     '''Calculate the sha256 of the given file'''
 477     sha = hashlib.sha256()
 478     with open(filename, 'rb') as f:
 479         while True:
 480             t = f.read(16384)
 481             if len(t) == 0:
 482                 break
 483             sha.update(t)
 484     return sha.hexdigest()
 485
 486
 487 def has_known_vulnerability(filename):
 488     """checks for known vulnerabilities in the APK
 489
 490     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 491     version.  Google also enforces this:
 492     https://support.google.com/faqs/answer/6376725?hl=en
 493
 494     Checks whether there are more than one classes.dex or AndroidManifest.xml
 495     files, which is invalid and an essential part of the "Master Key" attack.
 496
 497     http://www.saurik.com/id/17
 498     """
 499
 500     # statically load this pattern
 501     if not hasattr(has_known_vulnerability, "pattern"):
 502         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 503
 504     files_in_apk = set()
 505     with zipfile.ZipFile(filename) as zf:
 506         for name in zf.namelist():
 507             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 508                 lib = zf.open(name)
 509                 while True:
 510                     chunk = lib.read(4096)
 511                     if chunk == b'':
 512                         break
 513                     m = has_known_vulnerability.pattern.search(chunk)
 514                     if m:
 515                         version = m.group(1).decode('ascii')
 516                         if version.startswith('1.0.1') and version[5] >= 'r' \
 517                            or version.startswith('1.0.2') and version[5] >= 'f':
 518                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 519                         else:
 520                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 521                             return True
 522                         break
 523             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 524                 if name in files_in_apk:
 525                     return True
 526                 files_in_apk.add(name)
 527
 528     return False
 529
 530
 531 def insert_obbs(repodir, apps, apks):
 532     """Scans the .obb files in a given repo directory and adds them to the
 533     relevant APK instances.  OBB files have versionCodes like APK
 534     files, and they are loosely associated.  If there is an OBB file
 535     present, then any APK with the same or higher versionCode will use
 536     that OBB file.  There are two OBB types: main and patch, each APK
 537     can only have only have one of each.
 538
 539     https://developer.android.com/google/play/expansion-files.html
 540
 541     :param repodir: repo directory to scan
 542     :param apps: list of current, valid apps
 543     :param apks: current information on all APKs
 544
 545     """
 546
 547     def obbWarnDelete(f, msg):
 548         logging.warning(msg + f)
 549         if options.delete_unknown:
 550             logging.error("Deleting unknown file: " + f)
 551             os.remove(f)
 552
 553     obbs = []
 554     java_Integer_MIN_VALUE = -pow(2, 31)
 555     currentPackageNames = apps.keys()
 556     for f in glob.glob(os.path.join(repodir, '*.obb')):
 557         obbfile = os.path.basename(f)
 558         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 559         chunks = obbfile.split('.')
 560         if chunks[0] != 'main' and chunks[0] != 'patch':
 561             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 562             continue
 563         if not re.match(r'^-?[0-9]+$', chunks[1]):
 564             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 565             continue
 566         versionCode = int(chunks[1])
 567         packagename = ".".join(chunks[2:-1])
 568
 569         highestVersionCode = java_Integer_MIN_VALUE
 570         if packagename not in currentPackageNames:
 571             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 572             continue
 573         for apk in apks:
 574             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 575                 highestVersionCode = apk['versionCode']
 576         if versionCode > highestVersionCode:
 577             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 578                           + ') than any APK: ')
 579             continue
 580         obbsha256 = sha256sum(f)
 581         obbs.append((packagename, versionCode, obbfile, obbsha256))
 582
 583     for apk in apks:
 584         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 585             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 586                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 587                     apk['obbMainFile'] = obbfile
 588                     apk['obbMainFileSha256'] = obbsha256
 589                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 590                     apk['obbPatchFile'] = obbfile
 591                     apk['obbPatchFileSha256'] = obbsha256
 592             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 593                 break
 594
 595
 596 def _get_localized_dict(app, locale):
 597     '''get the dict to add localized store metadata to'''
 598     if 'localized' not in app:
 599         app['localized'] = collections.OrderedDict()
 600     if locale not in app['localized']:
 601         app['localized'][locale] = collections.OrderedDict()
 602     return app['localized'][locale]
 603
 604
 605 def _set_localized_text_entry(app, locale, key, f):
 606     limit = config['char_limits'][key]
 607     localized = _get_localized_dict(app, locale)
 608     with open(f) as fp:
 609         text = fp.read()[:limit]
 610         if len(text) > 0:
 611             localized[key] = text
 612
 613
 614 def _set_author_entry(app, key, f):
 615     limit = config['char_limits']['author']
 616     with open(f) as fp:
 617         text = fp.read()[:limit]
 618         if len(text) > 0:
 619             app[key] = text
 620
 621
 622 def copy_triple_t_store_metadata(apps):
 623     """Include store metadata from the app's source repo
 624
 625     The Triple-T Gradle Play Publisher is a plugin that has a standard
 626     file layout for all of the metadata and graphics that the Google
 627     Play Store accepts.  Since F-Droid has the git repo, it can just
 628     pluck those files directly.  This method reads any text files into
 629     the app dict, then copies any graphics into the fdroid repo
 630     directory structure.
 631
 632     This needs to be run before insert_localized_app_metadata() so that
 633     the graphics files that are copied into the fdroid repo get
 634     properly indexed.
 635
 636     https://github.com/Triple-T/gradle-play-publisher#upload-images
 637     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 638
 639     """
 640
 641     if not os.path.isdir('build'):
 642         return  # nothing to do
 643
 644     for packageName, app in apps.items():
 645         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 646             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 647             for root, dirs, files in os.walk(d):
 648                 segments = root.split('/')
 649                 locale = segments[-2]
 650                 for f in files:
 651                     if f == 'fulldescription':
 652                         _set_localized_text_entry(app, locale, 'description',
 653                                                   os.path.join(root, f))
 654                         continue
 655                     elif f == 'shortdescription':
 656                         _set_localized_text_entry(app, locale, 'summary',
 657                                                   os.path.join(root, f))
 658                         continue
 659                     elif f == 'title':
 660                         _set_localized_text_entry(app, locale, 'name',
 661                                                   os.path.join(root, f))
 662                         continue
 663                     elif f == 'video':
 664                         _set_localized_text_entry(app, locale, 'video',
 665                                                   os.path.join(root, f))
 666                         continue
 667                     elif f == 'whatsnew':
 668                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 669                                                   os.path.join(root, f))
 670                         continue
 671                     elif f == 'contactEmail':
 672                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 673                         continue
 674                     elif f == 'contactPhone':
 675                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 676                         continue
 677                     elif f == 'contactWebsite':
 678                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 679                         continue
 680
 681                     base, extension = common.get_extension(f)
 682                     dirname = os.path.basename(root)
 683                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 684                         if segments[-2] == 'listing':
 685                             locale = segments[-3]
 686                         else:
 687                             locale = segments[-2]
 688                         destdir = os.path.join('repo', packageName, locale)
 689                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 690                         sourcefile = os.path.join(root, f)
 691                         destfile = os.path.join(destdir, dirname + '.' + extension)
 692                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 693                         shutil.copy(sourcefile, destfile)
 694
 695
 696 def insert_localized_app_metadata(apps):
 697     """scans standard locations for graphics and localized text
 698
 699     Scans for localized description files, store graphics, and
 700     screenshot PNG files in statically defined screenshots directory
 701     and adds them to the app metadata.  The screenshots and graphic
 702     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 703     and must be in the following layout:
 704     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 705
 706     repo/packageName/locale/featureGraphic.png
 707     repo/packageName/locale/phoneScreenshots/1.png
 708     repo/packageName/locale/phoneScreenshots/2.png
 709
 710     The changelog files must be text files named with the versionCode
 711     ending with ".txt" and must be in the following layout:
 712     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 713
 714     repo/packageName/locale/changelogs/12345.txt
 715
 716     This will scan the each app's source repo then the metadata/ dir
 717     for these standard locations of changelog files.  If it finds
 718     them, they will be added to the dict of all packages, with the
 719     versions in the metadata/ folder taking precendence over the what
 720     is in the app's source repo.
 721
 722     Where "packageName" is the app's packageName and "locale" is the locale
 723     of the graphics, e.g. what language they are in, using the IETF RFC5646
 724     format (en-US, fr-CA, es-MX, etc).
 725
 726     This will also scan the app's git for a fastlane folder, and the
 727     metadata/ folder and the apps' source repos for standard locations
 728     of graphic and screenshot files.  If it finds them, it will copy
 729     them into the repo.  The fastlane files follow this pattern:
 730     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 731
 732     """
 733
 734     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 735     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 736     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 737
 738     for srcd in sorted(sourcedirs):
 739         if not os.path.isdir(srcd):
 740             continue
 741         for root, dirs, files in os.walk(srcd):
 742             segments = root.split('/')
 743             packageName = segments[1]
 744             if packageName not in apps:
 745                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 746                 continue
 747             locale = segments[-1]
 748             destdir = os.path.join('repo', packageName, locale)
 749             for f in files:
 750                 if f in ('description.txt', 'full_description.txt'):
 751                     _set_localized_text_entry(apps[packageName], locale, 'description',
 752                                               os.path.join(root, f))
 753                     continue
 754                 elif f in ('summary.txt', 'short_description.txt'):
 755                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 756                                               os.path.join(root, f))
 757                     continue
 758                 elif f in ('name.txt', 'title.txt'):
 759                     _set_localized_text_entry(apps[packageName], locale, 'name',
 760                                               os.path.join(root, f))
 761                     continue
 762                 elif f == 'video.txt':
 763                     _set_localized_text_entry(apps[packageName], locale, 'video',
 764                                               os.path.join(root, f))
 765                     continue
 766                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 767                     locale = segments[-2]
 768                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 769                                               os.path.join(root, f))
 770                     continue
 771
 772                 base, extension = common.get_extension(f)
 773                 if locale == 'images':
 774                     locale = segments[-2]
 775                     destdir = os.path.join('repo', packageName, locale)
 776                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 777                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 778                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 779                     shutil.copy(os.path.join(root, f), destdir)
 780             for d in dirs:
 781                 if d in SCREENSHOT_DIRS:
 782                     for f in glob.glob(os.path.join(root, d, '*.*')):
 783                         _, extension = common.get_extension(f)
 784                         if extension in ALLOWED_EXTENSIONS:
 785                             screenshotdestdir = os.path.join(destdir, d)
 786                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 787                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 788                             shutil.copy(f, screenshotdestdir)
 789
 790     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 791     for d in repofiles:
 792         if not os.path.isdir(d):
 793             continue
 794         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 795             if not os.path.isfile(f):
 796                 continue
 797             segments = f.split('/')
 798             packageName = segments[1]
 799             locale = segments[2]
 800             screenshotdir = segments[3]
 801             filename = os.path.basename(f)
 802             base, extension = common.get_extension(filename)
 803
 804             if packageName not in apps:
 805                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 806                                 % (filename, packageName))
 807                 continue
 808             graphics = _get_localized_dict(apps[packageName], locale)
 809
 810             if extension not in ALLOWED_EXTENSIONS:
 811                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 812             elif base in GRAPHIC_NAMES:
 813                 # there can only be zero or one of these per locale
 814                 graphics[base] = filename
 815             elif screenshotdir in SCREENSHOT_DIRS:
 816                 # there can any number of these per locale
 817                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 818                 if screenshotdir not in graphics:
 819                     graphics[screenshotdir] = []
 820                 graphics[screenshotdir].append(filename)
 821             else:
 822                 logging.warning('Unsupported graphics file found: ' + f)
 823
 824
 825 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 826     """Scan a repo for all files with an extension except APK/OBB
 827
 828     :param apkcache: current cached info about all repo files
 829     :param repodir: repo directory to scan
 830     :param knownapks: list of all known files, as per metadata.read_metadata
 831     :param use_date_from_file: use date from file (instead of current date)
 832                                for newly added files
 833     """
 834
 835     cachechanged = False
 836     repo_files = []
 837     repodir = repodir.encode('utf-8')
 838     for name in os.listdir(repodir):
 839         file_extension = common.get_file_extension(name)
 840         if file_extension == 'apk' or file_extension == 'obb':
 841             continue
 842         filename = os.path.join(repodir, name)
 843         name_utf8 = name.decode('utf-8')
 844         if filename.endswith(b'_src.tar.gz'):
 845             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 846             continue
 847         if not common.is_repo_file(filename):
 848             continue
 849         stat = os.stat(filename)
 850         if stat.st_size == 0:
 851             raise FDroidException(filename + ' is zero size!')
 852
 853         shasum = sha256sum(filename)
 854         usecache = False
 855         if name in apkcache:
 856             repo_file = apkcache[name]
 857             # added time is cached as tuple but used here as datetime instance
 858             if 'added' in repo_file:
 859                 a = repo_file['added']
 860                 if isinstance(a, datetime):
 861                     repo_file['added'] = a
 862                 else:
 863                     repo_file['added'] = datetime(*a[:6])
 864             if repo_file.get('hash') == shasum:
 865                 logging.debug("Reading " + name_utf8 + " from cache")
 866                 usecache = True
 867             else:
 868                 logging.debug("Ignoring stale cache data for " + name)
 869
 870         if not usecache:
 871             logging.debug("Processing " + name_utf8)
 872             repo_file = collections.OrderedDict()
 873             repo_file['name'] = os.path.splitext(name_utf8)[0]
 874             # TODO rename apkname globally to something more generic
 875             repo_file['apkName'] = name_utf8
 876             repo_file['hash'] = shasum
 877             repo_file['hashType'] = 'sha256'
 878             repo_file['versionCode'] = 0
 879             repo_file['versionName'] = shasum
 880             # the static ID is the SHA256 unless it is set in the metadata
 881             repo_file['packageName'] = shasum
 882
 883             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 884             if m:
 885                 repo_file['packageName'] = m.group(1)
 886                 repo_file['versionCode'] = int(m.group(2))
 887             srcfilename = name + b'_src.tar.gz'
 888             if os.path.exists(os.path.join(repodir, srcfilename)):
 889                 repo_file['srcname'] = srcfilename.decode('utf-8')
 890             repo_file['size'] = stat.st_size
 891
 892             apkcache[name] = repo_file
 893             cachechanged = True
 894
 895         if use_date_from_file:
 896             timestamp = stat.st_ctime
 897             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 898         else:
 899             default_date_param = None
 900
 901         # Record in knownapks, getting the added date at the same time..
 902         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 903                                     default_date=default_date_param)
 904         if added:
 905             repo_file['added'] = added
 906
 907         repo_files.append(repo_file)
 908
 909     return repo_files, cachechanged
 910
 911
 912 def scan_apk_aapt(apk, apkfile):
 913     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 914     if p.returncode != 0:
 915         if options.delete_unknown:
 916             if os.path.exists(apkfile):
 917                 logging.error("Failed to get apk information, deleting " + apkfile)
 918                 os.remove(apkfile)
 919             else:
 920                 logging.error("Could not find {0} to remove it".format(apkfile))
 921         else:
 922             logging.error("Failed to get apk information, skipping " + apkfile)
 923         raise BuildException("Invalid APK")
 924     for line in p.output.splitlines():
 925         if line.startswith("package:"):
 926             try:
 927                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 928                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 929                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 930             except Exception as e:
 931                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
 932         elif line.startswith("application:"):
 933             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 934             # Keep path to non-dpi icon in case we need it
 935             match = re.match(APK_ICON_PAT_NODPI, line)
 936             if match:
 937                 apk['icons_src']['-1'] = match.group(1)
 938         elif line.startswith("launchable-activity:"):
 939             # Only use launchable-activity as fallback to application
 940             if not apk['name']:
 941                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 942             if '-1' not in apk['icons_src']:
 943                 match = re.match(APK_ICON_PAT_NODPI, line)
 944                 if match:
 945                     apk['icons_src']['-1'] = match.group(1)
 946         elif line.startswith("application-icon-"):
 947             match = re.match(APK_ICON_PAT, line)
 948             if match:
 949                 density = match.group(1)
 950                 path = match.group(2)
 951                 apk['icons_src'][density] = path
 952         elif line.startswith("sdkVersion:"):
 953             m = re.match(APK_SDK_VERSION_PAT, line)
 954             if m is None:
 955                 logging.error(line.replace('sdkVersion:', '')
 956                               + ' is not a valid minSdkVersion!')
 957             else:
 958                 apk['minSdkVersion'] = m.group(1)
 959                 # if target not set, default to min
 960                 if 'targetSdkVersion' not in apk:
 961                     apk['targetSdkVersion'] = m.group(1)
 962         elif line.startswith("targetSdkVersion:"):
 963             m = re.match(APK_SDK_VERSION_PAT, line)
 964             if m is None:
 965                 logging.error(line.replace('targetSdkVersion:', '')
 966                               + ' is not a valid targetSdkVersion!')
 967             else:
 968                 apk['targetSdkVersion'] = m.group(1)
 969         elif line.startswith("maxSdkVersion:"):
 970             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 971         elif line.startswith("native-code:"):
 972             apk['nativecode'] = []
 973             for arch in line[13:].split(' '):
 974                 apk['nativecode'].append(arch[1:-1])
 975         elif line.startswith('uses-permission:'):
 976             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 977             if perm_match['maxSdkVersion']:
 978                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 979             permission = UsesPermission(
 980                 perm_match['name'],
 981                 perm_match['maxSdkVersion']
 982             )
 983
 984             apk['uses-permission'].append(permission)
 985         elif line.startswith('uses-permission-sdk-23:'):
 986             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 987             if perm_match['maxSdkVersion']:
 988                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 989             permission_sdk_23 = UsesPermissionSdk23(
 990                 perm_match['name'],
 991                 perm_match['maxSdkVersion']
 992             )
 993
 994             apk['uses-permission-sdk-23'].append(permission_sdk_23)
 995
 996         elif line.startswith('uses-feature:'):
 997             feature = re.match(APK_FEATURE_PAT, line).group(1)
 998             # Filter out this, it's only added with the latest SDK tools and
 999             # causes problems for lots of apps.
1000             if feature != "android.hardware.screen.portrait" \
1001                     and feature != "android.hardware.screen.landscape":
1002                 if feature.startswith("android.feature."):
1003                     feature = feature[16:]
1004                 apk['features'].add(feature)
1005
1006
1007 def scan_apk_androguard(apk, apkfile):
1008     try:
1009         from androguard.core.bytecodes.apk import APK
1010         apkobject = APK(apkfile)
1011         if apkobject.is_valid_APK():
1012             arsc = apkobject.get_android_resources()
1013         else:
1014             if options.delete_unknown:
1015                 if os.path.exists(apkfile):
1016                     logging.error("Failed to get apk information, deleting " + apkfile)
1017                     os.remove(apkfile)
1018                 else:
1019                     logging.error("Could not find {0} to remove it".format(apkfile))
1020             else:
1021                 logging.error("Failed to get apk information, skipping " + apkfile)
1022             raise BuildException("Invaild APK")
1023     except ImportError:
1024         raise FDroidException("androguard library is not installed and aapt not present")
1025     except FileNotFoundError:
1026         logging.error("Could not open apk file for analysis")
1027         raise BuildException("Invalid APK")
1028
1029     apk['packageName'] = apkobject.get_package()
1030     apk['versionCode'] = int(apkobject.get_androidversion_code())
1031     apk['versionName'] = apkobject.get_androidversion_name()
1032     if apk['versionName'][0] == "@":
1033         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1034         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1035         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1036     apk['name'] = apkobject.get_app_name()
1037
1038     if apkobject.get_max_sdk_version() is not None:
1039         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1040     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1041     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1042
1043     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1044     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1045
1046     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1047
1048     for file in apkobject.get_files():
1049         d_re = density_re.match(file)
1050         if d_re:
1051             folder = d_re.group(1).split('-')
1052             if len(folder) > 1:
1053                 resolution = folder[1]
1054             else:
1055                 resolution = 'mdpi'
1056             density = screen_resolutions[resolution]
1057             apk['icons_src'][density] = d_re.group(0)
1058
1059     if apk['icons_src'].get('-1') is None:
1060         apk['icons_src']['-1'] = apk['icons_src']['160']
1061
1062     arch_re = re.compile("^lib/(.*)/.*$")
1063     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1064     if len(arch) >= 1:
1065         apk['nativecode'] = []
1066         apk['nativecode'].extend(sorted(list(arch)))
1067
1068     xml = apkobject.get_android_manifest_xml()
1069
1070     for item in xml.getElementsByTagName('uses-permission'):
1071         name = str(item.getAttribute("android:name"))
1072         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1073         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1074         permission = UsesPermission(
1075             name,
1076             maxSdkVersion
1077         )
1078         apk['uses-permission'].append(permission)
1079
1080     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1081         name = str(item.getAttribute("android:name"))
1082         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1083         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1084         permission_sdk_23 = UsesPermissionSdk23(
1085             name,
1086             maxSdkVersion
1087         )
1088         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1089
1090     for item in xml.getElementsByTagName('uses-feature'):
1091         feature = str(item.getAttribute("android:name"))
1092         if feature != "android.hardware.screen.portrait" \
1093                 and feature != "android.hardware.screen.landscape":
1094             if feature.startswith("android.feature."):
1095                 feature = feature[16:]
1096         apk['features'].append(feature)
1097
1098
1099 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1100              allow_disabled_algorithms=False, archive_bad_sig=False):
1101     """Scan the apk with the given filename in the given repo directory.
1102
1103     This also extracts the icons.
1104
1105     :param apkcache: current apk cache information
1106     :param apkfilename: the filename of the apk to scan
1107     :param repodir: repo directory to scan
1108     :param knownapks: known apks info
1109     :param use_date_from_apk: use date from APK (instead of current date)
1110                               for newly added APKs
1111     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1112                                       disabled algorithms in the signature (e.g. MD5)
1113     :param archive_bad_sig: move APKs with a bad signature to the archive
1114     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1115      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1116     """
1117
1118     if ' ' in apkfilename:
1119         if options.rename_apks:
1120             newfilename = apkfilename.replace(' ', '_')
1121             os.rename(os.path.join(repodir, apkfilename),
1122                       os.path.join(repodir, newfilename))
1123             apkfilename = newfilename
1124         else:
1125             logging.critical("Spaces in filenames are not allowed.")
1126             return True, None, False
1127
1128     apkfile = os.path.join(repodir, apkfilename)
1129     shasum = sha256sum(apkfile)
1130
1131     cachechanged = False
1132     usecache = False
1133     if apkfilename in apkcache:
1134         apk = apkcache[apkfilename]
1135         if apk.get('hash') == shasum:
1136             logging.debug("Reading " + apkfilename + " from cache")
1137             usecache = True
1138         else:
1139             logging.debug("Ignoring stale cache data for " + apkfilename)
1140
1141     if not usecache:
1142         logging.debug("Processing " + apkfilename)
1143         apk = {}
1144         apk['hash'] = shasum
1145         apk['hashType'] = 'sha256'
1146         apk['uses-permission'] = []
1147         apk['uses-permission-sdk-23'] = []
1148         apk['features'] = []
1149         apk['icons_src'] = {}
1150         apk['icons'] = {}
1151         apk['antiFeatures'] = set()
1152
1153         try:
1154             if SdkToolsPopen(['aapt', 'version'], output=False):
1155                 scan_apk_aapt(apk, apkfile)
1156             else:
1157                 scan_apk_androguard(apk, apkfile)
1158         except BuildException:
1159             return True, None, False
1160
1161         if 'minSdkVersion' not in apk:
1162             logging.warn("No SDK version information found in {0}".format(apkfile))
1163             apk['minSdkVersion'] = 1
1164
1165         # Check for debuggable apks...
1166         if common.isApkAndDebuggable(apkfile):
1167             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1168
1169         # Get the signature (or md5 of, to be precise)...
1170         logging.debug('Getting signature of {0}'.format(apkfile))
1171         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1172         if not apk['sig']:
1173             logging.critical("Failed to get apk signature")
1174             return True, None, False
1175
1176         if options.rename_apks:
1177             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1178             std_short_name = os.path.join(repodir, n)
1179             if apkfile != std_short_name:
1180                 if os.path.exists(std_short_name):
1181                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1182                     if apkfile != std_long_name:
1183                         if os.path.exists(std_long_name):
1184                             dupdir = os.path.join('duplicates', repodir)
1185                             if not os.path.isdir(dupdir):
1186                                 os.makedirs(dupdir, exist_ok=True)
1187                             dupfile = os.path.join('duplicates', std_long_name)
1188                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1189                             os.rename(apkfile, dupfile)
1190                             return True, None, False
1191                         else:
1192                             os.rename(apkfile, std_long_name)
1193                     apkfile = std_long_name
1194                 else:
1195                     os.rename(apkfile, std_short_name)
1196                     apkfile = std_short_name
1197                 apkfilename = apkfile[len(repodir) + 1:]
1198
1199         apk['apkName'] = apkfilename
1200         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1201         if os.path.exists(os.path.join(repodir, srcfilename)):
1202             apk['srcname'] = srcfilename
1203         apk['size'] = os.path.getsize(apkfile)
1204
1205         # verify the jar signature is correct, allow deprecated
1206         # algorithms only if the APK is in the archive.
1207         skipapk = False
1208         if not common.verify_apk_signature(apkfile):
1209             if repodir == 'archive' or allow_disabled_algorithms:
1210                 if common.verify_old_apk_signature(apkfile):
1211                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1212                 else:
1213                     skipapk = True
1214             else:
1215                 skipapk = True
1216
1217         if skipapk:
1218             if archive_bad_sig:
1219                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1220                 move_apk_between_sections(repodir, 'archive', apk)
1221             else:
1222                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1223             return True, None, False
1224
1225         if 'KnownVuln' not in apk['antiFeatures']:
1226             if has_known_vulnerability(apkfile):
1227                 apk['antiFeatures'].add('KnownVuln')
1228
1229         apkzip = zipfile.ZipFile(apkfile, 'r')
1230
1231         # if an APK has files newer than the system time, suggest updating
1232         # the system clock.  This is useful for offline systems, used for
1233         # signing, which do not have another source of clock sync info. It
1234         # has to be more than 24 hours newer because ZIP/APK files do not
1235         # store timezone info
1236         manifest = apkzip.getinfo('AndroidManifest.xml')
1237         if manifest.date_time[1] == 0:  # month can't be zero
1238             logging.debug('AndroidManifest.xml has no date')
1239         else:
1240             dt_obj = datetime(*manifest.date_time)
1241             checkdt = dt_obj - timedelta(1)
1242             if datetime.today() < checkdt:
1243                 logging.warn('System clock is older than manifest in: '
1244                              + apkfilename
1245                              + '\nSet clock to that time using:\n'
1246                              + 'sudo date -s "' + str(dt_obj) + '"')
1247
1248         iconfilename = "%s.%s.png" % (
1249             apk['packageName'],
1250             apk['versionCode'])
1251
1252         # Extract the icon file...
1253         empty_densities = []
1254         for density in screen_densities:
1255             if density not in apk['icons_src']:
1256                 empty_densities.append(density)
1257                 continue
1258             iconsrc = apk['icons_src'][density]
1259             icon_dir = get_icon_dir(repodir, density)
1260             icondest = os.path.join(icon_dir, iconfilename)
1261
1262             try:
1263                 with open(icondest, 'wb') as f:
1264                     f.write(get_icon_bytes(apkzip, iconsrc))
1265                 apk['icons'][density] = iconfilename
1266             except (zipfile.BadZipFile, ValueError, KeyError) as e:
1267                 logging.warning("Error retrieving icon file: %s" % (icondest))
1268                 del apk['icons_src'][density]
1269                 empty_densities.append(density)
1270
1271         if '-1' in apk['icons_src']:
1272             iconsrc = apk['icons_src']['-1']
1273             iconpath = os.path.join(
1274                 get_icon_dir(repodir, '0'), iconfilename)
1275             with open(iconpath, 'wb') as f:
1276                 f.write(get_icon_bytes(apkzip, iconsrc))
1277             try:
1278                 im = Image.open(iconpath)
1279                 dpi = px_to_dpi(im.size[0])
1280                 for density in screen_densities:
1281                     if density in apk['icons']:
1282                         break
1283                     if density == screen_densities[-1] or dpi >= int(density):
1284                         apk['icons'][density] = iconfilename
1285                         shutil.move(iconpath,
1286                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
1287                         empty_densities.remove(density)
1288                         break
1289             except Exception as e:
1290                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1291
1292         if apk['icons']:
1293             apk['icon'] = iconfilename
1294
1295         apkzip.close()
1296
1297         # First try resizing down to not lose quality
1298         last_density = None
1299         for density in screen_densities:
1300             if density not in empty_densities:
1301                 last_density = density
1302                 continue
1303             if last_density is None:
1304                 continue
1305             logging.debug("Density %s not available, resizing down from %s"
1306                           % (density, last_density))
1307
1308             last_iconpath = os.path.join(
1309                 get_icon_dir(repodir, last_density), iconfilename)
1310             iconpath = os.path.join(
1311                 get_icon_dir(repodir, density), iconfilename)
1312             fp = None
1313             try:
1314                 fp = open(last_iconpath, 'rb')
1315                 im = Image.open(fp)
1316
1317                 size = dpi_to_px(density)
1318
1319                 im.thumbnail((size, size), Image.ANTIALIAS)
1320                 im.save(iconpath, "PNG")
1321                 empty_densities.remove(density)
1322             except Exception as e:
1323                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1324             finally:
1325                 if fp:
1326                     fp.close()
1327
1328         # Then just copy from the highest resolution available
1329         last_density = None
1330         for density in reversed(screen_densities):
1331             if density not in empty_densities:
1332                 last_density = density
1333                 continue
1334             if last_density is None:
1335                 continue
1336             logging.debug("Density %s not available, copying from lower density %s"
1337                           % (density, last_density))
1338
1339             shutil.copyfile(
1340                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1341                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1342
1343             empty_densities.remove(density)
1344
1345         for density in screen_densities:
1346             icon_dir = get_icon_dir(repodir, density)
1347             icondest = os.path.join(icon_dir, iconfilename)
1348             resize_icon(icondest, density)
1349
1350         # Copy from icons-mdpi to icons since mdpi is the baseline density
1351         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1352         if os.path.isfile(baseline):
1353             apk['icons']['0'] = iconfilename
1354             shutil.copyfile(baseline,
1355                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1356
1357         if use_date_from_apk and manifest.date_time[1] != 0:
1358             default_date_param = datetime(*manifest.date_time)
1359         else:
1360             default_date_param = None
1361
1362         # Record in known apks, getting the added date at the same time..
1363         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1364                                     default_date=default_date_param)
1365         if added:
1366             apk['added'] = added
1367
1368         apkcache[apkfilename] = apk
1369         cachechanged = True
1370
1371     return False, apk, cachechanged
1372
1373
1374 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1375     """Scan the apks in the given repo directory.
1376
1377     This also extracts the icons.
1378
1379     :param apkcache: current apk cache information
1380     :param repodir: repo directory to scan
1381     :param knownapks: known apks info
1382     :param use_date_from_apk: use date from APK (instead of current date)
1383                               for newly added APKs
1384     :returns: (apks, cachechanged) where apks is a list of apk information,
1385               and cachechanged is True if the apkcache got changed.
1386     """
1387
1388     cachechanged = False
1389
1390     for icon_dir in get_all_icon_dirs(repodir):
1391         if os.path.exists(icon_dir):
1392             if options.clean:
1393                 shutil.rmtree(icon_dir)
1394                 os.makedirs(icon_dir)
1395         else:
1396             os.makedirs(icon_dir)
1397
1398     apks = []
1399     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1400         apkfilename = apkfile[len(repodir) + 1:]
1401         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1402         (skip, apk, cachethis) = scan_apk(apkcache, apkfilename, repodir, knownapks,
1403                                           use_date_from_apk, ada, True)
1404         if skip:
1405             continue
1406         apks.append(apk)
1407         cachechanged = cachechanged or cachethis
1408
1409     return apks, cachechanged
1410
1411
1412 def apply_info_from_latest_apk(apps, apks):
1413     """
1414     Some information from the apks needs to be applied up to the application level.
1415     When doing this, we use the info from the most recent version's apk.
1416     We deal with figuring out when the app was added and last updated at the same time.
1417     """
1418     for appid, app in apps.items():
1419         bestver = UNSET_VERSION_CODE
1420         for apk in apks:
1421             if apk['packageName'] == appid:
1422                 if apk['versionCode'] > bestver:
1423                     bestver = apk['versionCode']
1424                     bestapk = apk
1425
1426                 if 'added' in apk:
1427                     if not app.added or apk['added'] < app.added:
1428                         app.added = apk['added']
1429                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1430                         app.lastUpdated = apk['added']
1431
1432         if not app.added:
1433             logging.debug("Don't know when " + appid + " was added")
1434         if not app.lastUpdated:
1435             logging.debug("Don't know when " + appid + " was last updated")
1436
1437         if bestver == UNSET_VERSION_CODE:
1438
1439             if app.Name is None:
1440                 app.Name = app.AutoName or appid
1441             app.icon = None
1442             logging.debug("Application " + appid + " has no packages")
1443         else:
1444             if app.Name is None:
1445                 app.Name = bestapk['name']
1446             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1447             if app.CurrentVersionCode is None:
1448                 app.CurrentVersionCode = str(bestver)
1449
1450
1451 def make_categories_txt(repodir, categories):
1452     '''Write a category list in the repo to allow quick access'''
1453     catdata = ''
1454     for cat in sorted(categories):
1455         catdata += cat + '\n'
1456     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1457         f.write(catdata)
1458
1459
1460 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1461
1462     def filter_apk_list_sorted(apk_list):
1463         res = []
1464         for apk in apk_list:
1465             if apk['packageName'] == appid:
1466                 res.append(apk)
1467
1468         # Sort the apk list by version code. First is highest/newest.
1469         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1470
1471     for appid, app in apps.items():
1472
1473         if app.ArchivePolicy:
1474             keepversions = int(app.ArchivePolicy[:-9])
1475         else:
1476             keepversions = defaultkeepversions
1477
1478         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1479                       .format(appid, len(apks), keepversions, len(archapks)))
1480
1481         current_app_apks = filter_apk_list_sorted(apks)
1482         if len(current_app_apks) > keepversions:
1483             # Move back the ones we don't want.
1484             for apk in current_app_apks[keepversions:]:
1485                 move_apk_between_sections(repodir, archivedir, apk)
1486                 archapks.append(apk)
1487                 apks.remove(apk)
1488
1489         current_app_archapks = filter_apk_list_sorted(archapks)
1490         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1491             kept = 0
1492             # Move forward the ones we want again, except DisableAlgorithm
1493             for apk in current_app_archapks:
1494                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1495                     move_apk_between_sections(archivedir, repodir, apk)
1496                     archapks.remove(apk)
1497                     apks.append(apk)
1498                     kept += 1
1499                 if kept == keepversions:
1500                     break
1501
1502
1503 def move_apk_between_sections(from_dir, to_dir, apk):
1504     """move an APK from repo to archive or vice versa"""
1505
1506     def _move_file(from_dir, to_dir, filename, ignore_missing):
1507         from_path = os.path.join(from_dir, filename)
1508         if ignore_missing and not os.path.exists(from_path):
1509             return
1510         to_path = os.path.join(to_dir, filename)
1511         if not os.path.exists(to_dir):
1512             os.mkdir(to_dir)
1513         shutil.move(from_path, to_path)
1514
1515     if from_dir == to_dir:
1516         return
1517
1518     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1519     _move_file(from_dir, to_dir, apk['apkName'], False)
1520     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1521     for density in all_screen_densities:
1522         from_icon_dir = get_icon_dir(from_dir, density)
1523         to_icon_dir = get_icon_dir(to_dir, density)
1524         if density not in apk['icons']:
1525             continue
1526         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1527     if 'srcname' in apk:
1528         _move_file(from_dir, to_dir, apk['srcname'], False)
1529
1530
1531 def add_apks_to_per_app_repos(repodir, apks):
1532     apks_per_app = dict()
1533     for apk in apks:
1534         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1535         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1536         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1537         apks_per_app[apk['packageName']] = apk
1538
1539         if not os.path.exists(apk['per_app_icons']):
1540             logging.info('Adding new repo for only ' + apk['packageName'])
1541             os.makedirs(apk['per_app_icons'])
1542
1543         apkpath = os.path.join(repodir, apk['apkName'])
1544         shutil.copy(apkpath, apk['per_app_repo'])
1545         apksigpath = apkpath + '.sig'
1546         if os.path.exists(apksigpath):
1547             shutil.copy(apksigpath, apk['per_app_repo'])
1548         apkascpath = apkpath + '.asc'
1549         if os.path.exists(apkascpath):
1550             shutil.copy(apkascpath, apk['per_app_repo'])
1551
1552
1553 config = None
1554 options = None
1555
1556
1557 def main():
1558
1559     global config, options
1560
1561     # Parse command line...
1562     parser = ArgumentParser()
1563     common.setup_global_opts(parser)
1564     parser.add_argument("--create-key", action="store_true", default=False,
1565                         help="Create a repo signing key in a keystore")
1566     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1567                         help="Create skeleton metadata files that are missing")
1568     parser.add_argument("--delete-unknown", action="store_true", default=False,
1569                         help="Delete APKs and/or OBBs without metadata from the repo")
1570     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1571                         help="Report on build data status")
1572     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1573                         help="Interactively ask about things that need updating.")
1574     parser.add_argument("-I", "--icons", action="store_true", default=False,
1575                         help="Resize all the icons exceeding the max pixel size and exit")
1576     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1577                         help="Specify editor to use in interactive mode. Default " +
1578                         "is /etc/alternatives/editor")
1579     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1580                         help="Update the wiki")
1581     parser.add_argument("--pretty", action="store_true", default=False,
1582                         help="Produce human-readable index.xml")
1583     parser.add_argument("--clean", action="store_true", default=False,
1584                         help="Clean update - don't uses caches, reprocess all apks")
1585     parser.add_argument("--nosign", action="store_true", default=False,
1586                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1587     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1588                         help="Use date from apk instead of current time for newly added apks")
1589     parser.add_argument("--rename-apks", action="store_true", default=False,
1590                         help="Rename APK files that do not match package.name_123.apk")
1591     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1592                         help="Include APKs that are signed with disabled algorithms like MD5")
1593     metadata.add_metadata_arguments(parser)
1594     options = parser.parse_args()
1595     metadata.warnings_action = options.W
1596
1597     config = common.read_config(options)
1598
1599     if not ('jarsigner' in config and 'keytool' in config):
1600         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1601
1602     repodirs = ['repo']
1603     if config['archive_older'] != 0:
1604         repodirs.append('archive')
1605         if not os.path.exists('archive'):
1606             os.mkdir('archive')
1607
1608     if options.icons:
1609         resize_all_icons(repodirs)
1610         sys.exit(0)
1611
1612     if options.rename_apks:
1613         options.clean = True
1614
1615     # check that icons exist now, rather than fail at the end of `fdroid update`
1616     for k in ['repo_icon', 'archive_icon']:
1617         if k in config:
1618             if not os.path.exists(config[k]):
1619                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1620                 sys.exit(1)
1621
1622     # if the user asks to create a keystore, do it now, reusing whatever it can
1623     if options.create_key:
1624         if os.path.exists(config['keystore']):
1625             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1626             logging.critical("\t'" + config['keystore'] + "'")
1627             sys.exit(1)
1628
1629         if 'repo_keyalias' not in config:
1630             config['repo_keyalias'] = socket.getfqdn()
1631             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1632         if 'keydname' not in config:
1633             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1634             common.write_to_config(config, 'keydname', config['keydname'])
1635         if 'keystore' not in config:
1636             config['keystore'] = common.default_config['keystore']
1637             common.write_to_config(config, 'keystore', config['keystore'])
1638
1639         password = common.genpassword()
1640         if 'keystorepass' not in config:
1641             config['keystorepass'] = password
1642             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1643         if 'keypass' not in config:
1644             config['keypass'] = password
1645             common.write_to_config(config, 'keypass', config['keypass'])
1646         common.genkeystore(config)
1647
1648     # Get all apps...
1649     apps = metadata.read_metadata()
1650
1651     # Generate a list of categories...
1652     categories = set()
1653     for app in apps.values():
1654         categories.update(app.Categories)
1655
1656     # Read known apks data (will be updated and written back when we've finished)
1657     knownapks = common.KnownApks()
1658
1659     # Get APK cache
1660     apkcache = get_cache()
1661
1662     # Delete builds for disabled apps
1663     delete_disabled_builds(apps, apkcache, repodirs)
1664
1665     # Scan all apks in the main repo
1666     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1667
1668     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1669                                            options.use_date_from_apk)
1670     cachechanged = cachechanged or fcachechanged
1671     apks += files
1672     # Generate warnings for apk's with no metadata (or create skeleton
1673     # metadata files, if requested on the command line)
1674     newmetadata = False
1675     for apk in apks:
1676         if apk['packageName'] not in apps:
1677             if options.create_metadata:
1678                 if 'name' not in apk:
1679                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1680                     continue
1681                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1682                 f.write("License:Unknown\n")
1683                 f.write("Web Site:\n")
1684                 f.write("Source Code:\n")
1685                 f.write("Issue Tracker:\n")
1686                 f.write("Changelog:\n")
1687                 f.write("Summary:" + apk['name'] + "\n")
1688                 f.write("Description:\n")
1689                 f.write(apk['name'] + "\n")
1690                 f.write(".\n")
1691                 f.write("Name:" + apk['name'] + "\n")
1692                 f.close()
1693                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1694                 newmetadata = True
1695             else:
1696                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1697                 if options.delete_unknown:
1698                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1699                     rmf = os.path.join(repodirs[0], apk['apkName'])
1700                     if not os.path.exists(rmf):
1701                         logging.error("Could not find {0} to remove it".format(rmf))
1702                     else:
1703                         os.remove(rmf)
1704                 else:
1705                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1706
1707     # update the metadata with the newly created ones included
1708     if newmetadata:
1709         apps = metadata.read_metadata()
1710
1711     copy_triple_t_store_metadata(apps)
1712     insert_obbs(repodirs[0], apps, apks)
1713     insert_localized_app_metadata(apps)
1714
1715     # Scan the archive repo for apks as well
1716     if len(repodirs) > 1:
1717         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1718         if cc:
1719             cachechanged = True
1720     else:
1721         archapks = []
1722
1723     # Apply information from latest apks to the application and update dates
1724     apply_info_from_latest_apk(apps, apks + archapks)
1725
1726     # Sort the app list by name, then the web site doesn't have to by default.
1727     # (we had to wait until we'd scanned the apks to do this, because mostly the
1728     # name comes from there!)
1729     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1730
1731     # APKs are placed into multiple repos based on the app package, providing
1732     # per-app subscription feeds for nightly builds and things like it
1733     if config['per_app_repos']:
1734         add_apks_to_per_app_repos(repodirs[0], apks)
1735         for appid, app in apps.items():
1736             repodir = os.path.join(appid, 'fdroid', 'repo')
1737             appdict = dict()
1738             appdict[appid] = app
1739             if os.path.isdir(repodir):
1740                 index.make(appdict, [appid], apks, repodir, False)
1741             else:
1742                 logging.info('Skipping index generation for ' + appid)
1743         return
1744
1745     if len(repodirs) > 1:
1746         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1747
1748     # Make the index for the main repo...
1749     index.make(apps, sortedids, apks, repodirs[0], False)
1750     make_categories_txt(repodirs[0], categories)
1751
1752     # If there's an archive repo,  make the index for it. We already scanned it
1753     # earlier on.
1754     if len(repodirs) > 1:
1755         index.make(apps, sortedids, archapks, repodirs[1], True)
1756
1757     git_remote = config.get('binary_transparency_remote')
1758     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1759         from . import btlog
1760         btlog.make_binary_transparency_log(repodirs)
1761
1762     if config['update_stats']:
1763         # Update known apks info...
1764         knownapks.writeifchanged()
1765
1766         # Generate latest apps data for widget
1767         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1768             data = ''
1769             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1770                 for line in f:
1771                     appid = line.rstrip()
1772                     data += appid + "\t"
1773                     app = apps[appid]
1774                     data += app.Name + "\t"
1775                     if app.icon is not None:
1776                         data += app.icon + "\t"
1777                     data += app.License + "\n"
1778             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1779                 f.write(data)
1780
1781     if cachechanged:
1782         write_cache(apkcache)
1783
1784     # Update the wiki...
1785     if options.wiki:
1786         update_wiki(apps, sortedids, apks + archapks)
1787
1788     logging.info("Finished.")
1789
1790
1791 if __name__ == "__main__":
1792     main()