fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import common
  41 from . import index
  42 from . import metadata
  43 from .common import SdkToolsPopen
  44 from .exception import BuildException, FDroidException
  45
  46 METADATA_VERSION = 18
  47
  48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  49 UNSET_VERSION_CODE = -0x100000000
  50
  51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85
  86 def dpi_to_px(density):
  87     return (int(density) * 48) / 160
  88
  89
  90 def px_to_dpi(px):
  91     return (int(px) * 160) / 48
  92
  93
  94 def get_icon_dir(repodir, density):
  95     if density == '0':
  96         return os.path.join(repodir, "icons")
  97     return os.path.join(repodir, "icons-%s" % density)
  98
  99
 100 def get_icon_dirs(repodir):
 101     for density in screen_densities:
 102         yield get_icon_dir(repodir, density)
 103
 104
 105 def get_all_icon_dirs(repodir):
 106     for density in all_screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def update_wiki(apps, sortedids, apks):
 111     """Update the wiki
 112
 113     :param apps: fully populated list of all applications
 114     :param apks: all apks, except...
 115     """
 116     logging.info("Updating wiki")
 117     wikicat = 'Apps'
 118     wikiredircat = 'App Redirects'
 119     import mwclient
 120     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 121                          path=config['wiki_path'])
 122     site.login(config['wiki_user'], config['wiki_password'])
 123     generated_pages = {}
 124     generated_redirects = {}
 125
 126     for appid in sortedids:
 127         app = metadata.App(apps[appid])
 128
 129         wikidata = ''
 130         if app.Disabled:
 131             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 132         if app.AntiFeatures:
 133             for af in app.AntiFeatures:
 134                 wikidata += '{{AntiFeature|' + af + '}}\n'
 135         if app.RequiresRoot:
 136             requiresroot = 'Yes'
 137         else:
 138             requiresroot = 'No'
 139         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 140             appid,
 141             app.Name,
 142             app.added.strftime('%Y-%m-%d') if app.added else '',
 143             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 144             app.SourceCode,
 145             app.IssueTracker,
 146             app.WebSite,
 147             app.Changelog,
 148             app.Donate,
 149             app.FlattrID,
 150             app.Bitcoin,
 151             app.Litecoin,
 152             app.License,
 153             requiresroot,
 154             app.AuthorName,
 155             app.AuthorEmail)
 156
 157         if app.Provides:
 158             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 159
 160         wikidata += app.Summary
 161         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 162
 163         wikidata += "=Description=\n"
 164         wikidata += metadata.description_wiki(app.Description) + "\n"
 165
 166         wikidata += "=Maintainer Notes=\n"
 167         if app.MaintainerNotes:
 168             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 169         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 170
 171         # Get a list of all packages for this application...
 172         apklist = []
 173         gotcurrentver = False
 174         cantupdate = False
 175         buildfails = False
 176         for apk in apks:
 177             if apk['packageName'] == appid:
 178                 if str(apk['versionCode']) == app.CurrentVersionCode:
 179                     gotcurrentver = True
 180                 apklist.append(apk)
 181         # Include ones we can't build, as a special case...
 182         for build in app.builds:
 183             if build.disable:
 184                 if build.versionCode == app.CurrentVersionCode:
 185                     cantupdate = True
 186                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 187                 apklist.append({'versionCode': int(build.versionCode),
 188                                 'versionName': build.versionName,
 189                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 190                                 })
 191             else:
 192                 builtit = False
 193                 for apk in apklist:
 194                     if apk['versionCode'] == int(build.versionCode):
 195                         builtit = True
 196                         break
 197                 if not builtit:
 198                     buildfails = True
 199                     apklist.append({'versionCode': int(build.versionCode),
 200                                     'versionName': build.versionName,
 201                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 202                                     })
 203         if app.CurrentVersionCode == '0':
 204             cantupdate = True
 205         # Sort with most recent first...
 206         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 207
 208         wikidata += "=Versions=\n"
 209         if len(apklist) == 0:
 210             wikidata += "We currently have no versions of this app available."
 211         elif not gotcurrentver:
 212             wikidata += "We don't have the current version of this app."
 213         else:
 214             wikidata += "We have the current version of this app."
 215         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 216         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 217         if len(app.NoSourceSince) > 0:
 218             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 219         if len(app.CurrentVersion) > 0:
 220             wikidata += "The current (recommended) version is " + app.CurrentVersion
 221             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 222         validapks = 0
 223         for apk in apklist:
 224             wikidata += "==" + apk['versionName'] + "==\n"
 225
 226             if 'buildproblem' in apk:
 227                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 228             else:
 229                 validapks += 1
 230                 wikidata += "This version is built and signed by "
 231                 if 'srcname' in apk:
 232                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 233                 else:
 234                     wikidata += "the original developer.\n\n"
 235             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 236
 237         wikidata += '\n[[Category:' + wikicat + ']]\n'
 238         if len(app.NoSourceSince) > 0:
 239             wikidata += '\n[[Category:Apps missing source code]]\n'
 240         if validapks == 0 and not app.Disabled:
 241             wikidata += '\n[[Category:Apps with no packages]]\n'
 242         if cantupdate and not app.Disabled:
 243             wikidata += "\n[[Category:Apps we cannot update]]\n"
 244         if buildfails and not app.Disabled:
 245             wikidata += "\n[[Category:Apps with failing builds]]\n"
 246         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 247             wikidata += '\n[[Category:Apps to Update]]\n'
 248         if app.Disabled:
 249             wikidata += '\n[[Category:Apps that are disabled]]\n'
 250         if app.UpdateCheckMode == 'None' and not app.Disabled:
 251             wikidata += '\n[[Category:Apps with no update check]]\n'
 252         for appcat in app.Categories:
 253             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 254
 255         # We can't have underscores in the page name, even if they're in
 256         # the package ID, because MediaWiki messes with them...
 257         pagename = appid.replace('_', ' ')
 258
 259         # Drop a trailing newline, because mediawiki is going to drop it anyway
 260         # and it we don't we'll think the page has changed when it hasn't...
 261         if wikidata.endswith('\n'):
 262             wikidata = wikidata[:-1]
 263
 264         generated_pages[pagename] = wikidata
 265
 266         # Make a redirect from the name to the ID too, unless there's
 267         # already an existing page with the name and it isn't a redirect.
 268         noclobber = False
 269         apppagename = app.Name.replace('_', ' ')
 270         apppagename = apppagename.replace('{', '')
 271         apppagename = apppagename.replace('}', ' ')
 272         apppagename = apppagename.replace(':', ' ')
 273         apppagename = apppagename.replace('[', ' ')
 274         apppagename = apppagename.replace(']', ' ')
 275         # Drop double spaces caused mostly by replacing ':' above
 276         apppagename = apppagename.replace('  ', ' ')
 277         for expagename in site.allpages(prefix=apppagename,
 278                                         filterredir='nonredirects',
 279                                         generator=False):
 280             if expagename == apppagename:
 281                 noclobber = True
 282         # Another reason not to make the redirect page is if the app name
 283         # is the same as it's ID, because that will overwrite the real page
 284         # with an redirect to itself! (Although it seems like an odd
 285         # scenario this happens a lot, e.g. where there is metadata but no
 286         # builds or binaries to extract a name from.
 287         if apppagename == pagename:
 288             noclobber = True
 289         if not noclobber:
 290             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 291
 292     for tcat, genp in [(wikicat, generated_pages),
 293                        (wikiredircat, generated_redirects)]:
 294         catpages = site.Pages['Category:' + tcat]
 295         existingpages = []
 296         for page in catpages:
 297             existingpages.append(page.name)
 298             if page.name in genp:
 299                 pagetxt = page.edit()
 300                 if pagetxt != genp[page.name]:
 301                     logging.debug("Updating modified page " + page.name)
 302                     page.save(genp[page.name], summary='Auto-updated')
 303                 else:
 304                     logging.debug("Page " + page.name + " is unchanged")
 305             else:
 306                 logging.warn("Deleting page " + page.name)
 307                 page.delete('No longer published')
 308         for pagename, text in genp.items():
 309             logging.debug("Checking " + pagename)
 310             if pagename not in existingpages:
 311                 logging.debug("Creating page " + pagename)
 312                 try:
 313                     newpage = site.Pages[pagename]
 314                     newpage.save(text, summary='Auto-created')
 315                 except Exception as e:
 316                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 317
 318     # Purge server cache to ensure counts are up to date
 319     site.pages['Repository Maintenance'].purge()
 320
 321
 322 def delete_disabled_builds(apps, apkcache, repodirs):
 323     """Delete disabled build outputs.
 324
 325     :param apps: list of all applications, as per metadata.read_metadata
 326     :param apkcache: current apk cache information
 327     :param repodirs: the repo directories to process
 328     """
 329     for appid, app in apps.items():
 330         for build in app['builds']:
 331             if not build.disable:
 332                 continue
 333             apkfilename = common.get_release_filename(app, build)
 334             iconfilename = "%s.%s.png" % (
 335                 appid,
 336                 build.versionCode)
 337             for repodir in repodirs:
 338                 files = [
 339                     os.path.join(repodir, apkfilename),
 340                     os.path.join(repodir, apkfilename + '.asc'),
 341                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 342                 ]
 343                 for density in all_screen_densities:
 344                     repo_dir = get_icon_dir(repodir, density)
 345                     files.append(os.path.join(repo_dir, iconfilename))
 346
 347                 for f in files:
 348                     if os.path.exists(f):
 349                         logging.info("Deleting disabled build output " + f)
 350                         os.remove(f)
 351             if apkfilename in apkcache:
 352                 del apkcache[apkfilename]
 353
 354
 355 def resize_icon(iconpath, density):
 356
 357     if not os.path.isfile(iconpath):
 358         return
 359
 360     fp = None
 361     try:
 362         fp = open(iconpath, 'rb')
 363         im = Image.open(fp)
 364         size = dpi_to_px(density)
 365
 366         if any(length > size for length in im.size):
 367             oldsize = im.size
 368             im.thumbnail((size, size), Image.ANTIALIAS)
 369             logging.debug("%s was too large at %s - new size is %s" % (
 370                 iconpath, oldsize, im.size))
 371             im.save(iconpath, "PNG")
 372
 373     except Exception as e:
 374         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 375
 376     finally:
 377         if fp:
 378             fp.close()
 379
 380
 381 def resize_all_icons(repodirs):
 382     """Resize all icons that exceed the max size
 383
 384     :param repodirs: the repo directories to process
 385     """
 386     for repodir in repodirs:
 387         for density in screen_densities:
 388             icon_dir = get_icon_dir(repodir, density)
 389             icon_glob = os.path.join(icon_dir, '*.png')
 390             for iconpath in glob.glob(icon_glob):
 391                 resize_icon(iconpath, density)
 392
 393
 394 def getsig(apkpath):
 395     """ Get the signing certificate of an apk. To get the same md5 has that
 396     Android gets, we encode the .RSA certificate in a specific format and pass
 397     it hex-encoded to the md5 digest algorithm.
 398
 399     :param apkpath: path to the apk
 400     :returns: A string containing the md5 of the signature of the apk or None
 401               if an error occurred.
 402     """
 403
 404     with zipfile.ZipFile(apkpath, 'r') as apk:
 405         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 406
 407         if len(certs) < 1:
 408             logging.error("Found no signing certificates on %s" % apkpath)
 409             return None
 410         if len(certs) > 1:
 411             logging.error("Found multiple signing certificates on %s" % apkpath)
 412             return None
 413
 414         cert = apk.read(certs[0])
 415
 416     cert_encoded = common.get_certificate(cert)
 417
 418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 419
 420
 421 def get_cache_file():
 422     return os.path.join('tmp', 'apkcache')
 423
 424
 425 def get_cache():
 426     """Get the cached dict of the APK index
 427
 428     Gather information about all the apk files in the repo directory,
 429     using cached data if possible. Some of the index operations take a
 430     long time, like calculating the SHA-256 and verifying the APK
 431     signature.
 432
 433     The cache is invalidated if the metadata version is different, or
 434     the 'allow_disabled_algorithms' config/option is different.  In
 435     those cases, there is no easy way to know what has changed from
 436     the cache, so just rerun the whole thing.
 437
 438     :return: apkcache
 439
 440     """
 441     apkcachefile = get_cache_file()
 442     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 443     if not options.clean and os.path.exists(apkcachefile):
 444         with open(apkcachefile, 'rb') as cf:
 445             apkcache = pickle.load(cf, encoding='utf-8')
 446         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 447            or apkcache.get('allow_disabled_algorithms') != ada:
 448             apkcache = {}
 449     else:
 450         apkcache = {}
 451
 452     apkcache["METADATA_VERSION"] = METADATA_VERSION
 453     apkcache['allow_disabled_algorithms'] = ada
 454
 455     return apkcache
 456
 457
 458 def write_cache(apkcache):
 459     apkcachefile = get_cache_file()
 460     cache_path = os.path.dirname(apkcachefile)
 461     if not os.path.exists(cache_path):
 462         os.makedirs(cache_path)
 463     with open(apkcachefile, 'wb') as cf:
 464         pickle.dump(apkcache, cf)
 465
 466
 467 def get_icon_bytes(apkzip, iconsrc):
 468     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 469     try:
 470         return apkzip.read(iconsrc)
 471     except KeyError:
 472         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 473
 474
 475 def sha256sum(filename):
 476     '''Calculate the sha256 of the given file'''
 477     sha = hashlib.sha256()
 478     with open(filename, 'rb') as f:
 479         while True:
 480             t = f.read(16384)
 481             if len(t) == 0:
 482                 break
 483             sha.update(t)
 484     return sha.hexdigest()
 485
 486
 487 def has_known_vulnerability(filename):
 488     """checks for known vulnerabilities in the APK
 489
 490     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 491     version.  Google also enforces this:
 492     https://support.google.com/faqs/answer/6376725?hl=en
 493
 494     Checks whether there are more than one classes.dex or AndroidManifest.xml
 495     files, which is invalid and an essential part of the "Master Key" attack.
 496
 497     http://www.saurik.com/id/17
 498     """
 499
 500     # statically load this pattern
 501     if not hasattr(has_known_vulnerability, "pattern"):
 502         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 503
 504     files_in_apk = set()
 505     with zipfile.ZipFile(filename) as zf:
 506         for name in zf.namelist():
 507             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 508                 lib = zf.open(name)
 509                 while True:
 510                     chunk = lib.read(4096)
 511                     if chunk == b'':
 512                         break
 513                     m = has_known_vulnerability.pattern.search(chunk)
 514                     if m:
 515                         version = m.group(1).decode('ascii')
 516                         if version.startswith('1.0.1') and version[5] >= 'r' \
 517                            or version.startswith('1.0.2') and version[5] >= 'f':
 518                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 519                         else:
 520                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 521                             return True
 522                         break
 523             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 524                 if name in files_in_apk:
 525                     return True
 526                 files_in_apk.add(name)
 527
 528     return False
 529
 530
 531 def insert_obbs(repodir, apps, apks):
 532     """Scans the .obb files in a given repo directory and adds them to the
 533     relevant APK instances.  OBB files have versionCodes like APK
 534     files, and they are loosely associated.  If there is an OBB file
 535     present, then any APK with the same or higher versionCode will use
 536     that OBB file.  There are two OBB types: main and patch, each APK
 537     can only have only have one of each.
 538
 539     https://developer.android.com/google/play/expansion-files.html
 540
 541     :param repodir: repo directory to scan
 542     :param apps: list of current, valid apps
 543     :param apks: current information on all APKs
 544
 545     """
 546
 547     def obbWarnDelete(f, msg):
 548         logging.warning(msg + f)
 549         if options.delete_unknown:
 550             logging.error("Deleting unknown file: " + f)
 551             os.remove(f)
 552
 553     obbs = []
 554     java_Integer_MIN_VALUE = -pow(2, 31)
 555     currentPackageNames = apps.keys()
 556     for f in glob.glob(os.path.join(repodir, '*.obb')):
 557         obbfile = os.path.basename(f)
 558         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 559         chunks = obbfile.split('.')
 560         if chunks[0] != 'main' and chunks[0] != 'patch':
 561             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 562             continue
 563         if not re.match(r'^-?[0-9]+$', chunks[1]):
 564             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 565             continue
 566         versionCode = int(chunks[1])
 567         packagename = ".".join(chunks[2:-1])
 568
 569         highestVersionCode = java_Integer_MIN_VALUE
 570         if packagename not in currentPackageNames:
 571             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 572             continue
 573         for apk in apks:
 574             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 575                 highestVersionCode = apk['versionCode']
 576         if versionCode > highestVersionCode:
 577             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 578                           + ') than any APK: ')
 579             continue
 580         obbsha256 = sha256sum(f)
 581         obbs.append((packagename, versionCode, obbfile, obbsha256))
 582
 583     for apk in apks:
 584         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 585             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 586                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 587                     apk['obbMainFile'] = obbfile
 588                     apk['obbMainFileSha256'] = obbsha256
 589                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 590                     apk['obbPatchFile'] = obbfile
 591                     apk['obbPatchFileSha256'] = obbsha256
 592             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 593                 break
 594
 595
 596 def translate_per_build_anti_features(apps, apks):
 597     """Grab the anti-features list from the build metadata
 598
 599     For most Anti-Features, they are really most applicable per-APK,
 600     not for an app.  An app can fix a vulnerability, add/remove
 601     tracking, etc.  This reads the 'antifeatures' list from the Build
 602     entries in the fdroiddata metadata file, then transforms it into
 603     the 'antiFeatures' list of unique items for the index.
 604
 605     The field key is all lower case in the metadata file to match the
 606     rest of the Build fields.  It is 'antiFeatures' camel case in the
 607     implementation, index, and fdroidclient since it is translated
 608     from the build 'antifeatures' field, not directly included.
 609
 610     """
 611
 612     antiFeatures = dict()
 613     for packageName, app in apps.items():
 614         d = dict()
 615         for build in app['builds']:
 616             afl = build.get('antifeatures')
 617             if afl:
 618                 d[int(build.versionCode)] = afl
 619         if len(d) > 0:
 620             antiFeatures[packageName] = d
 621
 622     for apk in apks:
 623         d = antiFeatures.get(apk['packageName'])
 624         if d:
 625             afl = d.get(apk['versionCode'])
 626             if afl:
 627                 apk['antiFeatures'].update(afl)
 628
 629
 630 def _get_localized_dict(app, locale):
 631     '''get the dict to add localized store metadata to'''
 632     if 'localized' not in app:
 633         app['localized'] = collections.OrderedDict()
 634     if locale not in app['localized']:
 635         app['localized'][locale] = collections.OrderedDict()
 636     return app['localized'][locale]
 637
 638
 639 def _set_localized_text_entry(app, locale, key, f):
 640     limit = config['char_limits'][key]
 641     localized = _get_localized_dict(app, locale)
 642     with open(f) as fp:
 643         text = fp.read()[:limit]
 644         if len(text) > 0:
 645             localized[key] = text
 646
 647
 648 def _set_author_entry(app, key, f):
 649     limit = config['char_limits']['author']
 650     with open(f) as fp:
 651         text = fp.read()[:limit]
 652         if len(text) > 0:
 653             app[key] = text
 654
 655
 656 def copy_triple_t_store_metadata(apps):
 657     """Include store metadata from the app's source repo
 658
 659     The Triple-T Gradle Play Publisher is a plugin that has a standard
 660     file layout for all of the metadata and graphics that the Google
 661     Play Store accepts.  Since F-Droid has the git repo, it can just
 662     pluck those files directly.  This method reads any text files into
 663     the app dict, then copies any graphics into the fdroid repo
 664     directory structure.
 665
 666     This needs to be run before insert_localized_app_metadata() so that
 667     the graphics files that are copied into the fdroid repo get
 668     properly indexed.
 669
 670     https://github.com/Triple-T/gradle-play-publisher#upload-images
 671     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 672
 673     """
 674
 675     if not os.path.isdir('build'):
 676         return  # nothing to do
 677
 678     for packageName, app in apps.items():
 679         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 680             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 681             for root, dirs, files in os.walk(d):
 682                 segments = root.split('/')
 683                 locale = segments[-2]
 684                 for f in files:
 685                     if f == 'fulldescription':
 686                         _set_localized_text_entry(app, locale, 'description',
 687                                                   os.path.join(root, f))
 688                         continue
 689                     elif f == 'shortdescription':
 690                         _set_localized_text_entry(app, locale, 'summary',
 691                                                   os.path.join(root, f))
 692                         continue
 693                     elif f == 'title':
 694                         _set_localized_text_entry(app, locale, 'name',
 695                                                   os.path.join(root, f))
 696                         continue
 697                     elif f == 'video':
 698                         _set_localized_text_entry(app, locale, 'video',
 699                                                   os.path.join(root, f))
 700                         continue
 701                     elif f == 'whatsnew':
 702                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 703                                                   os.path.join(root, f))
 704                         continue
 705                     elif f == 'contactEmail':
 706                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 707                         continue
 708                     elif f == 'contactPhone':
 709                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 710                         continue
 711                     elif f == 'contactWebsite':
 712                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 713                         continue
 714
 715                     base, extension = common.get_extension(f)
 716                     dirname = os.path.basename(root)
 717                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 718                         if segments[-2] == 'listing':
 719                             locale = segments[-3]
 720                         else:
 721                             locale = segments[-2]
 722                         destdir = os.path.join('repo', packageName, locale)
 723                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 724                         sourcefile = os.path.join(root, f)
 725                         destfile = os.path.join(destdir, dirname + '.' + extension)
 726                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 727                         shutil.copy(sourcefile, destfile)
 728
 729
 730 def insert_localized_app_metadata(apps):
 731     """scans standard locations for graphics and localized text
 732
 733     Scans for localized description files, store graphics, and
 734     screenshot PNG files in statically defined screenshots directory
 735     and adds them to the app metadata.  The screenshots and graphic
 736     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 737     and must be in the following layout:
 738     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 739
 740     repo/packageName/locale/featureGraphic.png
 741     repo/packageName/locale/phoneScreenshots/1.png
 742     repo/packageName/locale/phoneScreenshots/2.png
 743
 744     The changelog files must be text files named with the versionCode
 745     ending with ".txt" and must be in the following layout:
 746     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 747
 748     repo/packageName/locale/changelogs/12345.txt
 749
 750     This will scan the each app's source repo then the metadata/ dir
 751     for these standard locations of changelog files.  If it finds
 752     them, they will be added to the dict of all packages, with the
 753     versions in the metadata/ folder taking precendence over the what
 754     is in the app's source repo.
 755
 756     Where "packageName" is the app's packageName and "locale" is the locale
 757     of the graphics, e.g. what language they are in, using the IETF RFC5646
 758     format (en-US, fr-CA, es-MX, etc).
 759
 760     This will also scan the app's git for a fastlane folder, and the
 761     metadata/ folder and the apps' source repos for standard locations
 762     of graphic and screenshot files.  If it finds them, it will copy
 763     them into the repo.  The fastlane files follow this pattern:
 764     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 765
 766     """
 767
 768     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 769     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 770     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 771
 772     for srcd in sorted(sourcedirs):
 773         if not os.path.isdir(srcd):
 774             continue
 775         for root, dirs, files in os.walk(srcd):
 776             segments = root.split('/')
 777             packageName = segments[1]
 778             if packageName not in apps:
 779                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 780                 continue
 781             locale = segments[-1]
 782             destdir = os.path.join('repo', packageName, locale)
 783             for f in files:
 784                 if f in ('description.txt', 'full_description.txt'):
 785                     _set_localized_text_entry(apps[packageName], locale, 'description',
 786                                               os.path.join(root, f))
 787                     continue
 788                 elif f in ('summary.txt', 'short_description.txt'):
 789                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 790                                               os.path.join(root, f))
 791                     continue
 792                 elif f in ('name.txt', 'title.txt'):
 793                     _set_localized_text_entry(apps[packageName], locale, 'name',
 794                                               os.path.join(root, f))
 795                     continue
 796                 elif f == 'video.txt':
 797                     _set_localized_text_entry(apps[packageName], locale, 'video',
 798                                               os.path.join(root, f))
 799                     continue
 800                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 801                     locale = segments[-2]
 802                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 803                                               os.path.join(root, f))
 804                     continue
 805
 806                 base, extension = common.get_extension(f)
 807                 if locale == 'images':
 808                     locale = segments[-2]
 809                     destdir = os.path.join('repo', packageName, locale)
 810                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 811                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 812                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 813                     shutil.copy(os.path.join(root, f), destdir)
 814             for d in dirs:
 815                 if d in SCREENSHOT_DIRS:
 816                     for f in glob.glob(os.path.join(root, d, '*.*')):
 817                         _, extension = common.get_extension(f)
 818                         if extension in ALLOWED_EXTENSIONS:
 819                             screenshotdestdir = os.path.join(destdir, d)
 820                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 821                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 822                             shutil.copy(f, screenshotdestdir)
 823
 824     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 825     for d in repofiles:
 826         if not os.path.isdir(d):
 827             continue
 828         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 829             if not os.path.isfile(f):
 830                 continue
 831             segments = f.split('/')
 832             packageName = segments[1]
 833             locale = segments[2]
 834             screenshotdir = segments[3]
 835             filename = os.path.basename(f)
 836             base, extension = common.get_extension(filename)
 837
 838             if packageName not in apps:
 839                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 840                                 % (filename, packageName))
 841                 continue
 842             graphics = _get_localized_dict(apps[packageName], locale)
 843
 844             if extension not in ALLOWED_EXTENSIONS:
 845                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 846             elif base in GRAPHIC_NAMES:
 847                 # there can only be zero or one of these per locale
 848                 graphics[base] = filename
 849             elif screenshotdir in SCREENSHOT_DIRS:
 850                 # there can any number of these per locale
 851                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 852                 if screenshotdir not in graphics:
 853                     graphics[screenshotdir] = []
 854                 graphics[screenshotdir].append(filename)
 855             else:
 856                 logging.warning('Unsupported graphics file found: ' + f)
 857
 858
 859 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 860     """Scan a repo for all files with an extension except APK/OBB
 861
 862     :param apkcache: current cached info about all repo files
 863     :param repodir: repo directory to scan
 864     :param knownapks: list of all known files, as per metadata.read_metadata
 865     :param use_date_from_file: use date from file (instead of current date)
 866                                for newly added files
 867     """
 868
 869     cachechanged = False
 870     repo_files = []
 871     repodir = repodir.encode('utf-8')
 872     for name in os.listdir(repodir):
 873         file_extension = common.get_file_extension(name)
 874         if file_extension == 'apk' or file_extension == 'obb':
 875             continue
 876         filename = os.path.join(repodir, name)
 877         name_utf8 = name.decode('utf-8')
 878         if filename.endswith(b'_src.tar.gz'):
 879             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 880             continue
 881         if not common.is_repo_file(filename):
 882             continue
 883         stat = os.stat(filename)
 884         if stat.st_size == 0:
 885             raise FDroidException(filename + ' is zero size!')
 886
 887         shasum = sha256sum(filename)
 888         usecache = False
 889         if name in apkcache:
 890             repo_file = apkcache[name]
 891             # added time is cached as tuple but used here as datetime instance
 892             if 'added' in repo_file:
 893                 a = repo_file['added']
 894                 if isinstance(a, datetime):
 895                     repo_file['added'] = a
 896                 else:
 897                     repo_file['added'] = datetime(*a[:6])
 898             if repo_file.get('hash') == shasum:
 899                 logging.debug("Reading " + name_utf8 + " from cache")
 900                 usecache = True
 901             else:
 902                 logging.debug("Ignoring stale cache data for " + name)
 903
 904         if not usecache:
 905             logging.debug("Processing " + name_utf8)
 906             repo_file = collections.OrderedDict()
 907             repo_file['name'] = os.path.splitext(name_utf8)[0]
 908             # TODO rename apkname globally to something more generic
 909             repo_file['apkName'] = name_utf8
 910             repo_file['hash'] = shasum
 911             repo_file['hashType'] = 'sha256'
 912             repo_file['versionCode'] = 0
 913             repo_file['versionName'] = shasum
 914             # the static ID is the SHA256 unless it is set in the metadata
 915             repo_file['packageName'] = shasum
 916
 917             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 918             if m:
 919                 repo_file['packageName'] = m.group(1)
 920                 repo_file['versionCode'] = int(m.group(2))
 921             srcfilename = name + b'_src.tar.gz'
 922             if os.path.exists(os.path.join(repodir, srcfilename)):
 923                 repo_file['srcname'] = srcfilename.decode('utf-8')
 924             repo_file['size'] = stat.st_size
 925
 926             apkcache[name] = repo_file
 927             cachechanged = True
 928
 929         if use_date_from_file:
 930             timestamp = stat.st_ctime
 931             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 932         else:
 933             default_date_param = None
 934
 935         # Record in knownapks, getting the added date at the same time..
 936         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 937                                     default_date=default_date_param)
 938         if added:
 939             repo_file['added'] = added
 940
 941         repo_files.append(repo_file)
 942
 943     return repo_files, cachechanged
 944
 945
 946 def scan_apk(apk_file):
 947     """
 948     Scans an APK file and returns dictionary with metadata of the APK.
 949
 950     Attention: This does *not* verify that the APK signature is correct.
 951
 952     :param apk_file: The (ideally absolute) path to the APK file
 953     :raises BuildException
 954     :return A dict containing APK metadata
 955     """
 956     apk = {
 957         'hash': sha256sum(apk_file),
 958         'hashType': 'sha256',
 959         'uses-permission': [],
 960         'uses-permission-sdk-23': [],
 961         'features': [],
 962         'icons_src': {},
 963         'icons': {},
 964         'antiFeatures': set(),
 965     }
 966
 967     if SdkToolsPopen(['aapt', 'version'], output=False):
 968         scan_apk_aapt(apk, apk_file)
 969     else:
 970         scan_apk_androguard(apk, apk_file)
 971
 972     # Get the signature
 973     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 974     apk['sig'] = getsig(apk_file)
 975     if not apk['sig']:
 976         raise BuildException("Failed to get apk signature")
 977
 978     # Get size of the APK
 979     apk['size'] = os.path.getsize(apk_file)
 980
 981     if 'minSdkVersion' not in apk:
 982         logging.warning("No SDK version information found in {0}".format(apk_file))
 983         apk['minSdkVersion'] = 1
 984
 985     # Check for known vulnerabilities
 986     if has_known_vulnerability(apk_file):
 987         apk['antiFeatures'].add('KnownVuln')
 988
 989     return apk
 990
 991
 992 def scan_apk_aapt(apk, apkfile):
 993     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 994     if p.returncode != 0:
 995         if options.delete_unknown:
 996             if os.path.exists(apkfile):
 997                 logging.error("Failed to get apk information, deleting " + apkfile)
 998                 os.remove(apkfile)
 999             else:
1000                 logging.error("Could not find {0} to remove it".format(apkfile))
1001         else:
1002             logging.error("Failed to get apk information, skipping " + apkfile)
1003         raise BuildException("Invalid APK")
1004     for line in p.output.splitlines():
1005         if line.startswith("package:"):
1006             try:
1007                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1008                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1009                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1010             except Exception as e:
1011                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1012         elif line.startswith("application:"):
1013             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1014             # Keep path to non-dpi icon in case we need it
1015             match = re.match(APK_ICON_PAT_NODPI, line)
1016             if match:
1017                 apk['icons_src']['-1'] = match.group(1)
1018         elif line.startswith("launchable-activity:"):
1019             # Only use launchable-activity as fallback to application
1020             if not apk['name']:
1021                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1022             if '-1' not in apk['icons_src']:
1023                 match = re.match(APK_ICON_PAT_NODPI, line)
1024                 if match:
1025                     apk['icons_src']['-1'] = match.group(1)
1026         elif line.startswith("application-icon-"):
1027             match = re.match(APK_ICON_PAT, line)
1028             if match:
1029                 density = match.group(1)
1030                 path = match.group(2)
1031                 apk['icons_src'][density] = path
1032         elif line.startswith("sdkVersion:"):
1033             m = re.match(APK_SDK_VERSION_PAT, line)
1034             if m is None:
1035                 logging.error(line.replace('sdkVersion:', '')
1036                               + ' is not a valid minSdkVersion!')
1037             else:
1038                 apk['minSdkVersion'] = m.group(1)
1039                 # if target not set, default to min
1040                 if 'targetSdkVersion' not in apk:
1041                     apk['targetSdkVersion'] = m.group(1)
1042         elif line.startswith("targetSdkVersion:"):
1043             m = re.match(APK_SDK_VERSION_PAT, line)
1044             if m is None:
1045                 logging.error(line.replace('targetSdkVersion:', '')
1046                               + ' is not a valid targetSdkVersion!')
1047             else:
1048                 apk['targetSdkVersion'] = m.group(1)
1049         elif line.startswith("maxSdkVersion:"):
1050             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1051         elif line.startswith("native-code:"):
1052             apk['nativecode'] = []
1053             for arch in line[13:].split(' '):
1054                 apk['nativecode'].append(arch[1:-1])
1055         elif line.startswith('uses-permission:'):
1056             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1057             if perm_match['maxSdkVersion']:
1058                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1059             permission = UsesPermission(
1060                 perm_match['name'],
1061                 perm_match['maxSdkVersion']
1062             )
1063
1064             apk['uses-permission'].append(permission)
1065         elif line.startswith('uses-permission-sdk-23:'):
1066             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1067             if perm_match['maxSdkVersion']:
1068                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1069             permission_sdk_23 = UsesPermissionSdk23(
1070                 perm_match['name'],
1071                 perm_match['maxSdkVersion']
1072             )
1073
1074             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1075
1076         elif line.startswith('uses-feature:'):
1077             feature = re.match(APK_FEATURE_PAT, line).group(1)
1078             # Filter out this, it's only added with the latest SDK tools and
1079             # causes problems for lots of apps.
1080             if feature != "android.hardware.screen.portrait" \
1081                     and feature != "android.hardware.screen.landscape":
1082                 if feature.startswith("android.feature."):
1083                     feature = feature[16:]
1084                 apk['features'].add(feature)
1085
1086
1087 def scan_apk_androguard(apk, apkfile):
1088     try:
1089         from androguard.core.bytecodes.apk import APK
1090         apkobject = APK(apkfile)
1091         if apkobject.is_valid_APK():
1092             arsc = apkobject.get_android_resources()
1093         else:
1094             if options.delete_unknown:
1095                 if os.path.exists(apkfile):
1096                     logging.error("Failed to get apk information, deleting " + apkfile)
1097                     os.remove(apkfile)
1098                 else:
1099                     logging.error("Could not find {0} to remove it".format(apkfile))
1100             else:
1101                 logging.error("Failed to get apk information, skipping " + apkfile)
1102             raise BuildException("Invaild APK")
1103     except ImportError:
1104         raise FDroidException("androguard library is not installed and aapt not present")
1105     except FileNotFoundError:
1106         logging.error("Could not open apk file for analysis")
1107         raise BuildException("Invalid APK")
1108
1109     apk['packageName'] = apkobject.get_package()
1110     apk['versionCode'] = int(apkobject.get_androidversion_code())
1111     apk['versionName'] = apkobject.get_androidversion_name()
1112     if apk['versionName'][0] == "@":
1113         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1114         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1115         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1116     apk['name'] = apkobject.get_app_name()
1117
1118     if apkobject.get_max_sdk_version() is not None:
1119         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1120     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1121     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1122
1123     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1124     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1125
1126     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1127
1128     for file in apkobject.get_files():
1129         d_re = density_re.match(file)
1130         if d_re:
1131             folder = d_re.group(1).split('-')
1132             if len(folder) > 1:
1133                 resolution = folder[1]
1134             else:
1135                 resolution = 'mdpi'
1136             density = screen_resolutions[resolution]
1137             apk['icons_src'][density] = d_re.group(0)
1138
1139     if apk['icons_src'].get('-1') is None:
1140         apk['icons_src']['-1'] = apk['icons_src']['160']
1141
1142     arch_re = re.compile("^lib/(.*)/.*$")
1143     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1144     if len(arch) >= 1:
1145         apk['nativecode'] = []
1146         apk['nativecode'].extend(sorted(list(arch)))
1147
1148     xml = apkobject.get_android_manifest_xml()
1149
1150     for item in xml.getElementsByTagName('uses-permission'):
1151         name = str(item.getAttribute("android:name"))
1152         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1153         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1154         permission = UsesPermission(
1155             name,
1156             maxSdkVersion
1157         )
1158         apk['uses-permission'].append(permission)
1159
1160     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1161         name = str(item.getAttribute("android:name"))
1162         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1163         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1164         permission_sdk_23 = UsesPermissionSdk23(
1165             name,
1166             maxSdkVersion
1167         )
1168         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1169
1170     for item in xml.getElementsByTagName('uses-feature'):
1171         feature = str(item.getAttribute("android:name"))
1172         if feature != "android.hardware.screen.portrait" \
1173                 and feature != "android.hardware.screen.landscape":
1174             if feature.startswith("android.feature."):
1175                 feature = feature[16:]
1176         apk['features'].append(feature)
1177
1178
1179 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1180                 allow_disabled_algorithms=False, archive_bad_sig=False):
1181     """Processes the apk with the given filename in the given repo directory.
1182
1183     This also extracts the icons.
1184
1185     :param apkcache: current apk cache information
1186     :param apkfilename: the filename of the apk to scan
1187     :param repodir: repo directory to scan
1188     :param knownapks: known apks info
1189     :param use_date_from_apk: use date from APK (instead of current date)
1190                               for newly added APKs
1191     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1192                                       disabled algorithms in the signature (e.g. MD5)
1193     :param archive_bad_sig: move APKs with a bad signature to the archive
1194     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1195      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1196     """
1197
1198     if ' ' in apkfilename:
1199         if options.rename_apks:
1200             newfilename = apkfilename.replace(' ', '_')
1201             os.rename(os.path.join(repodir, apkfilename),
1202                       os.path.join(repodir, newfilename))
1203             apkfilename = newfilename
1204         else:
1205             logging.critical("Spaces in filenames are not allowed.")
1206             return True, None, False
1207
1208     apk = {}
1209     apkfile = os.path.join(repodir, apkfilename)
1210
1211     cachechanged = False
1212     usecache = False
1213     if apkfilename in apkcache:
1214         apk = apkcache[apkfilename]
1215         if apk.get('hash') == sha256sum(apkfile):
1216             logging.debug("Reading " + apkfilename + " from cache")
1217             usecache = True
1218         else:
1219             logging.debug("Ignoring stale cache data for " + apkfilename)
1220
1221     if not usecache:
1222         logging.debug("Processing " + apkfilename)
1223
1224         try:
1225             apk = scan_apk(apkfile)
1226         except BuildException:
1227             logging.warning('Skipping "%s" with invalid signature!', apkfilename)
1228             return True, None, False
1229
1230         # Check for debuggable apks...
1231         if common.isApkAndDebuggable(apkfile):
1232             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1233
1234         if options.rename_apks:
1235             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1236             std_short_name = os.path.join(repodir, n)
1237             if apkfile != std_short_name:
1238                 if os.path.exists(std_short_name):
1239                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1240                     if apkfile != std_long_name:
1241                         if os.path.exists(std_long_name):
1242                             dupdir = os.path.join('duplicates', repodir)
1243                             if not os.path.isdir(dupdir):
1244                                 os.makedirs(dupdir, exist_ok=True)
1245                             dupfile = os.path.join('duplicates', std_long_name)
1246                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1247                             os.rename(apkfile, dupfile)
1248                             return True, None, False
1249                         else:
1250                             os.rename(apkfile, std_long_name)
1251                     apkfile = std_long_name
1252                 else:
1253                     os.rename(apkfile, std_short_name)
1254                     apkfile = std_short_name
1255                 apkfilename = apkfile[len(repodir) + 1:]
1256
1257         apk['apkName'] = apkfilename
1258         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1259         if os.path.exists(os.path.join(repodir, srcfilename)):
1260             apk['srcname'] = srcfilename
1261
1262         # verify the jar signature is correct, allow deprecated
1263         # algorithms only if the APK is in the archive.
1264         skipapk = False
1265         if not common.verify_apk_signature(apkfile):
1266             if repodir == 'archive' or allow_disabled_algorithms:
1267                 if common.verify_old_apk_signature(apkfile):
1268                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1269                 else:
1270                     skipapk = True
1271             else:
1272                 skipapk = True
1273
1274         if skipapk:
1275             if archive_bad_sig:
1276                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1277                 move_apk_between_sections(repodir, 'archive', apk)
1278             else:
1279                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1280             return True, None, False
1281
1282         apkzip = zipfile.ZipFile(apkfile, 'r')
1283
1284         # if an APK has files newer than the system time, suggest updating
1285         # the system clock.  This is useful for offline systems, used for
1286         # signing, which do not have another source of clock sync info. It
1287         # has to be more than 24 hours newer because ZIP/APK files do not
1288         # store timezone info
1289         manifest = apkzip.getinfo('AndroidManifest.xml')
1290         if manifest.date_time[1] == 0:  # month can't be zero
1291             logging.debug('AndroidManifest.xml has no date')
1292         else:
1293             dt_obj = datetime(*manifest.date_time)
1294             checkdt = dt_obj - timedelta(1)
1295             if datetime.today() < checkdt:
1296                 logging.warning('System clock is older than manifest in: '
1297                                 + apkfilename
1298                                 + '\nSet clock to that time using:\n'
1299                                 + 'sudo date -s "' + str(dt_obj) + '"')
1300
1301         # extract icons from APK zip file
1302         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1303         try:
1304             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1305         finally:
1306             apkzip.close()  # ensure that APK zip file gets closed
1307
1308         # resize existing icons for densities missing in the APK
1309         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1310
1311         if use_date_from_apk and manifest.date_time[1] != 0:
1312             default_date_param = datetime(*manifest.date_time)
1313         else:
1314             default_date_param = None
1315
1316         # Record in known apks, getting the added date at the same time..
1317         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1318                                     default_date=default_date_param)
1319         if added:
1320             apk['added'] = added
1321
1322         apkcache[apkfilename] = apk
1323         cachechanged = True
1324
1325     return False, apk, cachechanged
1326
1327
1328 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1329     """Processes the apks in the given repo directory.
1330
1331     This also extracts the icons.
1332
1333     :param apkcache: current apk cache information
1334     :param repodir: repo directory to scan
1335     :param knownapks: known apks info
1336     :param use_date_from_apk: use date from APK (instead of current date)
1337                               for newly added APKs
1338     :returns: (apks, cachechanged) where apks is a list of apk information,
1339               and cachechanged is True if the apkcache got changed.
1340     """
1341
1342     cachechanged = False
1343
1344     for icon_dir in get_all_icon_dirs(repodir):
1345         if os.path.exists(icon_dir):
1346             if options.clean:
1347                 shutil.rmtree(icon_dir)
1348                 os.makedirs(icon_dir)
1349         else:
1350             os.makedirs(icon_dir)
1351
1352     apks = []
1353     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1354         apkfilename = apkfile[len(repodir) + 1:]
1355         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1356         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1357                                              use_date_from_apk, ada, True)
1358         if skip:
1359             continue
1360         apks.append(apk)
1361         cachechanged = cachechanged or cachethis
1362
1363     return apks, cachechanged
1364
1365
1366 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1367     """
1368     Extracts icons from the given APK zip in various densities,
1369     saves them into given repo directory
1370     and stores their names in the APK metadata dictionary.
1371
1372     :param icon_filename: A string representing the icon's file name
1373     :param apk: A populated dictionary containing APK metadata.
1374                 Needs to have 'icons_src' key
1375     :param apkzip: An opened zipfile.ZipFile of the APK file
1376     :param repo_dir: The directory of the APK's repository
1377     :return: A list of icon densities that are missing
1378     """
1379     empty_densities = []
1380     for density in screen_densities:
1381         if density not in apk['icons_src']:
1382             empty_densities.append(density)
1383             continue
1384         icon_src = apk['icons_src'][density]
1385         icon_dir = get_icon_dir(repo_dir, density)
1386         icon_dest = os.path.join(icon_dir, icon_filename)
1387
1388         # Extract the icon files per density
1389         if icon_src.endswith('.xml'):
1390             png = os.path.basename(icon_src)[:-4] + '.png'
1391             for f in apkzip.namelist():
1392                 if f.endswith(png):
1393                     m = re.match(r'res/drawable-(x*[hlm]dpi).*/', f)
1394                     if m and screen_resolutions[m.group(1)] == density:
1395                         icon_src = f
1396         try:
1397             with open(icon_dest, 'wb') as f:
1398                 f.write(get_icon_bytes(apkzip, icon_src))
1399             apk['icons'][density] = icon_filename
1400         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1401             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1402             del apk['icons_src'][density]
1403             empty_densities.append(density)
1404
1405     if '-1' in apk['icons_src']:
1406         icon_src = apk['icons_src']['-1']
1407         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1408         with open(icon_path, 'wb') as f:
1409             f.write(get_icon_bytes(apkzip, icon_src))
1410         try:
1411             im = Image.open(icon_path)
1412             dpi = px_to_dpi(im.size[0])
1413             for density in screen_densities:
1414                 if density in apk['icons']:
1415                     break
1416                 if density == screen_densities[-1] or dpi >= int(density):
1417                     apk['icons'][density] = icon_filename
1418                     shutil.move(icon_path,
1419                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1420                     empty_densities.remove(density)
1421                     break
1422         except Exception as e:
1423             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1424
1425     if apk['icons']:
1426         apk['icon'] = icon_filename
1427
1428     return empty_densities
1429
1430
1431 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1432     """
1433     Resize existing icons for densities missing in the APK to ensure all densities are available
1434
1435     :param empty_densities: A list of icon densities that are missing
1436     :param icon_filename: A string representing the icon's file name
1437     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1438     :param repo_dir: The directory of the APK's repository
1439     """
1440     # First try resizing down to not lose quality
1441     last_density = None
1442     for density in screen_densities:
1443         if density not in empty_densities:
1444             last_density = density
1445             continue
1446         if last_density is None:
1447             continue
1448         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1449
1450         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1451         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1452         fp = None
1453         try:
1454             fp = open(last_icon_path, 'rb')
1455             im = Image.open(fp)
1456
1457             size = dpi_to_px(density)
1458
1459             im.thumbnail((size, size), Image.ANTIALIAS)
1460             im.save(icon_path, "PNG")
1461             empty_densities.remove(density)
1462         except Exception as e:
1463             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1464         finally:
1465             if fp:
1466                 fp.close()
1467
1468     # Then just copy from the highest resolution available
1469     last_density = None
1470     for density in reversed(screen_densities):
1471         if density not in empty_densities:
1472             last_density = density
1473             continue
1474
1475         if last_density is None:
1476             continue
1477
1478         shutil.copyfile(
1479             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1480             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1481         )
1482         empty_densities.remove(density)
1483
1484     for density in screen_densities:
1485         icon_dir = get_icon_dir(repo_dir, density)
1486         icon_dest = os.path.join(icon_dir, icon_filename)
1487         resize_icon(icon_dest, density)
1488
1489     # Copy from icons-mdpi to icons since mdpi is the baseline density
1490     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1491     if os.path.isfile(baseline):
1492         apk['icons']['0'] = icon_filename
1493         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1494
1495
1496 def apply_info_from_latest_apk(apps, apks):
1497     """
1498     Some information from the apks needs to be applied up to the application level.
1499     When doing this, we use the info from the most recent version's apk.
1500     We deal with figuring out when the app was added and last updated at the same time.
1501     """
1502     for appid, app in apps.items():
1503         bestver = UNSET_VERSION_CODE
1504         for apk in apks:
1505             if apk['packageName'] == appid:
1506                 if apk['versionCode'] > bestver:
1507                     bestver = apk['versionCode']
1508                     bestapk = apk
1509
1510                 if 'added' in apk:
1511                     if not app.added or apk['added'] < app.added:
1512                         app.added = apk['added']
1513                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1514                         app.lastUpdated = apk['added']
1515
1516         if not app.added:
1517             logging.debug("Don't know when " + appid + " was added")
1518         if not app.lastUpdated:
1519             logging.debug("Don't know when " + appid + " was last updated")
1520
1521         if bestver == UNSET_VERSION_CODE:
1522
1523             if app.Name is None:
1524                 app.Name = app.AutoName or appid
1525             app.icon = None
1526             logging.debug("Application " + appid + " has no packages")
1527         else:
1528             if app.Name is None:
1529                 app.Name = bestapk['name']
1530             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1531             if app.CurrentVersionCode is None:
1532                 app.CurrentVersionCode = str(bestver)
1533
1534
1535 def make_categories_txt(repodir, categories):
1536     '''Write a category list in the repo to allow quick access'''
1537     catdata = ''
1538     for cat in sorted(categories):
1539         catdata += cat + '\n'
1540     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1541         f.write(catdata)
1542
1543
1544 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1545
1546     def filter_apk_list_sorted(apk_list):
1547         res = []
1548         for apk in apk_list:
1549             if apk['packageName'] == appid:
1550                 res.append(apk)
1551
1552         # Sort the apk list by version code. First is highest/newest.
1553         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1554
1555     for appid, app in apps.items():
1556
1557         if app.ArchivePolicy:
1558             keepversions = int(app.ArchivePolicy[:-9])
1559         else:
1560             keepversions = defaultkeepversions
1561
1562         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1563                       .format(appid, len(apks), keepversions, len(archapks)))
1564
1565         current_app_apks = filter_apk_list_sorted(apks)
1566         if len(current_app_apks) > keepversions:
1567             # Move back the ones we don't want.
1568             for apk in current_app_apks[keepversions:]:
1569                 move_apk_between_sections(repodir, archivedir, apk)
1570                 archapks.append(apk)
1571                 apks.remove(apk)
1572
1573         current_app_archapks = filter_apk_list_sorted(archapks)
1574         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1575             kept = 0
1576             # Move forward the ones we want again, except DisableAlgorithm
1577             for apk in current_app_archapks:
1578                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1579                     move_apk_between_sections(archivedir, repodir, apk)
1580                     archapks.remove(apk)
1581                     apks.append(apk)
1582                     kept += 1
1583                 if kept == keepversions:
1584                     break
1585
1586
1587 def move_apk_between_sections(from_dir, to_dir, apk):
1588     """move an APK from repo to archive or vice versa"""
1589
1590     def _move_file(from_dir, to_dir, filename, ignore_missing):
1591         from_path = os.path.join(from_dir, filename)
1592         if ignore_missing and not os.path.exists(from_path):
1593             return
1594         to_path = os.path.join(to_dir, filename)
1595         if not os.path.exists(to_dir):
1596             os.mkdir(to_dir)
1597         shutil.move(from_path, to_path)
1598
1599     if from_dir == to_dir:
1600         return
1601
1602     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1603     _move_file(from_dir, to_dir, apk['apkName'], False)
1604     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1605     for density in all_screen_densities:
1606         from_icon_dir = get_icon_dir(from_dir, density)
1607         to_icon_dir = get_icon_dir(to_dir, density)
1608         if density not in apk['icons']:
1609             continue
1610         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1611     if 'srcname' in apk:
1612         _move_file(from_dir, to_dir, apk['srcname'], False)
1613
1614
1615 def add_apks_to_per_app_repos(repodir, apks):
1616     apks_per_app = dict()
1617     for apk in apks:
1618         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1619         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1620         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1621         apks_per_app[apk['packageName']] = apk
1622
1623         if not os.path.exists(apk['per_app_icons']):
1624             logging.info('Adding new repo for only ' + apk['packageName'])
1625             os.makedirs(apk['per_app_icons'])
1626
1627         apkpath = os.path.join(repodir, apk['apkName'])
1628         shutil.copy(apkpath, apk['per_app_repo'])
1629         apksigpath = apkpath + '.sig'
1630         if os.path.exists(apksigpath):
1631             shutil.copy(apksigpath, apk['per_app_repo'])
1632         apkascpath = apkpath + '.asc'
1633         if os.path.exists(apkascpath):
1634             shutil.copy(apkascpath, apk['per_app_repo'])
1635
1636
1637 config = None
1638 options = None
1639
1640
1641 def main():
1642
1643     global config, options
1644
1645     # Parse command line...
1646     parser = ArgumentParser()
1647     common.setup_global_opts(parser)
1648     parser.add_argument("--create-key", action="store_true", default=False,
1649                         help="Create a repo signing key in a keystore")
1650     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1651                         help="Create skeleton metadata files that are missing")
1652     parser.add_argument("--delete-unknown", action="store_true", default=False,
1653                         help="Delete APKs and/or OBBs without metadata from the repo")
1654     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1655                         help="Report on build data status")
1656     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1657                         help="Interactively ask about things that need updating.")
1658     parser.add_argument("-I", "--icons", action="store_true", default=False,
1659                         help="Resize all the icons exceeding the max pixel size and exit")
1660     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1661                         help="Specify editor to use in interactive mode. Default " +
1662                         "is /etc/alternatives/editor")
1663     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1664                         help="Update the wiki")
1665     parser.add_argument("--pretty", action="store_true", default=False,
1666                         help="Produce human-readable index.xml")
1667     parser.add_argument("--clean", action="store_true", default=False,
1668                         help="Clean update - don't uses caches, reprocess all apks")
1669     parser.add_argument("--nosign", action="store_true", default=False,
1670                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1671     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1672                         help="Use date from apk instead of current time for newly added apks")
1673     parser.add_argument("--rename-apks", action="store_true", default=False,
1674                         help="Rename APK files that do not match package.name_123.apk")
1675     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1676                         help="Include APKs that are signed with disabled algorithms like MD5")
1677     metadata.add_metadata_arguments(parser)
1678     options = parser.parse_args()
1679     metadata.warnings_action = options.W
1680
1681     config = common.read_config(options)
1682
1683     if not ('jarsigner' in config and 'keytool' in config):
1684         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1685
1686     repodirs = ['repo']
1687     if config['archive_older'] != 0:
1688         repodirs.append('archive')
1689         if not os.path.exists('archive'):
1690             os.mkdir('archive')
1691
1692     if options.icons:
1693         resize_all_icons(repodirs)
1694         sys.exit(0)
1695
1696     if options.rename_apks:
1697         options.clean = True
1698
1699     # check that icons exist now, rather than fail at the end of `fdroid update`
1700     for k in ['repo_icon', 'archive_icon']:
1701         if k in config:
1702             if not os.path.exists(config[k]):
1703                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1704                 sys.exit(1)
1705
1706     # if the user asks to create a keystore, do it now, reusing whatever it can
1707     if options.create_key:
1708         if os.path.exists(config['keystore']):
1709             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1710             logging.critical("\t'" + config['keystore'] + "'")
1711             sys.exit(1)
1712
1713         if 'repo_keyalias' not in config:
1714             config['repo_keyalias'] = socket.getfqdn()
1715             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1716         if 'keydname' not in config:
1717             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1718             common.write_to_config(config, 'keydname', config['keydname'])
1719         if 'keystore' not in config:
1720             config['keystore'] = common.default_config['keystore']
1721             common.write_to_config(config, 'keystore', config['keystore'])
1722
1723         password = common.genpassword()
1724         if 'keystorepass' not in config:
1725             config['keystorepass'] = password
1726             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1727         if 'keypass' not in config:
1728             config['keypass'] = password
1729             common.write_to_config(config, 'keypass', config['keypass'])
1730         common.genkeystore(config)
1731
1732     # Get all apps...
1733     apps = metadata.read_metadata()
1734
1735     # Generate a list of categories...
1736     categories = set()
1737     for app in apps.values():
1738         categories.update(app.Categories)
1739
1740     # Read known apks data (will be updated and written back when we've finished)
1741     knownapks = common.KnownApks()
1742
1743     # Get APK cache
1744     apkcache = get_cache()
1745
1746     # Delete builds for disabled apps
1747     delete_disabled_builds(apps, apkcache, repodirs)
1748
1749     # Scan all apks in the main repo
1750     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1751
1752     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1753                                            options.use_date_from_apk)
1754     cachechanged = cachechanged or fcachechanged
1755     apks += files
1756     # Generate warnings for apk's with no metadata (or create skeleton
1757     # metadata files, if requested on the command line)
1758     newmetadata = False
1759     for apk in apks:
1760         if apk['packageName'] not in apps:
1761             if options.create_metadata:
1762                 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1763                     app = metadata.App()
1764                     if 'name' in apk:
1765                         app.Name = apk['name']
1766                         app.Summary = apk['name']
1767                     else:
1768                         logging.warn(apk['packageName'] + ' does not have a name! Using package name instead.')
1769                         app.Name = apk['packageName']
1770                         app.Summary = apk['packageName']
1771                     app.CurrentVersionCode = 2147483647  # Java's Integer.MAX_VALUE
1772                     app.Categories = [os.path.basename(os.path.dirname(os.getcwd()))]
1773                     metadata.write_yaml(f, app)
1774                     logging.info("Generated skeleton metadata for " + apk['packageName'])
1775                     newmetadata = True
1776             else:
1777                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1778                 if options.delete_unknown:
1779                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1780                     rmf = os.path.join(repodirs[0], apk['apkName'])
1781                     if not os.path.exists(rmf):
1782                         logging.error("Could not find {0} to remove it".format(rmf))
1783                     else:
1784                         os.remove(rmf)
1785                 else:
1786                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1787
1788     # update the metadata with the newly created ones included
1789     if newmetadata:
1790         apps = metadata.read_metadata()
1791
1792     copy_triple_t_store_metadata(apps)
1793     insert_obbs(repodirs[0], apps, apks)
1794     insert_localized_app_metadata(apps)
1795     translate_per_build_anti_features(apps, apks)
1796
1797     # Scan the archive repo for apks as well
1798     if len(repodirs) > 1:
1799         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1800         if cc:
1801             cachechanged = True
1802     else:
1803         archapks = []
1804
1805     # Apply information from latest apks to the application and update dates
1806     apply_info_from_latest_apk(apps, apks + archapks)
1807
1808     # Sort the app list by name, then the web site doesn't have to by default.
1809     # (we had to wait until we'd scanned the apks to do this, because mostly the
1810     # name comes from there!)
1811     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1812
1813     # APKs are placed into multiple repos based on the app package, providing
1814     # per-app subscription feeds for nightly builds and things like it
1815     if config['per_app_repos']:
1816         add_apks_to_per_app_repos(repodirs[0], apks)
1817         for appid, app in apps.items():
1818             repodir = os.path.join(appid, 'fdroid', 'repo')
1819             appdict = dict()
1820             appdict[appid] = app
1821             if os.path.isdir(repodir):
1822                 index.make(appdict, [appid], apks, repodir, False)
1823             else:
1824                 logging.info('Skipping index generation for ' + appid)
1825         return
1826
1827     if len(repodirs) > 1:
1828         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1829
1830     # Make the index for the main repo...
1831     index.make(apps, sortedids, apks, repodirs[0], False)
1832     make_categories_txt(repodirs[0], categories)
1833
1834     # If there's an archive repo,  make the index for it. We already scanned it
1835     # earlier on.
1836     if len(repodirs) > 1:
1837         index.make(apps, sortedids, archapks, repodirs[1], True)
1838
1839     git_remote = config.get('binary_transparency_remote')
1840     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1841         from . import btlog
1842         btlog.make_binary_transparency_log(repodirs)
1843
1844     if config['update_stats']:
1845         # Update known apks info...
1846         knownapks.writeifchanged()
1847
1848         # Generate latest apps data for widget
1849         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1850             data = ''
1851             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1852                 for line in f:
1853                     appid = line.rstrip()
1854                     data += appid + "\t"
1855                     app = apps[appid]
1856                     data += app.Name + "\t"
1857                     if app.icon is not None:
1858                         data += app.icon + "\t"
1859                     data += app.License + "\n"
1860             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1861                 f.write(data)
1862
1863     if cachechanged:
1864         write_cache(apkcache)
1865
1866     # Update the wiki...
1867     if options.wiki:
1868         update_wiki(apps, sortedids, apks + archapks)
1869
1870     logging.info("Finished.")
1871
1872
1873 if __name__ == "__main__":
1874     main()