fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in sorted(app.AntiFeatures):
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.LiberapayID,
 153             app.Bitcoin,
 154             app.Litecoin,
 155             app.License,
 156             requiresroot,
 157             app.AuthorName,
 158             app.AuthorEmail)
 159
 160         if app.Provides:
 161             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 162
 163         wikidata += app.Summary
 164         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 165
 166         wikidata += "=Description=\n"
 167         wikidata += metadata.description_wiki(app.Description) + "\n"
 168
 169         wikidata += "=Maintainer Notes=\n"
 170         if app.MaintainerNotes:
 171             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 172         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 173
 174         # Get a list of all packages for this application...
 175         apklist = []
 176         gotcurrentver = False
 177         cantupdate = False
 178         buildfails = False
 179         for apk in apks:
 180             if apk['packageName'] == appid:
 181                 if str(apk['versionCode']) == app.CurrentVersionCode:
 182                     gotcurrentver = True
 183                 apklist.append(apk)
 184         # Include ones we can't build, as a special case...
 185         for build in app.builds:
 186             if build.disable:
 187                 if build.versionCode == app.CurrentVersionCode:
 188                     cantupdate = True
 189                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 190                 apklist.append({'versionCode': int(build.versionCode),
 191                                 'versionName': build.versionName,
 192                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 193                                 })
 194             else:
 195                 builtit = False
 196                 for apk in apklist:
 197                     if apk['versionCode'] == int(build.versionCode):
 198                         builtit = True
 199                         break
 200                 if not builtit:
 201                     buildfails = True
 202                     apklist.append({'versionCode': int(build.versionCode),
 203                                     'versionName': build.versionName,
 204                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 205                                     })
 206         if app.CurrentVersionCode == '0':
 207             cantupdate = True
 208         # Sort with most recent first...
 209         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 210
 211         wikidata += "=Versions=\n"
 212         if len(apklist) == 0:
 213             wikidata += "We currently have no versions of this app available."
 214         elif not gotcurrentver:
 215             wikidata += "We don't have the current version of this app."
 216         else:
 217             wikidata += "We have the current version of this app."
 218         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 219         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 220         if len(app.NoSourceSince) > 0:
 221             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 222         if len(app.CurrentVersion) > 0:
 223             wikidata += "The current (recommended) version is " + app.CurrentVersion
 224             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 225         validapks = 0
 226         for apk in apklist:
 227             wikidata += "==" + apk['versionName'] + "==\n"
 228
 229             if 'buildproblem' in apk:
 230                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 231             else:
 232                 validapks += 1
 233                 wikidata += "This version is built and signed by "
 234                 if 'srcname' in apk:
 235                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 236                 else:
 237                     wikidata += "the original developer.\n\n"
 238             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 239
 240         wikidata += '\n[[Category:' + wikicat + ']]\n'
 241         if len(app.NoSourceSince) > 0:
 242             wikidata += '\n[[Category:Apps missing source code]]\n'
 243         if validapks == 0 and not app.Disabled:
 244             wikidata += '\n[[Category:Apps with no packages]]\n'
 245         if cantupdate and not app.Disabled:
 246             wikidata += "\n[[Category:Apps we cannot update]]\n"
 247         if buildfails and not app.Disabled:
 248             wikidata += "\n[[Category:Apps with failing builds]]\n"
 249         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 250             wikidata += '\n[[Category:Apps to Update]]\n'
 251         if app.Disabled:
 252             wikidata += '\n[[Category:Apps that are disabled]]\n'
 253         if app.UpdateCheckMode == 'None' and not app.Disabled:
 254             wikidata += '\n[[Category:Apps with no update check]]\n'
 255         for appcat in app.Categories:
 256             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 257
 258         # We can't have underscores in the page name, even if they're in
 259         # the package ID, because MediaWiki messes with them...
 260         pagename = appid.replace('_', ' ')
 261
 262         # Drop a trailing newline, because mediawiki is going to drop it anyway
 263         # and it we don't we'll think the page has changed when it hasn't...
 264         if wikidata.endswith('\n'):
 265             wikidata = wikidata[:-1]
 266
 267         generated_pages[pagename] = wikidata
 268
 269         # Make a redirect from the name to the ID too, unless there's
 270         # already an existing page with the name and it isn't a redirect.
 271         noclobber = False
 272         apppagename = app.Name.replace('_', ' ')
 273         apppagename = apppagename.replace('{', '')
 274         apppagename = apppagename.replace('}', ' ')
 275         apppagename = apppagename.replace(':', ' ')
 276         apppagename = apppagename.replace('[', ' ')
 277         apppagename = apppagename.replace(']', ' ')
 278         # Drop double spaces caused mostly by replacing ':' above
 279         apppagename = apppagename.replace('  ', ' ')
 280         for expagename in site.allpages(prefix=apppagename,
 281                                         filterredir='nonredirects',
 282                                         generator=False):
 283             if expagename == apppagename:
 284                 noclobber = True
 285         # Another reason not to make the redirect page is if the app name
 286         # is the same as it's ID, because that will overwrite the real page
 287         # with an redirect to itself! (Although it seems like an odd
 288         # scenario this happens a lot, e.g. where there is metadata but no
 289         # builds or binaries to extract a name from.
 290         if apppagename == pagename:
 291             noclobber = True
 292         if not noclobber:
 293             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 294
 295     for tcat, genp in [(wikicat, generated_pages),
 296                        (wikiredircat, generated_redirects)]:
 297         catpages = site.Pages['Category:' + tcat]
 298         existingpages = []
 299         for page in catpages:
 300             existingpages.append(page.name)
 301             if page.name in genp:
 302                 pagetxt = page.edit()
 303                 if pagetxt != genp[page.name]:
 304                     logging.debug("Updating modified page " + page.name)
 305                     page.save(genp[page.name], summary='Auto-updated')
 306                 else:
 307                     logging.debug("Page " + page.name + " is unchanged")
 308             else:
 309                 logging.warn("Deleting page " + page.name)
 310                 page.delete('No longer published')
 311         for pagename, text in genp.items():
 312             logging.debug("Checking " + pagename)
 313             if pagename not in existingpages:
 314                 logging.debug("Creating page " + pagename)
 315                 try:
 316                     newpage = site.Pages[pagename]
 317                     newpage.save(text, summary='Auto-created')
 318                 except Exception as e:
 319                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 320
 321     # Purge server cache to ensure counts are up to date
 322     site.pages['Repository Maintenance'].purge()
 323
 324
 325 def delete_disabled_builds(apps, apkcache, repodirs):
 326     """Delete disabled build outputs.
 327
 328     :param apps: list of all applications, as per metadata.read_metadata
 329     :param apkcache: current apk cache information
 330     :param repodirs: the repo directories to process
 331     """
 332     for appid, app in apps.items():
 333         for build in app['builds']:
 334             if not build.disable:
 335                 continue
 336             apkfilename = common.get_release_filename(app, build)
 337             iconfilename = "%s.%s.png" % (
 338                 appid,
 339                 build.versionCode)
 340             for repodir in repodirs:
 341                 files = [
 342                     os.path.join(repodir, apkfilename),
 343                     os.path.join(repodir, apkfilename + '.asc'),
 344                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 345                 ]
 346                 for density in all_screen_densities:
 347                     repo_dir = get_icon_dir(repodir, density)
 348                     files.append(os.path.join(repo_dir, iconfilename))
 349
 350                 for f in files:
 351                     if os.path.exists(f):
 352                         logging.info("Deleting disabled build output " + f)
 353                         os.remove(f)
 354             if apkfilename in apkcache:
 355                 del apkcache[apkfilename]
 356
 357
 358 def resize_icon(iconpath, density):
 359
 360     if not os.path.isfile(iconpath):
 361         return
 362
 363     fp = None
 364     try:
 365         fp = open(iconpath, 'rb')
 366         im = Image.open(fp)
 367         size = dpi_to_px(density)
 368
 369         if any(length > size for length in im.size):
 370             oldsize = im.size
 371             im.thumbnail((size, size), Image.ANTIALIAS)
 372             logging.debug("%s was too large at %s - new size is %s" % (
 373                 iconpath, oldsize, im.size))
 374             im.save(iconpath, "PNG")
 375
 376     except Exception as e:
 377         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 378
 379     finally:
 380         if fp:
 381             fp.close()
 382
 383
 384 def resize_all_icons(repodirs):
 385     """Resize all icons that exceed the max size
 386
 387     :param repodirs: the repo directories to process
 388     """
 389     for repodir in repodirs:
 390         for density in screen_densities:
 391             icon_dir = get_icon_dir(repodir, density)
 392             icon_glob = os.path.join(icon_dir, '*.png')
 393             for iconpath in glob.glob(icon_glob):
 394                 resize_icon(iconpath, density)
 395
 396
 397 def getsig(apkpath):
 398     """ Get the signing certificate of an apk. To get the same md5 has that
 399     Android gets, we encode the .RSA certificate in a specific format and pass
 400     it hex-encoded to the md5 digest algorithm.
 401
 402     :param apkpath: path to the apk
 403     :returns: A string containing the md5 of the signature of the apk or None
 404               if an error occurred.
 405     """
 406
 407     with zipfile.ZipFile(apkpath, 'r') as apk:
 408         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 409
 410         if len(certs) < 1:
 411             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 412             return None
 413         if len(certs) > 1:
 414             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 415             return None
 416
 417         cert = apk.read(certs[0])
 418
 419     cert_encoded = common.get_certificate(cert)
 420
 421     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 422
 423
 424 def get_cache_file():
 425     return os.path.join('tmp', 'apkcache')
 426
 427
 428 def get_cache():
 429     """Get the cached dict of the APK index
 430
 431     Gather information about all the apk files in the repo directory,
 432     using cached data if possible. Some of the index operations take a
 433     long time, like calculating the SHA-256 and verifying the APK
 434     signature.
 435
 436     The cache is invalidated if the metadata version is different, or
 437     the 'allow_disabled_algorithms' config/option is different.  In
 438     those cases, there is no easy way to know what has changed from
 439     the cache, so just rerun the whole thing.
 440
 441     :return: apkcache
 442
 443     """
 444     apkcachefile = get_cache_file()
 445     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 446     if not options.clean and os.path.exists(apkcachefile):
 447         with open(apkcachefile, 'rb') as cf:
 448             apkcache = pickle.load(cf, encoding='utf-8')
 449         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 450            or apkcache.get('allow_disabled_algorithms') != ada:
 451             apkcache = {}
 452     else:
 453         apkcache = {}
 454
 455     apkcache["METADATA_VERSION"] = METADATA_VERSION
 456     apkcache['allow_disabled_algorithms'] = ada
 457
 458     return apkcache
 459
 460
 461 def write_cache(apkcache):
 462     apkcachefile = get_cache_file()
 463     cache_path = os.path.dirname(apkcachefile)
 464     if not os.path.exists(cache_path):
 465         os.makedirs(cache_path)
 466     with open(apkcachefile, 'wb') as cf:
 467         pickle.dump(apkcache, cf)
 468
 469
 470 def get_icon_bytes(apkzip, iconsrc):
 471     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 472     try:
 473         return apkzip.read(iconsrc)
 474     except KeyError:
 475         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 476
 477
 478 def sha256sum(filename):
 479     '''Calculate the sha256 of the given file'''
 480     sha = hashlib.sha256()
 481     with open(filename, 'rb') as f:
 482         while True:
 483             t = f.read(16384)
 484             if len(t) == 0:
 485                 break
 486             sha.update(t)
 487     return sha.hexdigest()
 488
 489
 490 def has_known_vulnerability(filename):
 491     """checks for known vulnerabilities in the APK
 492
 493     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 494     version.  Google also enforces this:
 495     https://support.google.com/faqs/answer/6376725?hl=en
 496
 497     Checks whether there are more than one classes.dex or AndroidManifest.xml
 498     files, which is invalid and an essential part of the "Master Key" attack.
 499
 500     http://www.saurik.com/id/17
 501     """
 502
 503     # statically load this pattern
 504     if not hasattr(has_known_vulnerability, "pattern"):
 505         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 506
 507     files_in_apk = set()
 508     with zipfile.ZipFile(filename) as zf:
 509         for name in zf.namelist():
 510             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 511                 lib = zf.open(name)
 512                 while True:
 513                     chunk = lib.read(4096)
 514                     if chunk == b'':
 515                         break
 516                     m = has_known_vulnerability.pattern.search(chunk)
 517                     if m:
 518                         version = m.group(1).decode('ascii')
 519                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 520                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 521                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 522                             logging.debug(_('"{path}" contains recent {name} ({version})')
 523                                           .format(path=filename, name=name, version=version))
 524                         else:
 525                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 526                                             .format(path=filename, name=name, version=version))
 527                             return True
 528                         break
 529             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 530                 if name in files_in_apk:
 531                     return True
 532                 files_in_apk.add(name)
 533
 534     return False
 535
 536
 537 def insert_obbs(repodir, apps, apks):
 538     """Scans the .obb files in a given repo directory and adds them to the
 539     relevant APK instances.  OBB files have versionCodes like APK
 540     files, and they are loosely associated.  If there is an OBB file
 541     present, then any APK with the same or higher versionCode will use
 542     that OBB file.  There are two OBB types: main and patch, each APK
 543     can only have only have one of each.
 544
 545     https://developer.android.com/google/play/expansion-files.html
 546
 547     :param repodir: repo directory to scan
 548     :param apps: list of current, valid apps
 549     :param apks: current information on all APKs
 550
 551     """
 552
 553     def obbWarnDelete(f, msg):
 554         logging.warning(msg + ' ' + f)
 555         if options.delete_unknown:
 556             logging.error(_("Deleting unknown file: {path}").format(path=f))
 557             os.remove(f)
 558
 559     obbs = []
 560     java_Integer_MIN_VALUE = -pow(2, 31)
 561     currentPackageNames = apps.keys()
 562     for f in glob.glob(os.path.join(repodir, '*.obb')):
 563         obbfile = os.path.basename(f)
 564         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 565         chunks = obbfile.split('.')
 566         if chunks[0] != 'main' and chunks[0] != 'patch':
 567             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 568             continue
 569         if not re.match(r'^-?[0-9]+$', chunks[1]):
 570             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 571                           .format(name=chunks[0]))
 572             continue
 573         versionCode = int(chunks[1])
 574         packagename = ".".join(chunks[2:-1])
 575
 576         highestVersionCode = java_Integer_MIN_VALUE
 577         if packagename not in currentPackageNames:
 578             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 579             continue
 580         for apk in apks:
 581             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 582                 highestVersionCode = apk['versionCode']
 583         if versionCode > highestVersionCode:
 584             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 585                           .format(integer=str(versionCode)))
 586             continue
 587         obbsha256 = sha256sum(f)
 588         obbs.append((packagename, versionCode, obbfile, obbsha256))
 589
 590     for apk in apks:
 591         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 592             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 593                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 594                     apk['obbMainFile'] = obbfile
 595                     apk['obbMainFileSha256'] = obbsha256
 596                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 597                     apk['obbPatchFile'] = obbfile
 598                     apk['obbPatchFileSha256'] = obbsha256
 599             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 600                 break
 601
 602
 603 def translate_per_build_anti_features(apps, apks):
 604     """Grab the anti-features list from the build metadata
 605
 606     For most Anti-Features, they are really most applicable per-APK,
 607     not for an app.  An app can fix a vulnerability, add/remove
 608     tracking, etc.  This reads the 'antifeatures' list from the Build
 609     entries in the fdroiddata metadata file, then transforms it into
 610     the 'antiFeatures' list of unique items for the index.
 611
 612     The field key is all lower case in the metadata file to match the
 613     rest of the Build fields.  It is 'antiFeatures' camel case in the
 614     implementation, index, and fdroidclient since it is translated
 615     from the build 'antifeatures' field, not directly included.
 616
 617     """
 618
 619     antiFeatures = dict()
 620     for packageName, app in apps.items():
 621         d = dict()
 622         for build in app['builds']:
 623             afl = build.get('antifeatures')
 624             if afl:
 625                 d[int(build.versionCode)] = afl
 626         if len(d) > 0:
 627             antiFeatures[packageName] = d
 628
 629     for apk in apks:
 630         d = antiFeatures.get(apk['packageName'])
 631         if d:
 632             afl = d.get(apk['versionCode'])
 633             if afl:
 634                 apk['antiFeatures'].update(afl)
 635
 636
 637 def _get_localized_dict(app, locale):
 638     '''get the dict to add localized store metadata to'''
 639     if 'localized' not in app:
 640         app['localized'] = collections.OrderedDict()
 641     if locale not in app['localized']:
 642         app['localized'][locale] = collections.OrderedDict()
 643     return app['localized'][locale]
 644
 645
 646 def _set_localized_text_entry(app, locale, key, f):
 647     limit = config['char_limits'][key]
 648     localized = _get_localized_dict(app, locale)
 649     with open(f) as fp:
 650         text = fp.read()[:limit]
 651         if len(text) > 0:
 652             localized[key] = text
 653
 654
 655 def _set_author_entry(app, key, f):
 656     limit = config['char_limits']['author']
 657     with open(f) as fp:
 658         text = fp.read()[:limit]
 659         if len(text) > 0:
 660             app[key] = text
 661
 662
 663 def copy_triple_t_store_metadata(apps):
 664     """Include store metadata from the app's source repo
 665
 666     The Triple-T Gradle Play Publisher is a plugin that has a standard
 667     file layout for all of the metadata and graphics that the Google
 668     Play Store accepts.  Since F-Droid has the git repo, it can just
 669     pluck those files directly.  This method reads any text files into
 670     the app dict, then copies any graphics into the fdroid repo
 671     directory structure.
 672
 673     This needs to be run before insert_localized_app_metadata() so that
 674     the graphics files that are copied into the fdroid repo get
 675     properly indexed.
 676
 677     https://github.com/Triple-T/gradle-play-publisher#upload-images
 678     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 679
 680     """
 681
 682     if not os.path.isdir('build'):
 683         return  # nothing to do
 684
 685     for packageName, app in apps.items():
 686         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 687             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 688             for root, dirs, files in os.walk(d):
 689                 segments = root.split('/')
 690                 locale = segments[-2]
 691                 for f in files:
 692                     if f == 'fulldescription':
 693                         _set_localized_text_entry(app, locale, 'description',
 694                                                   os.path.join(root, f))
 695                         continue
 696                     elif f == 'shortdescription':
 697                         _set_localized_text_entry(app, locale, 'summary',
 698                                                   os.path.join(root, f))
 699                         continue
 700                     elif f == 'title':
 701                         _set_localized_text_entry(app, locale, 'name',
 702                                                   os.path.join(root, f))
 703                         continue
 704                     elif f == 'video':
 705                         _set_localized_text_entry(app, locale, 'video',
 706                                                   os.path.join(root, f))
 707                         continue
 708                     elif f == 'whatsnew':
 709                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 710                                                   os.path.join(root, f))
 711                         continue
 712                     elif f == 'contactEmail':
 713                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 714                         continue
 715                     elif f == 'contactPhone':
 716                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 717                         continue
 718                     elif f == 'contactWebsite':
 719                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 720                         continue
 721
 722                     base, extension = common.get_extension(f)
 723                     dirname = os.path.basename(root)
 724                     if extension in ALLOWED_EXTENSIONS \
 725                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 726                         if segments[-2] == 'listing':
 727                             locale = segments[-3]
 728                         else:
 729                             locale = segments[-2]
 730                         destdir = os.path.join('repo', packageName, locale, dirname)
 731                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 732                         sourcefile = os.path.join(root, f)
 733                         destfile = os.path.join(destdir, os.path.basename(f))
 734                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 735                         shutil.copy(sourcefile, destfile)
 736
 737
 738 def insert_localized_app_metadata(apps):
 739     """scans standard locations for graphics and localized text
 740
 741     Scans for localized description files, store graphics, and
 742     screenshot PNG files in statically defined screenshots directory
 743     and adds them to the app metadata.  The screenshots and graphic
 744     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 745     and must be in the following layout:
 746     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 747
 748     repo/packageName/locale/featureGraphic.png
 749     repo/packageName/locale/phoneScreenshots/1.png
 750     repo/packageName/locale/phoneScreenshots/2.png
 751
 752     The changelog files must be text files named with the versionCode
 753     ending with ".txt" and must be in the following layout:
 754     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 755
 756     repo/packageName/locale/changelogs/12345.txt
 757
 758     This will scan the each app's source repo then the metadata/ dir
 759     for these standard locations of changelog files.  If it finds
 760     them, they will be added to the dict of all packages, with the
 761     versions in the metadata/ folder taking precendence over the what
 762     is in the app's source repo.
 763
 764     Where "packageName" is the app's packageName and "locale" is the locale
 765     of the graphics, e.g. what language they are in, using the IETF RFC5646
 766     format (en-US, fr-CA, es-MX, etc).
 767
 768     This will also scan the app's git for a fastlane folder, and the
 769     metadata/ folder and the apps' source repos for standard locations
 770     of graphic and screenshot files.  If it finds them, it will copy
 771     them into the repo.  The fastlane files follow this pattern:
 772     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 773
 774     """
 775
 776     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 777     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 778     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 779     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 780
 781     for srcd in sorted(sourcedirs):
 782         if not os.path.isdir(srcd):
 783             continue
 784         for root, dirs, files in os.walk(srcd):
 785             segments = root.split('/')
 786             packageName = segments[1]
 787             if packageName not in apps:
 788                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 789                 continue
 790             locale = segments[-1]
 791             destdir = os.path.join('repo', packageName, locale)
 792
 793             # flavours specified in build receipt
 794             build_flavours = ""
 795             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 796                     and 'gradle' in apps[packageName].builds[-1]:
 797                 build_flavours = apps[packageName].builds[-1].gradle
 798
 799             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 800                 logging.debug("ignoring due to wrong flavour")
 801                 continue
 802
 803             for f in files:
 804                 if f in ('description.txt', 'full_description.txt'):
 805                     _set_localized_text_entry(apps[packageName], locale, 'description',
 806                                               os.path.join(root, f))
 807                     continue
 808                 elif f in ('summary.txt', 'short_description.txt'):
 809                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 810                                               os.path.join(root, f))
 811                     continue
 812                 elif f in ('name.txt', 'title.txt'):
 813                     _set_localized_text_entry(apps[packageName], locale, 'name',
 814                                               os.path.join(root, f))
 815                     continue
 816                 elif f == 'video.txt':
 817                     _set_localized_text_entry(apps[packageName], locale, 'video',
 818                                               os.path.join(root, f))
 819                     continue
 820                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 821                     locale = segments[-2]
 822                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 823                                               os.path.join(root, f))
 824                     continue
 825
 826                 base, extension = common.get_extension(f)
 827                 if locale == 'images':
 828                     locale = segments[-2]
 829                     destdir = os.path.join('repo', packageName, locale)
 830                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 831                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 832                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 833                     shutil.copy(os.path.join(root, f), destdir)
 834             for d in dirs:
 835                 if d in SCREENSHOT_DIRS:
 836                     if locale == 'images':
 837                         locale = segments[-2]
 838                         destdir = os.path.join('repo', packageName, locale)
 839                     for f in glob.glob(os.path.join(root, d, '*.*')):
 840                         _ignored, extension = common.get_extension(f)
 841                         if extension in ALLOWED_EXTENSIONS:
 842                             screenshotdestdir = os.path.join(destdir, d)
 843                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 844                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 845                             shutil.copy(f, screenshotdestdir)
 846
 847     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 848     for d in repofiles:
 849         if not os.path.isdir(d):
 850             continue
 851         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 852             if not os.path.isfile(f):
 853                 continue
 854             segments = f.split('/')
 855             packageName = segments[1]
 856             locale = segments[2]
 857             screenshotdir = segments[3]
 858             filename = os.path.basename(f)
 859             base, extension = common.get_extension(filename)
 860
 861             if packageName not in apps:
 862                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 863                                 .format(path=filename, name=packageName))
 864                 continue
 865             graphics = _get_localized_dict(apps[packageName], locale)
 866
 867             if extension not in ALLOWED_EXTENSIONS:
 868                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 869             elif base in GRAPHIC_NAMES:
 870                 # there can only be zero or one of these per locale
 871                 graphics[base] = filename
 872             elif screenshotdir in SCREENSHOT_DIRS:
 873                 # there can any number of these per locale
 874                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 875                 if screenshotdir not in graphics:
 876                     graphics[screenshotdir] = []
 877                 graphics[screenshotdir].append(filename)
 878             else:
 879                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 880
 881
 882 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 883     """Scan a repo for all files with an extension except APK/OBB
 884
 885     :param apkcache: current cached info about all repo files
 886     :param repodir: repo directory to scan
 887     :param knownapks: list of all known files, as per metadata.read_metadata
 888     :param use_date_from_file: use date from file (instead of current date)
 889                                for newly added files
 890     """
 891
 892     cachechanged = False
 893     repo_files = []
 894     repodir = repodir.encode('utf-8')
 895     for name in os.listdir(repodir):
 896         file_extension = common.get_file_extension(name)
 897         if file_extension == 'apk' or file_extension == 'obb':
 898             continue
 899         filename = os.path.join(repodir, name)
 900         name_utf8 = name.decode('utf-8')
 901         if filename.endswith(b'_src.tar.gz'):
 902             logging.debug(_('skipping source tarball: {path}')
 903                           .format(path=filename.decode('utf-8')))
 904             continue
 905         if not common.is_repo_file(filename):
 906             continue
 907         stat = os.stat(filename)
 908         if stat.st_size == 0:
 909             raise FDroidException(_('{path} is zero size!')
 910                                   .format(path=filename))
 911
 912         shasum = sha256sum(filename)
 913         usecache = False
 914         if name in apkcache:
 915             repo_file = apkcache[name]
 916             # added time is cached as tuple but used here as datetime instance
 917             if 'added' in repo_file:
 918                 a = repo_file['added']
 919                 if isinstance(a, datetime):
 920                     repo_file['added'] = a
 921                 else:
 922                     repo_file['added'] = datetime(*a[:6])
 923             if repo_file.get('hash') == shasum:
 924                 logging.debug(_("Reading {apkfilename} from cache")
 925                               .format(apkfilename=name_utf8))
 926                 usecache = True
 927             else:
 928                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 929                               .format(apkfilename=name_utf8))
 930
 931         if not usecache:
 932             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 933             repo_file = collections.OrderedDict()
 934             repo_file['name'] = os.path.splitext(name_utf8)[0]
 935             # TODO rename apkname globally to something more generic
 936             repo_file['apkName'] = name_utf8
 937             repo_file['hash'] = shasum
 938             repo_file['hashType'] = 'sha256'
 939             repo_file['versionCode'] = 0
 940             repo_file['versionName'] = shasum
 941             # the static ID is the SHA256 unless it is set in the metadata
 942             repo_file['packageName'] = shasum
 943
 944             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 945             if m:
 946                 repo_file['packageName'] = m.group(1)
 947                 repo_file['versionCode'] = int(m.group(2))
 948             srcfilename = name + b'_src.tar.gz'
 949             if os.path.exists(os.path.join(repodir, srcfilename)):
 950                 repo_file['srcname'] = srcfilename.decode('utf-8')
 951             repo_file['size'] = stat.st_size
 952
 953             apkcache[name] = repo_file
 954             cachechanged = True
 955
 956         if use_date_from_file:
 957             timestamp = stat.st_ctime
 958             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 959         else:
 960             default_date_param = None
 961
 962         # Record in knownapks, getting the added date at the same time..
 963         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 964                                     default_date=default_date_param)
 965         if added:
 966             repo_file['added'] = added
 967
 968         repo_files.append(repo_file)
 969
 970     return repo_files, cachechanged
 971
 972
 973 def scan_apk(apk_file):
 974     """
 975     Scans an APK file and returns dictionary with metadata of the APK.
 976
 977     Attention: This does *not* verify that the APK signature is correct.
 978
 979     :param apk_file: The (ideally absolute) path to the APK file
 980     :raises BuildException
 981     :return A dict containing APK metadata
 982     """
 983     apk = {
 984         'hash': sha256sum(apk_file),
 985         'hashType': 'sha256',
 986         'uses-permission': [],
 987         'uses-permission-sdk-23': [],
 988         'features': [],
 989         'icons_src': {},
 990         'icons': {},
 991         'antiFeatures': set(),
 992     }
 993
 994     if SdkToolsPopen(['aapt', 'version'], output=False):
 995         scan_apk_aapt(apk, apk_file)
 996     else:
 997         scan_apk_androguard(apk, apk_file)
 998
 999     # Get the signature, or rather the signing key fingerprints
1000     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1001     apk['sig'] = getsig(apk_file)
1002     if not apk['sig']:
1003         raise BuildException("Failed to get apk signature")
1004     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1005                                                                apk_file))
1006     if not apk.get('signer'):
1007         raise BuildException("Failed to get apk signing key fingerprint")
1008
1009     # Get size of the APK
1010     apk['size'] = os.path.getsize(apk_file)
1011
1012     if 'minSdkVersion' not in apk:
1013         logging.warning("No SDK version information found in {0}".format(apk_file))
1014         apk['minSdkVersion'] = 1
1015
1016     # Check for known vulnerabilities
1017     if has_known_vulnerability(apk_file):
1018         apk['antiFeatures'].add('KnownVuln')
1019
1020     return apk
1021
1022
1023 def scan_apk_aapt(apk, apkfile):
1024     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1025     if p.returncode != 0:
1026         if options.delete_unknown:
1027             if os.path.exists(apkfile):
1028                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1029                 os.remove(apkfile)
1030             else:
1031                 logging.error("Could not find {0} to remove it".format(apkfile))
1032         else:
1033             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1034         raise BuildException(_("Invalid APK"))
1035     for line in p.output.splitlines():
1036         if line.startswith("package:"):
1037             try:
1038                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1039                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1040                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1041             except Exception as e:
1042                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1043         elif line.startswith("application:"):
1044             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1045             # Keep path to non-dpi icon in case we need it
1046             match = re.match(APK_ICON_PAT_NODPI, line)
1047             if match:
1048                 apk['icons_src']['-1'] = match.group(1)
1049         elif line.startswith("launchable-activity:"):
1050             # Only use launchable-activity as fallback to application
1051             if not apk['name']:
1052                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1053             if '-1' not in apk['icons_src']:
1054                 match = re.match(APK_ICON_PAT_NODPI, line)
1055                 if match:
1056                     apk['icons_src']['-1'] = match.group(1)
1057         elif line.startswith("application-icon-"):
1058             match = re.match(APK_ICON_PAT, line)
1059             if match:
1060                 density = match.group(1)
1061                 path = match.group(2)
1062                 apk['icons_src'][density] = path
1063         elif line.startswith("sdkVersion:"):
1064             m = re.match(APK_SDK_VERSION_PAT, line)
1065             if m is None:
1066                 logging.error(line.replace('sdkVersion:', '')
1067                               + ' is not a valid minSdkVersion!')
1068             else:
1069                 apk['minSdkVersion'] = m.group(1)
1070                 # if target not set, default to min
1071                 if 'targetSdkVersion' not in apk:
1072                     apk['targetSdkVersion'] = m.group(1)
1073         elif line.startswith("targetSdkVersion:"):
1074             m = re.match(APK_SDK_VERSION_PAT, line)
1075             if m is None:
1076                 logging.error(line.replace('targetSdkVersion:', '')
1077                               + ' is not a valid targetSdkVersion!')
1078             else:
1079                 apk['targetSdkVersion'] = m.group(1)
1080         elif line.startswith("maxSdkVersion:"):
1081             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1082         elif line.startswith("native-code:"):
1083             apk['nativecode'] = []
1084             for arch in line[13:].split(' '):
1085                 apk['nativecode'].append(arch[1:-1])
1086         elif line.startswith('uses-permission:'):
1087             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1088             if perm_match['maxSdkVersion']:
1089                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1090             permission = UsesPermission(
1091                 perm_match['name'],
1092                 perm_match['maxSdkVersion']
1093             )
1094
1095             apk['uses-permission'].append(permission)
1096         elif line.startswith('uses-permission-sdk-23:'):
1097             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1098             if perm_match['maxSdkVersion']:
1099                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1100             permission_sdk_23 = UsesPermissionSdk23(
1101                 perm_match['name'],
1102                 perm_match['maxSdkVersion']
1103             )
1104
1105             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1106
1107         elif line.startswith('uses-feature:'):
1108             feature = re.match(APK_FEATURE_PAT, line).group(1)
1109             # Filter out this, it's only added with the latest SDK tools and
1110             # causes problems for lots of apps.
1111             if feature != "android.hardware.screen.portrait" \
1112                     and feature != "android.hardware.screen.landscape":
1113                 if feature.startswith("android.feature."):
1114                     feature = feature[16:]
1115                 apk['features'].add(feature)
1116
1117
1118 def scan_apk_androguard(apk, apkfile):
1119     try:
1120         from androguard.core.bytecodes.apk import APK
1121         apkobject = APK(apkfile)
1122         if apkobject.is_valid_APK():
1123             arsc = apkobject.get_android_resources()
1124         else:
1125             if options.delete_unknown:
1126                 if os.path.exists(apkfile):
1127                     logging.error(_("Failed to get apk information, deleting {path}")
1128                                   .format(path=apkfile))
1129                     os.remove(apkfile)
1130                 else:
1131                     logging.error(_("Could not find {path} to remove it")
1132                                   .format(path=apkfile))
1133             else:
1134                 logging.error(_("Failed to get apk information, skipping {path}")
1135                               .format(path=apkfile))
1136             raise BuildException(_("Invalid APK"))
1137     except ImportError:
1138         raise FDroidException("androguard library is not installed and aapt not present")
1139     except FileNotFoundError:
1140         logging.error(_("Could not open apk file for analysis"))
1141         raise BuildException(_("Invalid APK"))
1142
1143     apk['packageName'] = apkobject.get_package()
1144     apk['versionCode'] = int(apkobject.get_androidversion_code())
1145     apk['versionName'] = apkobject.get_androidversion_name()
1146     if apk['versionName'][0] == "@":
1147         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1148         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1149         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1150     apk['name'] = apkobject.get_app_name()
1151
1152     if apkobject.get_max_sdk_version() is not None:
1153         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1154     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1155     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1156
1157     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1158     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1159
1160     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1161
1162     for file in apkobject.get_files():
1163         d_re = density_re.match(file)
1164         if d_re:
1165             folder = d_re.group(1).split('-')
1166             if len(folder) > 1:
1167                 resolution = folder[1]
1168             else:
1169                 resolution = 'mdpi'
1170             density = screen_resolutions[resolution]
1171             apk['icons_src'][density] = d_re.group(0)
1172
1173     if apk['icons_src'].get('-1') is None:
1174         apk['icons_src']['-1'] = apk['icons_src']['160']
1175
1176     arch_re = re.compile("^lib/(.*)/.*$")
1177     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1178     if len(arch) >= 1:
1179         apk['nativecode'] = []
1180         apk['nativecode'].extend(sorted(list(arch)))
1181
1182     xml = apkobject.get_android_manifest_xml()
1183
1184     for item in xml.getElementsByTagName('uses-permission'):
1185         name = str(item.getAttribute("android:name"))
1186         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1187         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1188         permission = UsesPermission(
1189             name,
1190             maxSdkVersion
1191         )
1192         apk['uses-permission'].append(permission)
1193
1194     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1195         name = str(item.getAttribute("android:name"))
1196         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1197         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1198         permission_sdk_23 = UsesPermissionSdk23(
1199             name,
1200             maxSdkVersion
1201         )
1202         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1203
1204     for item in xml.getElementsByTagName('uses-feature'):
1205         feature = str(item.getAttribute("android:name"))
1206         if feature != "android.hardware.screen.portrait" \
1207                 and feature != "android.hardware.screen.landscape":
1208             if feature.startswith("android.feature."):
1209                 feature = feature[16:]
1210         apk['features'].append(feature)
1211
1212
1213 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1214                 allow_disabled_algorithms=False, archive_bad_sig=False):
1215     """Processes the apk with the given filename in the given repo directory.
1216
1217     This also extracts the icons.
1218
1219     :param apkcache: current apk cache information
1220     :param apkfilename: the filename of the apk to scan
1221     :param repodir: repo directory to scan
1222     :param knownapks: known apks info
1223     :param use_date_from_apk: use date from APK (instead of current date)
1224                               for newly added APKs
1225     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1226                                       disabled algorithms in the signature (e.g. MD5)
1227     :param archive_bad_sig: move APKs with a bad signature to the archive
1228     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1229      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1230     """
1231
1232     apk = {}
1233     apkfile = os.path.join(repodir, apkfilename)
1234
1235     cachechanged = False
1236     usecache = False
1237     if apkfilename in apkcache:
1238         apk = apkcache[apkfilename]
1239         if apk.get('hash') == sha256sum(apkfile):
1240             logging.debug(_("Reading {apkfilename} from cache")
1241                           .format(apkfilename=apkfilename))
1242             usecache = True
1243         else:
1244             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1245                           .format(apkfilename=apkfilename))
1246
1247     if not usecache:
1248         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1249
1250         try:
1251             apk = scan_apk(apkfile)
1252         except BuildException:
1253             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1254                             .format(apkfilename=apkfilename))
1255             return True, None, False
1256
1257         # Check for debuggable apks...
1258         if common.isApkAndDebuggable(apkfile):
1259             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1260
1261         if options.rename_apks:
1262             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1263             std_short_name = os.path.join(repodir, n)
1264             if apkfile != std_short_name:
1265                 if os.path.exists(std_short_name):
1266                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1267                     if apkfile != std_long_name:
1268                         if os.path.exists(std_long_name):
1269                             dupdir = os.path.join('duplicates', repodir)
1270                             if not os.path.isdir(dupdir):
1271                                 os.makedirs(dupdir, exist_ok=True)
1272                             dupfile = os.path.join('duplicates', std_long_name)
1273                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1274                             os.rename(apkfile, dupfile)
1275                             return True, None, False
1276                         else:
1277                             os.rename(apkfile, std_long_name)
1278                     apkfile = std_long_name
1279                 else:
1280                     os.rename(apkfile, std_short_name)
1281                     apkfile = std_short_name
1282                 apkfilename = apkfile[len(repodir) + 1:]
1283
1284         apk['apkName'] = apkfilename
1285         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1286         if os.path.exists(os.path.join(repodir, srcfilename)):
1287             apk['srcname'] = srcfilename
1288
1289         # verify the jar signature is correct, allow deprecated
1290         # algorithms only if the APK is in the archive.
1291         skipapk = False
1292         if not common.verify_apk_signature(apkfile):
1293             if repodir == 'archive' or allow_disabled_algorithms:
1294                 if common.verify_old_apk_signature(apkfile):
1295                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1296                 else:
1297                     skipapk = True
1298             else:
1299                 skipapk = True
1300
1301         if skipapk:
1302             if archive_bad_sig:
1303                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1304                                 .format(apkfilename=apkfilename))
1305                 move_apk_between_sections(repodir, 'archive', apk)
1306             else:
1307                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1308                                 .format(apkfilename=apkfilename))
1309             return True, None, False
1310
1311         apkzip = zipfile.ZipFile(apkfile, 'r')
1312
1313         manifest = apkzip.getinfo('AndroidManifest.xml')
1314         if manifest.date_time[1] == 0:  # month can't be zero
1315             logging.debug(_('AndroidManifest.xml has no date'))
1316         else:
1317             common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1318
1319         # extract icons from APK zip file
1320         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1321         try:
1322             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1323         finally:
1324             apkzip.close()  # ensure that APK zip file gets closed
1325
1326         # resize existing icons for densities missing in the APK
1327         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1328
1329         if use_date_from_apk and manifest.date_time[1] != 0:
1330             default_date_param = datetime(*manifest.date_time)
1331         else:
1332             default_date_param = None
1333
1334         # Record in known apks, getting the added date at the same time..
1335         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1336                                     default_date=default_date_param)
1337         if added:
1338             apk['added'] = added
1339
1340         apkcache[apkfilename] = apk
1341         cachechanged = True
1342
1343     return False, apk, cachechanged
1344
1345
1346 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1347     """Processes the apks in the given repo directory.
1348
1349     This also extracts the icons.
1350
1351     :param apkcache: current apk cache information
1352     :param repodir: repo directory to scan
1353     :param knownapks: known apks info
1354     :param use_date_from_apk: use date from APK (instead of current date)
1355                               for newly added APKs
1356     :returns: (apks, cachechanged) where apks is a list of apk information,
1357               and cachechanged is True if the apkcache got changed.
1358     """
1359
1360     cachechanged = False
1361
1362     for icon_dir in get_all_icon_dirs(repodir):
1363         if os.path.exists(icon_dir):
1364             if options.clean:
1365                 shutil.rmtree(icon_dir)
1366                 os.makedirs(icon_dir)
1367         else:
1368             os.makedirs(icon_dir)
1369
1370     apks = []
1371     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1372         apkfilename = apkfile[len(repodir) + 1:]
1373         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1374         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1375                                              use_date_from_apk, ada, True)
1376         if skip:
1377             continue
1378         apks.append(apk)
1379         cachechanged = cachechanged or cachethis
1380
1381     return apks, cachechanged
1382
1383
1384 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1385     """
1386     Extracts icons from the given APK zip in various densities,
1387     saves them into given repo directory
1388     and stores their names in the APK metadata dictionary.
1389
1390     :param icon_filename: A string representing the icon's file name
1391     :param apk: A populated dictionary containing APK metadata.
1392                 Needs to have 'icons_src' key
1393     :param apkzip: An opened zipfile.ZipFile of the APK file
1394     :param repo_dir: The directory of the APK's repository
1395     :return: A list of icon densities that are missing
1396     """
1397     empty_densities = []
1398     for density in screen_densities:
1399         if density not in apk['icons_src']:
1400             empty_densities.append(density)
1401             continue
1402         icon_src = apk['icons_src'][density]
1403         icon_dir = get_icon_dir(repo_dir, density)
1404         icon_dest = os.path.join(icon_dir, icon_filename)
1405
1406         # Extract the icon files per density
1407         if icon_src.endswith('.xml'):
1408             png = os.path.basename(icon_src)[:-4] + '.png'
1409             for f in apkzip.namelist():
1410                 if f.endswith(png):
1411                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1412                     if m and screen_resolutions[m.group(2)] == density:
1413                         icon_src = f
1414             if icon_src.endswith('.xml'):
1415                 empty_densities.append(density)
1416                 continue
1417         try:
1418             with open(icon_dest, 'wb') as f:
1419                 f.write(get_icon_bytes(apkzip, icon_src))
1420             apk['icons'][density] = icon_filename
1421         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1422             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1423             del apk['icons_src'][density]
1424             empty_densities.append(density)
1425
1426     if '-1' in apk['icons_src']:
1427         icon_src = apk['icons_src']['-1']
1428         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1429         with open(icon_path, 'wb') as f:
1430             f.write(get_icon_bytes(apkzip, icon_src))
1431         try:
1432             im = Image.open(icon_path)
1433             dpi = px_to_dpi(im.size[0])
1434             for density in screen_densities:
1435                 if density in apk['icons']:
1436                     break
1437                 if density == screen_densities[-1] or dpi >= int(density):
1438                     apk['icons'][density] = icon_filename
1439                     shutil.move(icon_path,
1440                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1441                     empty_densities.remove(density)
1442                     break
1443         except Exception as e:
1444             logging.warning(_("Failed reading {path}: {error}")
1445                             .format(path=icon_path, error=e))
1446
1447     if apk['icons']:
1448         apk['icon'] = icon_filename
1449
1450     return empty_densities
1451
1452
1453 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1454     """
1455     Resize existing icons for densities missing in the APK to ensure all densities are available
1456
1457     :param empty_densities: A list of icon densities that are missing
1458     :param icon_filename: A string representing the icon's file name
1459     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1460     :param repo_dir: The directory of the APK's repository
1461     """
1462     # First try resizing down to not lose quality
1463     last_density = None
1464     for density in screen_densities:
1465         if density not in empty_densities:
1466             last_density = density
1467             continue
1468         if last_density is None:
1469             continue
1470         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1471
1472         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1473         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1474         fp = None
1475         try:
1476             fp = open(last_icon_path, 'rb')
1477             im = Image.open(fp)
1478
1479             size = dpi_to_px(density)
1480
1481             im.thumbnail((size, size), Image.ANTIALIAS)
1482             im.save(icon_path, "PNG")
1483             empty_densities.remove(density)
1484         except Exception as e:
1485             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1486         finally:
1487             if fp:
1488                 fp.close()
1489
1490     # Then just copy from the highest resolution available
1491     last_density = None
1492     for density in reversed(screen_densities):
1493         if density not in empty_densities:
1494             last_density = density
1495             continue
1496
1497         if last_density is None:
1498             continue
1499
1500         shutil.copyfile(
1501             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1502             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1503         )
1504         empty_densities.remove(density)
1505
1506     for density in screen_densities:
1507         icon_dir = get_icon_dir(repo_dir, density)
1508         icon_dest = os.path.join(icon_dir, icon_filename)
1509         resize_icon(icon_dest, density)
1510
1511     # Copy from icons-mdpi to icons since mdpi is the baseline density
1512     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1513     if os.path.isfile(baseline):
1514         apk['icons']['0'] = icon_filename
1515         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1516
1517
1518 def apply_info_from_latest_apk(apps, apks):
1519     """
1520     Some information from the apks needs to be applied up to the application level.
1521     When doing this, we use the info from the most recent version's apk.
1522     We deal with figuring out when the app was added and last updated at the same time.
1523     """
1524     for appid, app in apps.items():
1525         bestver = UNSET_VERSION_CODE
1526         for apk in apks:
1527             if apk['packageName'] == appid:
1528                 if apk['versionCode'] > bestver:
1529                     bestver = apk['versionCode']
1530                     bestapk = apk
1531
1532                 if 'added' in apk:
1533                     if not app.added or apk['added'] < app.added:
1534                         app.added = apk['added']
1535                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1536                         app.lastUpdated = apk['added']
1537
1538         if not app.added:
1539             logging.debug("Don't know when " + appid + " was added")
1540         if not app.lastUpdated:
1541             logging.debug("Don't know when " + appid + " was last updated")
1542
1543         if bestver == UNSET_VERSION_CODE:
1544
1545             if app.Name is None:
1546                 app.Name = app.AutoName or appid
1547             app.icon = None
1548             logging.debug("Application " + appid + " has no packages")
1549         else:
1550             if app.Name is None:
1551                 app.Name = bestapk['name']
1552             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1553             if app.CurrentVersionCode is None:
1554                 app.CurrentVersionCode = str(bestver)
1555
1556
1557 def make_categories_txt(repodir, categories):
1558     '''Write a category list in the repo to allow quick access'''
1559     catdata = ''
1560     for cat in sorted(categories):
1561         catdata += cat + '\n'
1562     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1563         f.write(catdata)
1564
1565
1566 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1567
1568     def filter_apk_list_sorted(apk_list):
1569         res = []
1570         for apk in apk_list:
1571             if apk['packageName'] == appid:
1572                 res.append(apk)
1573
1574         # Sort the apk list by version code. First is highest/newest.
1575         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1576
1577     for appid, app in apps.items():
1578
1579         if app.ArchivePolicy:
1580             keepversions = int(app.ArchivePolicy[:-9])
1581         else:
1582             keepversions = defaultkeepversions
1583
1584         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1585                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1586
1587         current_app_apks = filter_apk_list_sorted(apks)
1588         if len(current_app_apks) > keepversions:
1589             # Move back the ones we don't want.
1590             for apk in current_app_apks[keepversions:]:
1591                 move_apk_between_sections(repodir, archivedir, apk)
1592                 archapks.append(apk)
1593                 apks.remove(apk)
1594
1595         current_app_archapks = filter_apk_list_sorted(archapks)
1596         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1597             kept = 0
1598             # Move forward the ones we want again, except DisableAlgorithm
1599             for apk in current_app_archapks:
1600                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1601                     move_apk_between_sections(archivedir, repodir, apk)
1602                     archapks.remove(apk)
1603                     apks.append(apk)
1604                     kept += 1
1605                 if kept == keepversions:
1606                     break
1607
1608
1609 def move_apk_between_sections(from_dir, to_dir, apk):
1610     """move an APK from repo to archive or vice versa"""
1611
1612     def _move_file(from_dir, to_dir, filename, ignore_missing):
1613         from_path = os.path.join(from_dir, filename)
1614         if ignore_missing and not os.path.exists(from_path):
1615             return
1616         to_path = os.path.join(to_dir, filename)
1617         if not os.path.exists(to_dir):
1618             os.mkdir(to_dir)
1619         shutil.move(from_path, to_path)
1620
1621     if from_dir == to_dir:
1622         return
1623
1624     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1625     _move_file(from_dir, to_dir, apk['apkName'], False)
1626     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1627     for density in all_screen_densities:
1628         from_icon_dir = get_icon_dir(from_dir, density)
1629         to_icon_dir = get_icon_dir(to_dir, density)
1630         if density not in apk.get('icons', []):
1631             continue
1632         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1633     if 'srcname' in apk:
1634         _move_file(from_dir, to_dir, apk['srcname'], False)
1635
1636
1637 def add_apks_to_per_app_repos(repodir, apks):
1638     apks_per_app = dict()
1639     for apk in apks:
1640         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1641         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1642         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1643         apks_per_app[apk['packageName']] = apk
1644
1645         if not os.path.exists(apk['per_app_icons']):
1646             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1647             os.makedirs(apk['per_app_icons'])
1648
1649         apkpath = os.path.join(repodir, apk['apkName'])
1650         shutil.copy(apkpath, apk['per_app_repo'])
1651         apksigpath = apkpath + '.sig'
1652         if os.path.exists(apksigpath):
1653             shutil.copy(apksigpath, apk['per_app_repo'])
1654         apkascpath = apkpath + '.asc'
1655         if os.path.exists(apkascpath):
1656             shutil.copy(apkascpath, apk['per_app_repo'])
1657
1658
1659 def create_metadata_from_template(apk):
1660     '''create a new metadata file using internal or external template
1661
1662     Generate warnings for apk's with no metadata (or create skeleton
1663     metadata files, if requested on the command line).  Though the
1664     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1665     since those impose things on the metadata file made from the
1666     template: field sort order, empty field value, formatting, etc.
1667     '''
1668
1669     import yaml
1670     if os.path.exists('template.yml'):
1671         with open('template.yml') as f:
1672             metatxt = f.read()
1673         if 'name' in apk and apk['name'] != '':
1674             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1675                              r'\1 ' + apk['name'],
1676                              metatxt,
1677                              flags=re.IGNORECASE | re.MULTILINE)
1678         else:
1679             logging.warning(_('{appid} does not have a name! Using package name instead.')
1680                             .format(appid=apk['packageName']))
1681             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1682                              r'\1 ' + apk['packageName'],
1683                              metatxt,
1684                              flags=re.IGNORECASE | re.MULTILINE)
1685         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1686             f.write(metatxt)
1687     else:
1688         app = dict()
1689         app['Categories'] = [os.path.basename(os.getcwd())]
1690         # include some blanks as part of the template
1691         app['AuthorName'] = ''
1692         app['Summary'] = ''
1693         app['WebSite'] = ''
1694         app['IssueTracker'] = ''
1695         app['SourceCode'] = ''
1696         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1697         if 'name' in apk and apk['name'] != '':
1698             app['Name'] = apk['name']
1699         else:
1700             logging.warning(_('{appid} does not have a name! Using package name instead.')
1701                             .format(appid=apk['packageName']))
1702             app['Name'] = apk['packageName']
1703         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1704             yaml.dump(app, f, default_flow_style=False)
1705     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1706
1707
1708 config = None
1709 options = None
1710
1711
1712 def main():
1713
1714     global config, options
1715
1716     # Parse command line...
1717     parser = ArgumentParser()
1718     common.setup_global_opts(parser)
1719     parser.add_argument("--create-key", action="store_true", default=False,
1720                         help=_("Add a repo signing key to an unsigned repo"))
1721     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1722                         help=_("Add skeleton metadata files for APKs that are missing them"))
1723     parser.add_argument("--delete-unknown", action="store_true", default=False,
1724                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1725     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1726                         help=_("Report on build data status"))
1727     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1728                         help=_("Interactively ask about things that need updating."))
1729     parser.add_argument("-I", "--icons", action="store_true", default=False,
1730                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1731     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1732                         help=_("Specify editor to use in interactive mode. Default " +
1733                                "is {path}").format(path='/etc/alternatives/editor'))
1734     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1735                         help=_("Update the wiki"))
1736     parser.add_argument("--pretty", action="store_true", default=False,
1737                         help=_("Produce human-readable XML/JSON for index files"))
1738     parser.add_argument("--clean", action="store_true", default=False,
1739                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1740     parser.add_argument("--nosign", action="store_true", default=False,
1741                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1742     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1743                         help=_("Use date from APK instead of current time for newly added APKs"))
1744     parser.add_argument("--rename-apks", action="store_true", default=False,
1745                         help=_("Rename APK files that do not match package.name_123.apk"))
1746     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1747                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1748     metadata.add_metadata_arguments(parser)
1749     options = parser.parse_args()
1750     metadata.warnings_action = options.W
1751
1752     config = common.read_config(options)
1753
1754     if not ('jarsigner' in config and 'keytool' in config):
1755         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1756
1757     repodirs = ['repo']
1758     if config['archive_older'] != 0:
1759         repodirs.append('archive')
1760         if not os.path.exists('archive'):
1761             os.mkdir('archive')
1762
1763     if options.icons:
1764         resize_all_icons(repodirs)
1765         sys.exit(0)
1766
1767     if options.rename_apks:
1768         options.clean = True
1769
1770     # check that icons exist now, rather than fail at the end of `fdroid update`
1771     for k in ['repo_icon', 'archive_icon']:
1772         if k in config:
1773             if not os.path.exists(config[k]):
1774                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1775                                  .format(name=k, path=config[k]))
1776                 sys.exit(1)
1777
1778     # if the user asks to create a keystore, do it now, reusing whatever it can
1779     if options.create_key:
1780         if os.path.exists(config['keystore']):
1781             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1782             logging.critical("\t'" + config['keystore'] + "'")
1783             sys.exit(1)
1784
1785         if 'repo_keyalias' not in config:
1786             config['repo_keyalias'] = socket.getfqdn()
1787             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1788         if 'keydname' not in config:
1789             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1790             common.write_to_config(config, 'keydname', config['keydname'])
1791         if 'keystore' not in config:
1792             config['keystore'] = common.default_config['keystore']
1793             common.write_to_config(config, 'keystore', config['keystore'])
1794
1795         password = common.genpassword()
1796         if 'keystorepass' not in config:
1797             config['keystorepass'] = password
1798             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1799         if 'keypass' not in config:
1800             config['keypass'] = password
1801             common.write_to_config(config, 'keypass', config['keypass'])
1802         common.genkeystore(config)
1803
1804     # Get all apps...
1805     apps = metadata.read_metadata()
1806
1807     # Generate a list of categories...
1808     categories = set()
1809     for app in apps.values():
1810         categories.update(app.Categories)
1811
1812     # Read known apks data (will be updated and written back when we've finished)
1813     knownapks = common.KnownApks()
1814
1815     # Get APK cache
1816     apkcache = get_cache()
1817
1818     # Delete builds for disabled apps
1819     delete_disabled_builds(apps, apkcache, repodirs)
1820
1821     # Scan all apks in the main repo
1822     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1823
1824     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1825                                            options.use_date_from_apk)
1826     cachechanged = cachechanged or fcachechanged
1827     apks += files
1828     for apk in apks:
1829         if apk['packageName'] not in apps:
1830             if options.create_metadata:
1831                 create_metadata_from_template(apk)
1832                 apps = metadata.read_metadata()
1833             else:
1834                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1835                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1836                 if options.delete_unknown:
1837                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1838                                  .format(apkfilename=apk['apkName']))
1839                     rmf = os.path.join(repodirs[0], apk['apkName'])
1840                     if not os.path.exists(rmf):
1841                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1842                     else:
1843                         os.remove(rmf)
1844                 else:
1845                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1846
1847     copy_triple_t_store_metadata(apps)
1848     insert_obbs(repodirs[0], apps, apks)
1849     insert_localized_app_metadata(apps)
1850     translate_per_build_anti_features(apps, apks)
1851
1852     # Scan the archive repo for apks as well
1853     if len(repodirs) > 1:
1854         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1855         if cc:
1856             cachechanged = True
1857     else:
1858         archapks = []
1859
1860     # Apply information from latest apks to the application and update dates
1861     apply_info_from_latest_apk(apps, apks + archapks)
1862
1863     # Sort the app list by name, then the web site doesn't have to by default.
1864     # (we had to wait until we'd scanned the apks to do this, because mostly the
1865     # name comes from there!)
1866     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1867
1868     # APKs are placed into multiple repos based on the app package, providing
1869     # per-app subscription feeds for nightly builds and things like it
1870     if config['per_app_repos']:
1871         add_apks_to_per_app_repos(repodirs[0], apks)
1872         for appid, app in apps.items():
1873             repodir = os.path.join(appid, 'fdroid', 'repo')
1874             appdict = dict()
1875             appdict[appid] = app
1876             if os.path.isdir(repodir):
1877                 index.make(appdict, [appid], apks, repodir, False)
1878             else:
1879                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1880         return
1881
1882     if len(repodirs) > 1:
1883         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1884
1885     # Make the index for the main repo...
1886     index.make(apps, sortedids, apks, repodirs[0], False)
1887     make_categories_txt(repodirs[0], categories)
1888
1889     # If there's an archive repo,  make the index for it. We already scanned it
1890     # earlier on.
1891     if len(repodirs) > 1:
1892         index.make(apps, sortedids, archapks, repodirs[1], True)
1893
1894     git_remote = config.get('binary_transparency_remote')
1895     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1896         from . import btlog
1897         btlog.make_binary_transparency_log(repodirs)
1898
1899     if config['update_stats']:
1900         # Update known apks info...
1901         knownapks.writeifchanged()
1902
1903         # Generate latest apps data for widget
1904         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1905             data = ''
1906             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1907                 for line in f:
1908                     appid = line.rstrip()
1909                     data += appid + "\t"
1910                     app = apps[appid]
1911                     data += app.Name + "\t"
1912                     if app.icon is not None:
1913                         data += app.icon + "\t"
1914                     data += app.License + "\n"
1915             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1916                 f.write(data)
1917
1918     if cachechanged:
1919         write_cache(apkcache)
1920
1921     # Update the wiki...
1922     if options.wiki:
1923         update_wiki(apps, sortedids, apks + archapks)
1924
1925     logging.info(_("Finished"))
1926
1927
1928 if __name__ == "__main__":
1929     main()