fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in sorted(app.AntiFeatures):
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.Bitcoin,
 153             app.Litecoin,
 154             app.License,
 155             requiresroot,
 156             app.AuthorName,
 157             app.AuthorEmail)
 158
 159         if app.Provides:
 160             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 161
 162         wikidata += app.Summary
 163         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 164
 165         wikidata += "=Description=\n"
 166         wikidata += metadata.description_wiki(app.Description) + "\n"
 167
 168         wikidata += "=Maintainer Notes=\n"
 169         if app.MaintainerNotes:
 170             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 171         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 172
 173         # Get a list of all packages for this application...
 174         apklist = []
 175         gotcurrentver = False
 176         cantupdate = False
 177         buildfails = False
 178         for apk in apks:
 179             if apk['packageName'] == appid:
 180                 if str(apk['versionCode']) == app.CurrentVersionCode:
 181                     gotcurrentver = True
 182                 apklist.append(apk)
 183         # Include ones we can't build, as a special case...
 184         for build in app.builds:
 185             if build.disable:
 186                 if build.versionCode == app.CurrentVersionCode:
 187                     cantupdate = True
 188                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 189                 apklist.append({'versionCode': int(build.versionCode),
 190                                 'versionName': build.versionName,
 191                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 192                                 })
 193             else:
 194                 builtit = False
 195                 for apk in apklist:
 196                     if apk['versionCode'] == int(build.versionCode):
 197                         builtit = True
 198                         break
 199                 if not builtit:
 200                     buildfails = True
 201                     apklist.append({'versionCode': int(build.versionCode),
 202                                     'versionName': build.versionName,
 203                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 204                                     })
 205         if app.CurrentVersionCode == '0':
 206             cantupdate = True
 207         # Sort with most recent first...
 208         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 209
 210         wikidata += "=Versions=\n"
 211         if len(apklist) == 0:
 212             wikidata += "We currently have no versions of this app available."
 213         elif not gotcurrentver:
 214             wikidata += "We don't have the current version of this app."
 215         else:
 216             wikidata += "We have the current version of this app."
 217         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 218         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 219         if len(app.NoSourceSince) > 0:
 220             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 221         if len(app.CurrentVersion) > 0:
 222             wikidata += "The current (recommended) version is " + app.CurrentVersion
 223             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 224         validapks = 0
 225         for apk in apklist:
 226             wikidata += "==" + apk['versionName'] + "==\n"
 227
 228             if 'buildproblem' in apk:
 229                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 230             else:
 231                 validapks += 1
 232                 wikidata += "This version is built and signed by "
 233                 if 'srcname' in apk:
 234                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 235                 else:
 236                     wikidata += "the original developer.\n\n"
 237             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 238
 239         wikidata += '\n[[Category:' + wikicat + ']]\n'
 240         if len(app.NoSourceSince) > 0:
 241             wikidata += '\n[[Category:Apps missing source code]]\n'
 242         if validapks == 0 and not app.Disabled:
 243             wikidata += '\n[[Category:Apps with no packages]]\n'
 244         if cantupdate and not app.Disabled:
 245             wikidata += "\n[[Category:Apps we cannot update]]\n"
 246         if buildfails and not app.Disabled:
 247             wikidata += "\n[[Category:Apps with failing builds]]\n"
 248         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 249             wikidata += '\n[[Category:Apps to Update]]\n'
 250         if app.Disabled:
 251             wikidata += '\n[[Category:Apps that are disabled]]\n'
 252         if app.UpdateCheckMode == 'None' and not app.Disabled:
 253             wikidata += '\n[[Category:Apps with no update check]]\n'
 254         for appcat in app.Categories:
 255             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 256
 257         # We can't have underscores in the page name, even if they're in
 258         # the package ID, because MediaWiki messes with them...
 259         pagename = appid.replace('_', ' ')
 260
 261         # Drop a trailing newline, because mediawiki is going to drop it anyway
 262         # and it we don't we'll think the page has changed when it hasn't...
 263         if wikidata.endswith('\n'):
 264             wikidata = wikidata[:-1]
 265
 266         generated_pages[pagename] = wikidata
 267
 268         # Make a redirect from the name to the ID too, unless there's
 269         # already an existing page with the name and it isn't a redirect.
 270         noclobber = False
 271         apppagename = app.Name.replace('_', ' ')
 272         apppagename = apppagename.replace('{', '')
 273         apppagename = apppagename.replace('}', ' ')
 274         apppagename = apppagename.replace(':', ' ')
 275         apppagename = apppagename.replace('[', ' ')
 276         apppagename = apppagename.replace(']', ' ')
 277         # Drop double spaces caused mostly by replacing ':' above
 278         apppagename = apppagename.replace('  ', ' ')
 279         for expagename in site.allpages(prefix=apppagename,
 280                                         filterredir='nonredirects',
 281                                         generator=False):
 282             if expagename == apppagename:
 283                 noclobber = True
 284         # Another reason not to make the redirect page is if the app name
 285         # is the same as it's ID, because that will overwrite the real page
 286         # with an redirect to itself! (Although it seems like an odd
 287         # scenario this happens a lot, e.g. where there is metadata but no
 288         # builds or binaries to extract a name from.
 289         if apppagename == pagename:
 290             noclobber = True
 291         if not noclobber:
 292             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 293
 294     for tcat, genp in [(wikicat, generated_pages),
 295                        (wikiredircat, generated_redirects)]:
 296         catpages = site.Pages['Category:' + tcat]
 297         existingpages = []
 298         for page in catpages:
 299             existingpages.append(page.name)
 300             if page.name in genp:
 301                 pagetxt = page.edit()
 302                 if pagetxt != genp[page.name]:
 303                     logging.debug("Updating modified page " + page.name)
 304                     page.save(genp[page.name], summary='Auto-updated')
 305                 else:
 306                     logging.debug("Page " + page.name + " is unchanged")
 307             else:
 308                 logging.warn("Deleting page " + page.name)
 309                 page.delete('No longer published')
 310         for pagename, text in genp.items():
 311             logging.debug("Checking " + pagename)
 312             if pagename not in existingpages:
 313                 logging.debug("Creating page " + pagename)
 314                 try:
 315                     newpage = site.Pages[pagename]
 316                     newpage.save(text, summary='Auto-created')
 317                 except Exception as e:
 318                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 319
 320     # Purge server cache to ensure counts are up to date
 321     site.pages['Repository Maintenance'].purge()
 322
 323
 324 def delete_disabled_builds(apps, apkcache, repodirs):
 325     """Delete disabled build outputs.
 326
 327     :param apps: list of all applications, as per metadata.read_metadata
 328     :param apkcache: current apk cache information
 329     :param repodirs: the repo directories to process
 330     """
 331     for appid, app in apps.items():
 332         for build in app['builds']:
 333             if not build.disable:
 334                 continue
 335             apkfilename = common.get_release_filename(app, build)
 336             iconfilename = "%s.%s.png" % (
 337                 appid,
 338                 build.versionCode)
 339             for repodir in repodirs:
 340                 files = [
 341                     os.path.join(repodir, apkfilename),
 342                     os.path.join(repodir, apkfilename + '.asc'),
 343                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 344                 ]
 345                 for density in all_screen_densities:
 346                     repo_dir = get_icon_dir(repodir, density)
 347                     files.append(os.path.join(repo_dir, iconfilename))
 348
 349                 for f in files:
 350                     if os.path.exists(f):
 351                         logging.info("Deleting disabled build output " + f)
 352                         os.remove(f)
 353             if apkfilename in apkcache:
 354                 del apkcache[apkfilename]
 355
 356
 357 def resize_icon(iconpath, density):
 358
 359     if not os.path.isfile(iconpath):
 360         return
 361
 362     fp = None
 363     try:
 364         fp = open(iconpath, 'rb')
 365         im = Image.open(fp)
 366         size = dpi_to_px(density)
 367
 368         if any(length > size for length in im.size):
 369             oldsize = im.size
 370             im.thumbnail((size, size), Image.ANTIALIAS)
 371             logging.debug("%s was too large at %s - new size is %s" % (
 372                 iconpath, oldsize, im.size))
 373             im.save(iconpath, "PNG")
 374
 375     except Exception as e:
 376         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 377
 378     finally:
 379         if fp:
 380             fp.close()
 381
 382
 383 def resize_all_icons(repodirs):
 384     """Resize all icons that exceed the max size
 385
 386     :param repodirs: the repo directories to process
 387     """
 388     for repodir in repodirs:
 389         for density in screen_densities:
 390             icon_dir = get_icon_dir(repodir, density)
 391             icon_glob = os.path.join(icon_dir, '*.png')
 392             for iconpath in glob.glob(icon_glob):
 393                 resize_icon(iconpath, density)
 394
 395
 396 def getsig(apkpath):
 397     """ Get the signing certificate of an apk. To get the same md5 has that
 398     Android gets, we encode the .RSA certificate in a specific format and pass
 399     it hex-encoded to the md5 digest algorithm.
 400
 401     :param apkpath: path to the apk
 402     :returns: A string containing the md5 of the signature of the apk or None
 403               if an error occurred.
 404     """
 405
 406     with zipfile.ZipFile(apkpath, 'r') as apk:
 407         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 408
 409         if len(certs) < 1:
 410             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 411             return None
 412         if len(certs) > 1:
 413             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 414             return None
 415
 416         cert = apk.read(certs[0])
 417
 418     cert_encoded = common.get_certificate(cert)
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_cache_file():
 424     return os.path.join('tmp', 'apkcache')
 425
 426
 427 def get_cache():
 428     """Get the cached dict of the APK index
 429
 430     Gather information about all the apk files in the repo directory,
 431     using cached data if possible. Some of the index operations take a
 432     long time, like calculating the SHA-256 and verifying the APK
 433     signature.
 434
 435     The cache is invalidated if the metadata version is different, or
 436     the 'allow_disabled_algorithms' config/option is different.  In
 437     those cases, there is no easy way to know what has changed from
 438     the cache, so just rerun the whole thing.
 439
 440     :return: apkcache
 441
 442     """
 443     apkcachefile = get_cache_file()
 444     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 445     if not options.clean and os.path.exists(apkcachefile):
 446         with open(apkcachefile, 'rb') as cf:
 447             apkcache = pickle.load(cf, encoding='utf-8')
 448         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 449            or apkcache.get('allow_disabled_algorithms') != ada:
 450             apkcache = {}
 451     else:
 452         apkcache = {}
 453
 454     apkcache["METADATA_VERSION"] = METADATA_VERSION
 455     apkcache['allow_disabled_algorithms'] = ada
 456
 457     return apkcache
 458
 459
 460 def write_cache(apkcache):
 461     apkcachefile = get_cache_file()
 462     cache_path = os.path.dirname(apkcachefile)
 463     if not os.path.exists(cache_path):
 464         os.makedirs(cache_path)
 465     with open(apkcachefile, 'wb') as cf:
 466         pickle.dump(apkcache, cf)
 467
 468
 469 def get_icon_bytes(apkzip, iconsrc):
 470     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 471     try:
 472         return apkzip.read(iconsrc)
 473     except KeyError:
 474         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 475
 476
 477 def sha256sum(filename):
 478     '''Calculate the sha256 of the given file'''
 479     sha = hashlib.sha256()
 480     with open(filename, 'rb') as f:
 481         while True:
 482             t = f.read(16384)
 483             if len(t) == 0:
 484                 break
 485             sha.update(t)
 486     return sha.hexdigest()
 487
 488
 489 def has_known_vulnerability(filename):
 490     """checks for known vulnerabilities in the APK
 491
 492     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 493     version.  Google also enforces this:
 494     https://support.google.com/faqs/answer/6376725?hl=en
 495
 496     Checks whether there are more than one classes.dex or AndroidManifest.xml
 497     files, which is invalid and an essential part of the "Master Key" attack.
 498
 499     http://www.saurik.com/id/17
 500     """
 501
 502     # statically load this pattern
 503     if not hasattr(has_known_vulnerability, "pattern"):
 504         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 505
 506     files_in_apk = set()
 507     with zipfile.ZipFile(filename) as zf:
 508         for name in zf.namelist():
 509             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 510                 lib = zf.open(name)
 511                 while True:
 512                     chunk = lib.read(4096)
 513                     if chunk == b'':
 514                         break
 515                     m = has_known_vulnerability.pattern.search(chunk)
 516                     if m:
 517                         version = m.group(1).decode('ascii')
 518                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 519                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 520                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 521                             logging.debug(_('"{path}" contains recent {name} ({version})')
 522                                           .format(path=filename, name=name, version=version))
 523                         else:
 524                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 525                                             .format(path=filename, name=name, version=version))
 526                             return True
 527                         break
 528             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 529                 if name in files_in_apk:
 530                     return True
 531                 files_in_apk.add(name)
 532
 533     return False
 534
 535
 536 def insert_obbs(repodir, apps, apks):
 537     """Scans the .obb files in a given repo directory and adds them to the
 538     relevant APK instances.  OBB files have versionCodes like APK
 539     files, and they are loosely associated.  If there is an OBB file
 540     present, then any APK with the same or higher versionCode will use
 541     that OBB file.  There are two OBB types: main and patch, each APK
 542     can only have only have one of each.
 543
 544     https://developer.android.com/google/play/expansion-files.html
 545
 546     :param repodir: repo directory to scan
 547     :param apps: list of current, valid apps
 548     :param apks: current information on all APKs
 549
 550     """
 551
 552     def obbWarnDelete(f, msg):
 553         logging.warning(msg + ' ' + f)
 554         if options.delete_unknown:
 555             logging.error(_("Deleting unknown file: {path}").format(path=f))
 556             os.remove(f)
 557
 558     obbs = []
 559     java_Integer_MIN_VALUE = -pow(2, 31)
 560     currentPackageNames = apps.keys()
 561     for f in glob.glob(os.path.join(repodir, '*.obb')):
 562         obbfile = os.path.basename(f)
 563         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 564         chunks = obbfile.split('.')
 565         if chunks[0] != 'main' and chunks[0] != 'patch':
 566             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 567             continue
 568         if not re.match(r'^-?[0-9]+$', chunks[1]):
 569             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 570                           .format(name=chunks[0]))
 571             continue
 572         versionCode = int(chunks[1])
 573         packagename = ".".join(chunks[2:-1])
 574
 575         highestVersionCode = java_Integer_MIN_VALUE
 576         if packagename not in currentPackageNames:
 577             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 578             continue
 579         for apk in apks:
 580             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 581                 highestVersionCode = apk['versionCode']
 582         if versionCode > highestVersionCode:
 583             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 584                           .format(integer=str(versionCode)))
 585             continue
 586         obbsha256 = sha256sum(f)
 587         obbs.append((packagename, versionCode, obbfile, obbsha256))
 588
 589     for apk in apks:
 590         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 591             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 592                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 593                     apk['obbMainFile'] = obbfile
 594                     apk['obbMainFileSha256'] = obbsha256
 595                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 596                     apk['obbPatchFile'] = obbfile
 597                     apk['obbPatchFileSha256'] = obbsha256
 598             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 599                 break
 600
 601
 602 def translate_per_build_anti_features(apps, apks):
 603     """Grab the anti-features list from the build metadata
 604
 605     For most Anti-Features, they are really most applicable per-APK,
 606     not for an app.  An app can fix a vulnerability, add/remove
 607     tracking, etc.  This reads the 'antifeatures' list from the Build
 608     entries in the fdroiddata metadata file, then transforms it into
 609     the 'antiFeatures' list of unique items for the index.
 610
 611     The field key is all lower case in the metadata file to match the
 612     rest of the Build fields.  It is 'antiFeatures' camel case in the
 613     implementation, index, and fdroidclient since it is translated
 614     from the build 'antifeatures' field, not directly included.
 615
 616     """
 617
 618     antiFeatures = dict()
 619     for packageName, app in apps.items():
 620         d = dict()
 621         for build in app['builds']:
 622             afl = build.get('antifeatures')
 623             if afl:
 624                 d[int(build.versionCode)] = afl
 625         if len(d) > 0:
 626             antiFeatures[packageName] = d
 627
 628     for apk in apks:
 629         d = antiFeatures.get(apk['packageName'])
 630         if d:
 631             afl = d.get(apk['versionCode'])
 632             if afl:
 633                 apk['antiFeatures'].update(afl)
 634
 635
 636 def _get_localized_dict(app, locale):
 637     '''get the dict to add localized store metadata to'''
 638     if 'localized' not in app:
 639         app['localized'] = collections.OrderedDict()
 640     if locale not in app['localized']:
 641         app['localized'][locale] = collections.OrderedDict()
 642     return app['localized'][locale]
 643
 644
 645 def _set_localized_text_entry(app, locale, key, f):
 646     limit = config['char_limits'][key]
 647     localized = _get_localized_dict(app, locale)
 648     with open(f) as fp:
 649         text = fp.read()[:limit]
 650         if len(text) > 0:
 651             localized[key] = text
 652
 653
 654 def _set_author_entry(app, key, f):
 655     limit = config['char_limits']['author']
 656     with open(f) as fp:
 657         text = fp.read()[:limit]
 658         if len(text) > 0:
 659             app[key] = text
 660
 661
 662 def copy_triple_t_store_metadata(apps):
 663     """Include store metadata from the app's source repo
 664
 665     The Triple-T Gradle Play Publisher is a plugin that has a standard
 666     file layout for all of the metadata and graphics that the Google
 667     Play Store accepts.  Since F-Droid has the git repo, it can just
 668     pluck those files directly.  This method reads any text files into
 669     the app dict, then copies any graphics into the fdroid repo
 670     directory structure.
 671
 672     This needs to be run before insert_localized_app_metadata() so that
 673     the graphics files that are copied into the fdroid repo get
 674     properly indexed.
 675
 676     https://github.com/Triple-T/gradle-play-publisher#upload-images
 677     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 678
 679     """
 680
 681     if not os.path.isdir('build'):
 682         return  # nothing to do
 683
 684     for packageName, app in apps.items():
 685         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 686             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 687             for root, dirs, files in os.walk(d):
 688                 segments = root.split('/')
 689                 locale = segments[-2]
 690                 for f in files:
 691                     if f == 'fulldescription':
 692                         _set_localized_text_entry(app, locale, 'description',
 693                                                   os.path.join(root, f))
 694                         continue
 695                     elif f == 'shortdescription':
 696                         _set_localized_text_entry(app, locale, 'summary',
 697                                                   os.path.join(root, f))
 698                         continue
 699                     elif f == 'title':
 700                         _set_localized_text_entry(app, locale, 'name',
 701                                                   os.path.join(root, f))
 702                         continue
 703                     elif f == 'video':
 704                         _set_localized_text_entry(app, locale, 'video',
 705                                                   os.path.join(root, f))
 706                         continue
 707                     elif f == 'whatsnew':
 708                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 709                                                   os.path.join(root, f))
 710                         continue
 711                     elif f == 'contactEmail':
 712                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 713                         continue
 714                     elif f == 'contactPhone':
 715                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 716                         continue
 717                     elif f == 'contactWebsite':
 718                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 719                         continue
 720
 721                     base, extension = common.get_extension(f)
 722                     dirname = os.path.basename(root)
 723                     if extension in ALLOWED_EXTENSIONS \
 724                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 725                         if segments[-2] == 'listing':
 726                             locale = segments[-3]
 727                         else:
 728                             locale = segments[-2]
 729                         destdir = os.path.join('repo', packageName, locale, dirname)
 730                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 731                         sourcefile = os.path.join(root, f)
 732                         destfile = os.path.join(destdir, os.path.basename(f))
 733                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 734                         shutil.copy(sourcefile, destfile)
 735
 736
 737 def insert_localized_app_metadata(apps):
 738     """scans standard locations for graphics and localized text
 739
 740     Scans for localized description files, store graphics, and
 741     screenshot PNG files in statically defined screenshots directory
 742     and adds them to the app metadata.  The screenshots and graphic
 743     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 744     and must be in the following layout:
 745     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 746
 747     repo/packageName/locale/featureGraphic.png
 748     repo/packageName/locale/phoneScreenshots/1.png
 749     repo/packageName/locale/phoneScreenshots/2.png
 750
 751     The changelog files must be text files named with the versionCode
 752     ending with ".txt" and must be in the following layout:
 753     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 754
 755     repo/packageName/locale/changelogs/12345.txt
 756
 757     This will scan the each app's source repo then the metadata/ dir
 758     for these standard locations of changelog files.  If it finds
 759     them, they will be added to the dict of all packages, with the
 760     versions in the metadata/ folder taking precendence over the what
 761     is in the app's source repo.
 762
 763     Where "packageName" is the app's packageName and "locale" is the locale
 764     of the graphics, e.g. what language they are in, using the IETF RFC5646
 765     format (en-US, fr-CA, es-MX, etc).
 766
 767     This will also scan the app's git for a fastlane folder, and the
 768     metadata/ folder and the apps' source repos for standard locations
 769     of graphic and screenshot files.  If it finds them, it will copy
 770     them into the repo.  The fastlane files follow this pattern:
 771     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 772
 773     """
 774
 775     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 776     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 777     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 778     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 779
 780     for srcd in sorted(sourcedirs):
 781         if not os.path.isdir(srcd):
 782             continue
 783         for root, dirs, files in os.walk(srcd):
 784             segments = root.split('/')
 785             packageName = segments[1]
 786             if packageName not in apps:
 787                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 788                 continue
 789             locale = segments[-1]
 790             destdir = os.path.join('repo', packageName, locale)
 791
 792             # flavours specified in build receipt
 793             build_flavours = ""
 794             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 795                     and 'gradle' in apps[packageName].builds[-1]:
 796                 build_flavours = apps[packageName].builds[-1].gradle
 797
 798             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 799                 logging.debug("ignoring due to wrong flavour")
 800                 continue
 801
 802             for f in files:
 803                 if f in ('description.txt', 'full_description.txt'):
 804                     _set_localized_text_entry(apps[packageName], locale, 'description',
 805                                               os.path.join(root, f))
 806                     continue
 807                 elif f in ('summary.txt', 'short_description.txt'):
 808                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 809                                               os.path.join(root, f))
 810                     continue
 811                 elif f in ('name.txt', 'title.txt'):
 812                     _set_localized_text_entry(apps[packageName], locale, 'name',
 813                                               os.path.join(root, f))
 814                     continue
 815                 elif f == 'video.txt':
 816                     _set_localized_text_entry(apps[packageName], locale, 'video',
 817                                               os.path.join(root, f))
 818                     continue
 819                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 820                     locale = segments[-2]
 821                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 822                                               os.path.join(root, f))
 823                     continue
 824
 825                 base, extension = common.get_extension(f)
 826                 if locale == 'images':
 827                     locale = segments[-2]
 828                     destdir = os.path.join('repo', packageName, locale)
 829                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 830                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 831                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 832                     shutil.copy(os.path.join(root, f), destdir)
 833             for d in dirs:
 834                 if d in SCREENSHOT_DIRS:
 835                     if locale == 'images':
 836                         locale = segments[-2]
 837                         destdir = os.path.join('repo', packageName, locale)
 838                     for f in glob.glob(os.path.join(root, d, '*.*')):
 839                         _ignored, extension = common.get_extension(f)
 840                         if extension in ALLOWED_EXTENSIONS:
 841                             screenshotdestdir = os.path.join(destdir, d)
 842                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 843                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 844                             shutil.copy(f, screenshotdestdir)
 845
 846     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 847     for d in repofiles:
 848         if not os.path.isdir(d):
 849             continue
 850         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 851             if not os.path.isfile(f):
 852                 continue
 853             segments = f.split('/')
 854             packageName = segments[1]
 855             locale = segments[2]
 856             screenshotdir = segments[3]
 857             filename = os.path.basename(f)
 858             base, extension = common.get_extension(filename)
 859
 860             if packageName not in apps:
 861                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 862                                 .format(path=filename, name=packageName))
 863                 continue
 864             graphics = _get_localized_dict(apps[packageName], locale)
 865
 866             if extension not in ALLOWED_EXTENSIONS:
 867                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 868             elif base in GRAPHIC_NAMES:
 869                 # there can only be zero or one of these per locale
 870                 graphics[base] = filename
 871             elif screenshotdir in SCREENSHOT_DIRS:
 872                 # there can any number of these per locale
 873                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 874                 if screenshotdir not in graphics:
 875                     graphics[screenshotdir] = []
 876                 graphics[screenshotdir].append(filename)
 877             else:
 878                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 879
 880
 881 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 882     """Scan a repo for all files with an extension except APK/OBB
 883
 884     :param apkcache: current cached info about all repo files
 885     :param repodir: repo directory to scan
 886     :param knownapks: list of all known files, as per metadata.read_metadata
 887     :param use_date_from_file: use date from file (instead of current date)
 888                                for newly added files
 889     """
 890
 891     cachechanged = False
 892     repo_files = []
 893     repodir = repodir.encode('utf-8')
 894     for name in os.listdir(repodir):
 895         file_extension = common.get_file_extension(name)
 896         if file_extension == 'apk' or file_extension == 'obb':
 897             continue
 898         filename = os.path.join(repodir, name)
 899         name_utf8 = name.decode('utf-8')
 900         if filename.endswith(b'_src.tar.gz'):
 901             logging.debug(_('skipping source tarball: {path}')
 902                           .format(path=filename.decode('utf-8')))
 903             continue
 904         if not common.is_repo_file(filename):
 905             continue
 906         stat = os.stat(filename)
 907         if stat.st_size == 0:
 908             raise FDroidException(_('{path} is zero size!')
 909                                   .format(path=filename))
 910
 911         shasum = sha256sum(filename)
 912         usecache = False
 913         if name in apkcache:
 914             repo_file = apkcache[name]
 915             # added time is cached as tuple but used here as datetime instance
 916             if 'added' in repo_file:
 917                 a = repo_file['added']
 918                 if isinstance(a, datetime):
 919                     repo_file['added'] = a
 920                 else:
 921                     repo_file['added'] = datetime(*a[:6])
 922             if repo_file.get('hash') == shasum:
 923                 logging.debug(_("Reading {apkfilename} from cache")
 924                               .format(apkfilename=name_utf8))
 925                 usecache = True
 926             else:
 927                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 928                               .format(apkfilename=name_utf8))
 929
 930         if not usecache:
 931             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 932             repo_file = collections.OrderedDict()
 933             repo_file['name'] = os.path.splitext(name_utf8)[0]
 934             # TODO rename apkname globally to something more generic
 935             repo_file['apkName'] = name_utf8
 936             repo_file['hash'] = shasum
 937             repo_file['hashType'] = 'sha256'
 938             repo_file['versionCode'] = 0
 939             repo_file['versionName'] = shasum
 940             # the static ID is the SHA256 unless it is set in the metadata
 941             repo_file['packageName'] = shasum
 942
 943             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 944             if m:
 945                 repo_file['packageName'] = m.group(1)
 946                 repo_file['versionCode'] = int(m.group(2))
 947             srcfilename = name + b'_src.tar.gz'
 948             if os.path.exists(os.path.join(repodir, srcfilename)):
 949                 repo_file['srcname'] = srcfilename.decode('utf-8')
 950             repo_file['size'] = stat.st_size
 951
 952             apkcache[name] = repo_file
 953             cachechanged = True
 954
 955         if use_date_from_file:
 956             timestamp = stat.st_ctime
 957             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 958         else:
 959             default_date_param = None
 960
 961         # Record in knownapks, getting the added date at the same time..
 962         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 963                                     default_date=default_date_param)
 964         if added:
 965             repo_file['added'] = added
 966
 967         repo_files.append(repo_file)
 968
 969     return repo_files, cachechanged
 970
 971
 972 def scan_apk(apk_file):
 973     """
 974     Scans an APK file and returns dictionary with metadata of the APK.
 975
 976     Attention: This does *not* verify that the APK signature is correct.
 977
 978     :param apk_file: The (ideally absolute) path to the APK file
 979     :raises BuildException
 980     :return A dict containing APK metadata
 981     """
 982     apk = {
 983         'hash': sha256sum(apk_file),
 984         'hashType': 'sha256',
 985         'uses-permission': [],
 986         'uses-permission-sdk-23': [],
 987         'features': [],
 988         'icons_src': {},
 989         'icons': {},
 990         'antiFeatures': set(),
 991     }
 992
 993     if SdkToolsPopen(['aapt', 'version'], output=False):
 994         scan_apk_aapt(apk, apk_file)
 995     else:
 996         scan_apk_androguard(apk, apk_file)
 997
 998     # Get the signature, or rather the signing key fingerprints
 999     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1000     apk['sig'] = getsig(apk_file)
1001     if not apk['sig']:
1002         raise BuildException("Failed to get apk signature")
1003     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1004                                                                apk_file))
1005     if not apk.get('signer'):
1006         raise BuildException("Failed to get apk signing key fingerprint")
1007
1008     # Get size of the APK
1009     apk['size'] = os.path.getsize(apk_file)
1010
1011     if 'minSdkVersion' not in apk:
1012         logging.warning("No SDK version information found in {0}".format(apk_file))
1013         apk['minSdkVersion'] = 1
1014
1015     # Check for known vulnerabilities
1016     if has_known_vulnerability(apk_file):
1017         apk['antiFeatures'].add('KnownVuln')
1018
1019     return apk
1020
1021
1022 def scan_apk_aapt(apk, apkfile):
1023     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1024     if p.returncode != 0:
1025         if options.delete_unknown:
1026             if os.path.exists(apkfile):
1027                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1028                 os.remove(apkfile)
1029             else:
1030                 logging.error("Could not find {0} to remove it".format(apkfile))
1031         else:
1032             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1033         raise BuildException(_("Invalid APK"))
1034     for line in p.output.splitlines():
1035         if line.startswith("package:"):
1036             try:
1037                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1038                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1039                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1040             except Exception as e:
1041                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1042         elif line.startswith("application:"):
1043             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1044             # Keep path to non-dpi icon in case we need it
1045             match = re.match(APK_ICON_PAT_NODPI, line)
1046             if match:
1047                 apk['icons_src']['-1'] = match.group(1)
1048         elif line.startswith("launchable-activity:"):
1049             # Only use launchable-activity as fallback to application
1050             if not apk['name']:
1051                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1052             if '-1' not in apk['icons_src']:
1053                 match = re.match(APK_ICON_PAT_NODPI, line)
1054                 if match:
1055                     apk['icons_src']['-1'] = match.group(1)
1056         elif line.startswith("application-icon-"):
1057             match = re.match(APK_ICON_PAT, line)
1058             if match:
1059                 density = match.group(1)
1060                 path = match.group(2)
1061                 apk['icons_src'][density] = path
1062         elif line.startswith("sdkVersion:"):
1063             m = re.match(APK_SDK_VERSION_PAT, line)
1064             if m is None:
1065                 logging.error(line.replace('sdkVersion:', '')
1066                               + ' is not a valid minSdkVersion!')
1067             else:
1068                 apk['minSdkVersion'] = m.group(1)
1069                 # if target not set, default to min
1070                 if 'targetSdkVersion' not in apk:
1071                     apk['targetSdkVersion'] = m.group(1)
1072         elif line.startswith("targetSdkVersion:"):
1073             m = re.match(APK_SDK_VERSION_PAT, line)
1074             if m is None:
1075                 logging.error(line.replace('targetSdkVersion:', '')
1076                               + ' is not a valid targetSdkVersion!')
1077             else:
1078                 apk['targetSdkVersion'] = m.group(1)
1079         elif line.startswith("maxSdkVersion:"):
1080             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1081         elif line.startswith("native-code:"):
1082             apk['nativecode'] = []
1083             for arch in line[13:].split(' '):
1084                 apk['nativecode'].append(arch[1:-1])
1085         elif line.startswith('uses-permission:'):
1086             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1087             if perm_match['maxSdkVersion']:
1088                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1089             permission = UsesPermission(
1090                 perm_match['name'],
1091                 perm_match['maxSdkVersion']
1092             )
1093
1094             apk['uses-permission'].append(permission)
1095         elif line.startswith('uses-permission-sdk-23:'):
1096             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1097             if perm_match['maxSdkVersion']:
1098                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1099             permission_sdk_23 = UsesPermissionSdk23(
1100                 perm_match['name'],
1101                 perm_match['maxSdkVersion']
1102             )
1103
1104             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1105
1106         elif line.startswith('uses-feature:'):
1107             feature = re.match(APK_FEATURE_PAT, line).group(1)
1108             # Filter out this, it's only added with the latest SDK tools and
1109             # causes problems for lots of apps.
1110             if feature != "android.hardware.screen.portrait" \
1111                     and feature != "android.hardware.screen.landscape":
1112                 if feature.startswith("android.feature."):
1113                     feature = feature[16:]
1114                 apk['features'].add(feature)
1115
1116
1117 def scan_apk_androguard(apk, apkfile):
1118     try:
1119         from androguard.core.bytecodes.apk import APK
1120         apkobject = APK(apkfile)
1121         if apkobject.is_valid_APK():
1122             arsc = apkobject.get_android_resources()
1123         else:
1124             if options.delete_unknown:
1125                 if os.path.exists(apkfile):
1126                     logging.error(_("Failed to get apk information, deleting {path}")
1127                                   .format(path=apkfile))
1128                     os.remove(apkfile)
1129                 else:
1130                     logging.error(_("Could not find {path} to remove it")
1131                                   .format(path=apkfile))
1132             else:
1133                 logging.error(_("Failed to get apk information, skipping {path}")
1134                               .format(path=apkfile))
1135             raise BuildException(_("Invalid APK"))
1136     except ImportError:
1137         raise FDroidException("androguard library is not installed and aapt not present")
1138     except FileNotFoundError:
1139         logging.error(_("Could not open apk file for analysis"))
1140         raise BuildException(_("Invalid APK"))
1141
1142     apk['packageName'] = apkobject.get_package()
1143     apk['versionCode'] = int(apkobject.get_androidversion_code())
1144     apk['versionName'] = apkobject.get_androidversion_name()
1145     if apk['versionName'][0] == "@":
1146         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1147         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1148         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1149     apk['name'] = apkobject.get_app_name()
1150
1151     if apkobject.get_max_sdk_version() is not None:
1152         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1153     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1154     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1155
1156     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1157     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1158
1159     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1160
1161     for file in apkobject.get_files():
1162         d_re = density_re.match(file)
1163         if d_re:
1164             folder = d_re.group(1).split('-')
1165             if len(folder) > 1:
1166                 resolution = folder[1]
1167             else:
1168                 resolution = 'mdpi'
1169             density = screen_resolutions[resolution]
1170             apk['icons_src'][density] = d_re.group(0)
1171
1172     if apk['icons_src'].get('-1') is None:
1173         apk['icons_src']['-1'] = apk['icons_src']['160']
1174
1175     arch_re = re.compile("^lib/(.*)/.*$")
1176     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1177     if len(arch) >= 1:
1178         apk['nativecode'] = []
1179         apk['nativecode'].extend(sorted(list(arch)))
1180
1181     xml = apkobject.get_android_manifest_xml()
1182
1183     for item in xml.getElementsByTagName('uses-permission'):
1184         name = str(item.getAttribute("android:name"))
1185         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1186         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1187         permission = UsesPermission(
1188             name,
1189             maxSdkVersion
1190         )
1191         apk['uses-permission'].append(permission)
1192
1193     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1194         name = str(item.getAttribute("android:name"))
1195         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1196         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1197         permission_sdk_23 = UsesPermissionSdk23(
1198             name,
1199             maxSdkVersion
1200         )
1201         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1202
1203     for item in xml.getElementsByTagName('uses-feature'):
1204         feature = str(item.getAttribute("android:name"))
1205         if feature != "android.hardware.screen.portrait" \
1206                 and feature != "android.hardware.screen.landscape":
1207             if feature.startswith("android.feature."):
1208                 feature = feature[16:]
1209         apk['features'].append(feature)
1210
1211
1212 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1213                 allow_disabled_algorithms=False, archive_bad_sig=False):
1214     """Processes the apk with the given filename in the given repo directory.
1215
1216     This also extracts the icons.
1217
1218     :param apkcache: current apk cache information
1219     :param apkfilename: the filename of the apk to scan
1220     :param repodir: repo directory to scan
1221     :param knownapks: known apks info
1222     :param use_date_from_apk: use date from APK (instead of current date)
1223                               for newly added APKs
1224     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1225                                       disabled algorithms in the signature (e.g. MD5)
1226     :param archive_bad_sig: move APKs with a bad signature to the archive
1227     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1228      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1229     """
1230
1231     apk = {}
1232     apkfile = os.path.join(repodir, apkfilename)
1233
1234     cachechanged = False
1235     usecache = False
1236     if apkfilename in apkcache:
1237         apk = apkcache[apkfilename]
1238         if apk.get('hash') == sha256sum(apkfile):
1239             logging.debug(_("Reading {apkfilename} from cache")
1240                           .format(apkfilename=apkfilename))
1241             usecache = True
1242         else:
1243             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1244                           .format(apkfilename=apkfilename))
1245
1246     if not usecache:
1247         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1248
1249         try:
1250             apk = scan_apk(apkfile)
1251         except BuildException:
1252             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1253                             .format(apkfilename=apkfilename))
1254             return True, None, False
1255
1256         # Check for debuggable apks...
1257         if common.isApkAndDebuggable(apkfile):
1258             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1259
1260         if options.rename_apks:
1261             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1262             std_short_name = os.path.join(repodir, n)
1263             if apkfile != std_short_name:
1264                 if os.path.exists(std_short_name):
1265                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1266                     if apkfile != std_long_name:
1267                         if os.path.exists(std_long_name):
1268                             dupdir = os.path.join('duplicates', repodir)
1269                             if not os.path.isdir(dupdir):
1270                                 os.makedirs(dupdir, exist_ok=True)
1271                             dupfile = os.path.join('duplicates', std_long_name)
1272                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1273                             os.rename(apkfile, dupfile)
1274                             return True, None, False
1275                         else:
1276                             os.rename(apkfile, std_long_name)
1277                     apkfile = std_long_name
1278                 else:
1279                     os.rename(apkfile, std_short_name)
1280                     apkfile = std_short_name
1281                 apkfilename = apkfile[len(repodir) + 1:]
1282
1283         apk['apkName'] = apkfilename
1284         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1285         if os.path.exists(os.path.join(repodir, srcfilename)):
1286             apk['srcname'] = srcfilename
1287
1288         # verify the jar signature is correct, allow deprecated
1289         # algorithms only if the APK is in the archive.
1290         skipapk = False
1291         if not common.verify_apk_signature(apkfile):
1292             if repodir == 'archive' or allow_disabled_algorithms:
1293                 if common.verify_old_apk_signature(apkfile):
1294                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1295                 else:
1296                     skipapk = True
1297             else:
1298                 skipapk = True
1299
1300         if skipapk:
1301             if archive_bad_sig:
1302                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1303                                 .format(apkfilename=apkfilename))
1304                 move_apk_between_sections(repodir, 'archive', apk)
1305             else:
1306                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1307                                 .format(apkfilename=apkfilename))
1308             return True, None, False
1309
1310         apkzip = zipfile.ZipFile(apkfile, 'r')
1311
1312         manifest = apkzip.getinfo('AndroidManifest.xml')
1313         if manifest.date_time[1] == 0:  # month can't be zero
1314             logging.debug(_('AndroidManifest.xml has no date'))
1315         else:
1316             common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1317
1318         # extract icons from APK zip file
1319         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1320         try:
1321             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1322         finally:
1323             apkzip.close()  # ensure that APK zip file gets closed
1324
1325         # resize existing icons for densities missing in the APK
1326         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1327
1328         if use_date_from_apk and manifest.date_time[1] != 0:
1329             default_date_param = datetime(*manifest.date_time)
1330         else:
1331             default_date_param = None
1332
1333         # Record in known apks, getting the added date at the same time..
1334         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1335                                     default_date=default_date_param)
1336         if added:
1337             apk['added'] = added
1338
1339         apkcache[apkfilename] = apk
1340         cachechanged = True
1341
1342     return False, apk, cachechanged
1343
1344
1345 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1346     """Processes the apks in the given repo directory.
1347
1348     This also extracts the icons.
1349
1350     :param apkcache: current apk cache information
1351     :param repodir: repo directory to scan
1352     :param knownapks: known apks info
1353     :param use_date_from_apk: use date from APK (instead of current date)
1354                               for newly added APKs
1355     :returns: (apks, cachechanged) where apks is a list of apk information,
1356               and cachechanged is True if the apkcache got changed.
1357     """
1358
1359     cachechanged = False
1360
1361     for icon_dir in get_all_icon_dirs(repodir):
1362         if os.path.exists(icon_dir):
1363             if options.clean:
1364                 shutil.rmtree(icon_dir)
1365                 os.makedirs(icon_dir)
1366         else:
1367             os.makedirs(icon_dir)
1368
1369     apks = []
1370     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1371         apkfilename = apkfile[len(repodir) + 1:]
1372         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1373         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1374                                              use_date_from_apk, ada, True)
1375         if skip:
1376             continue
1377         apks.append(apk)
1378         cachechanged = cachechanged or cachethis
1379
1380     return apks, cachechanged
1381
1382
1383 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1384     """
1385     Extracts icons from the given APK zip in various densities,
1386     saves them into given repo directory
1387     and stores their names in the APK metadata dictionary.
1388
1389     :param icon_filename: A string representing the icon's file name
1390     :param apk: A populated dictionary containing APK metadata.
1391                 Needs to have 'icons_src' key
1392     :param apkzip: An opened zipfile.ZipFile of the APK file
1393     :param repo_dir: The directory of the APK's repository
1394     :return: A list of icon densities that are missing
1395     """
1396     empty_densities = []
1397     for density in screen_densities:
1398         if density not in apk['icons_src']:
1399             empty_densities.append(density)
1400             continue
1401         icon_src = apk['icons_src'][density]
1402         icon_dir = get_icon_dir(repo_dir, density)
1403         icon_dest = os.path.join(icon_dir, icon_filename)
1404
1405         # Extract the icon files per density
1406         if icon_src.endswith('.xml'):
1407             png = os.path.basename(icon_src)[:-4] + '.png'
1408             for f in apkzip.namelist():
1409                 if f.endswith(png):
1410                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1411                     if m and screen_resolutions[m.group(2)] == density:
1412                         icon_src = f
1413             if icon_src.endswith('.xml'):
1414                 empty_densities.append(density)
1415                 continue
1416         try:
1417             with open(icon_dest, 'wb') as f:
1418                 f.write(get_icon_bytes(apkzip, icon_src))
1419             apk['icons'][density] = icon_filename
1420         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1421             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1422             del apk['icons_src'][density]
1423             empty_densities.append(density)
1424
1425     if '-1' in apk['icons_src']:
1426         icon_src = apk['icons_src']['-1']
1427         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1428         with open(icon_path, 'wb') as f:
1429             f.write(get_icon_bytes(apkzip, icon_src))
1430         try:
1431             im = Image.open(icon_path)
1432             dpi = px_to_dpi(im.size[0])
1433             for density in screen_densities:
1434                 if density in apk['icons']:
1435                     break
1436                 if density == screen_densities[-1] or dpi >= int(density):
1437                     apk['icons'][density] = icon_filename
1438                     shutil.move(icon_path,
1439                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1440                     empty_densities.remove(density)
1441                     break
1442         except Exception as e:
1443             logging.warning(_("Failed reading {path}: {error}")
1444                             .format(path=icon_path, error=e))
1445
1446     if apk['icons']:
1447         apk['icon'] = icon_filename
1448
1449     return empty_densities
1450
1451
1452 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1453     """
1454     Resize existing icons for densities missing in the APK to ensure all densities are available
1455
1456     :param empty_densities: A list of icon densities that are missing
1457     :param icon_filename: A string representing the icon's file name
1458     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1459     :param repo_dir: The directory of the APK's repository
1460     """
1461     # First try resizing down to not lose quality
1462     last_density = None
1463     for density in screen_densities:
1464         if density not in empty_densities:
1465             last_density = density
1466             continue
1467         if last_density is None:
1468             continue
1469         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1470
1471         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1472         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1473         fp = None
1474         try:
1475             fp = open(last_icon_path, 'rb')
1476             im = Image.open(fp)
1477
1478             size = dpi_to_px(density)
1479
1480             im.thumbnail((size, size), Image.ANTIALIAS)
1481             im.save(icon_path, "PNG")
1482             empty_densities.remove(density)
1483         except Exception as e:
1484             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1485         finally:
1486             if fp:
1487                 fp.close()
1488
1489     # Then just copy from the highest resolution available
1490     last_density = None
1491     for density in reversed(screen_densities):
1492         if density not in empty_densities:
1493             last_density = density
1494             continue
1495
1496         if last_density is None:
1497             continue
1498
1499         shutil.copyfile(
1500             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1501             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1502         )
1503         empty_densities.remove(density)
1504
1505     for density in screen_densities:
1506         icon_dir = get_icon_dir(repo_dir, density)
1507         icon_dest = os.path.join(icon_dir, icon_filename)
1508         resize_icon(icon_dest, density)
1509
1510     # Copy from icons-mdpi to icons since mdpi is the baseline density
1511     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1512     if os.path.isfile(baseline):
1513         apk['icons']['0'] = icon_filename
1514         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1515
1516
1517 def apply_info_from_latest_apk(apps, apks):
1518     """
1519     Some information from the apks needs to be applied up to the application level.
1520     When doing this, we use the info from the most recent version's apk.
1521     We deal with figuring out when the app was added and last updated at the same time.
1522     """
1523     for appid, app in apps.items():
1524         bestver = UNSET_VERSION_CODE
1525         for apk in apks:
1526             if apk['packageName'] == appid:
1527                 if apk['versionCode'] > bestver:
1528                     bestver = apk['versionCode']
1529                     bestapk = apk
1530
1531                 if 'added' in apk:
1532                     if not app.added or apk['added'] < app.added:
1533                         app.added = apk['added']
1534                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1535                         app.lastUpdated = apk['added']
1536
1537         if not app.added:
1538             logging.debug("Don't know when " + appid + " was added")
1539         if not app.lastUpdated:
1540             logging.debug("Don't know when " + appid + " was last updated")
1541
1542         if bestver == UNSET_VERSION_CODE:
1543
1544             if app.Name is None:
1545                 app.Name = app.AutoName or appid
1546             app.icon = None
1547             logging.debug("Application " + appid + " has no packages")
1548         else:
1549             if app.Name is None:
1550                 app.Name = bestapk['name']
1551             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1552             if app.CurrentVersionCode is None:
1553                 app.CurrentVersionCode = str(bestver)
1554
1555
1556 def make_categories_txt(repodir, categories):
1557     '''Write a category list in the repo to allow quick access'''
1558     catdata = ''
1559     for cat in sorted(categories):
1560         catdata += cat + '\n'
1561     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1562         f.write(catdata)
1563
1564
1565 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1566
1567     def filter_apk_list_sorted(apk_list):
1568         res = []
1569         for apk in apk_list:
1570             if apk['packageName'] == appid:
1571                 res.append(apk)
1572
1573         # Sort the apk list by version code. First is highest/newest.
1574         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1575
1576     for appid, app in apps.items():
1577
1578         if app.ArchivePolicy:
1579             keepversions = int(app.ArchivePolicy[:-9])
1580         else:
1581             keepversions = defaultkeepversions
1582
1583         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1584                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1585
1586         current_app_apks = filter_apk_list_sorted(apks)
1587         if len(current_app_apks) > keepversions:
1588             # Move back the ones we don't want.
1589             for apk in current_app_apks[keepversions:]:
1590                 move_apk_between_sections(repodir, archivedir, apk)
1591                 archapks.append(apk)
1592                 apks.remove(apk)
1593
1594         current_app_archapks = filter_apk_list_sorted(archapks)
1595         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1596             kept = 0
1597             # Move forward the ones we want again, except DisableAlgorithm
1598             for apk in current_app_archapks:
1599                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1600                     move_apk_between_sections(archivedir, repodir, apk)
1601                     archapks.remove(apk)
1602                     apks.append(apk)
1603                     kept += 1
1604                 if kept == keepversions:
1605                     break
1606
1607
1608 def move_apk_between_sections(from_dir, to_dir, apk):
1609     """move an APK from repo to archive or vice versa"""
1610
1611     def _move_file(from_dir, to_dir, filename, ignore_missing):
1612         from_path = os.path.join(from_dir, filename)
1613         if ignore_missing and not os.path.exists(from_path):
1614             return
1615         to_path = os.path.join(to_dir, filename)
1616         if not os.path.exists(to_dir):
1617             os.mkdir(to_dir)
1618         shutil.move(from_path, to_path)
1619
1620     if from_dir == to_dir:
1621         return
1622
1623     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1624     _move_file(from_dir, to_dir, apk['apkName'], False)
1625     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1626     for density in all_screen_densities:
1627         from_icon_dir = get_icon_dir(from_dir, density)
1628         to_icon_dir = get_icon_dir(to_dir, density)
1629         if density not in apk.get('icons', []):
1630             continue
1631         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1632     if 'srcname' in apk:
1633         _move_file(from_dir, to_dir, apk['srcname'], False)
1634
1635
1636 def add_apks_to_per_app_repos(repodir, apks):
1637     apks_per_app = dict()
1638     for apk in apks:
1639         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1640         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1641         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1642         apks_per_app[apk['packageName']] = apk
1643
1644         if not os.path.exists(apk['per_app_icons']):
1645             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1646             os.makedirs(apk['per_app_icons'])
1647
1648         apkpath = os.path.join(repodir, apk['apkName'])
1649         shutil.copy(apkpath, apk['per_app_repo'])
1650         apksigpath = apkpath + '.sig'
1651         if os.path.exists(apksigpath):
1652             shutil.copy(apksigpath, apk['per_app_repo'])
1653         apkascpath = apkpath + '.asc'
1654         if os.path.exists(apkascpath):
1655             shutil.copy(apkascpath, apk['per_app_repo'])
1656
1657
1658 def create_metadata_from_template(apk):
1659     '''create a new metadata file using internal or external template
1660
1661     Generate warnings for apk's with no metadata (or create skeleton
1662     metadata files, if requested on the command line).  Though the
1663     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1664     since those impose things on the metadata file made from the
1665     template: field sort order, empty field value, formatting, etc.
1666     '''
1667
1668     import yaml
1669     if os.path.exists('template.yml'):
1670         with open('template.yml') as f:
1671             metatxt = f.read()
1672         if 'name' in apk and apk['name'] != '':
1673             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1674                              r'\1 ' + apk['name'],
1675                              metatxt,
1676                              flags=re.IGNORECASE | re.MULTILINE)
1677         else:
1678             logging.warning(_('{appid} does not have a name! Using package name instead.')
1679                             .format(appid=apk['packageName']))
1680             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1681                              r'\1 ' + apk['packageName'],
1682                              metatxt,
1683                              flags=re.IGNORECASE | re.MULTILINE)
1684         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1685             f.write(metatxt)
1686     else:
1687         app = dict()
1688         app['Categories'] = [os.path.basename(os.getcwd())]
1689         # include some blanks as part of the template
1690         app['AuthorName'] = ''
1691         app['Summary'] = ''
1692         app['WebSite'] = ''
1693         app['IssueTracker'] = ''
1694         app['SourceCode'] = ''
1695         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1696         if 'name' in apk and apk['name'] != '':
1697             app['Name'] = apk['name']
1698         else:
1699             logging.warning(_('{appid} does not have a name! Using package name instead.')
1700                             .format(appid=apk['packageName']))
1701             app['Name'] = apk['packageName']
1702         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1703             yaml.dump(app, f, default_flow_style=False)
1704     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1705
1706
1707 config = None
1708 options = None
1709
1710
1711 def main():
1712
1713     global config, options
1714
1715     # Parse command line...
1716     parser = ArgumentParser()
1717     common.setup_global_opts(parser)
1718     parser.add_argument("--create-key", action="store_true", default=False,
1719                         help=_("Add a repo signing key to an unsigned repo"))
1720     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1721                         help=_("Add skeleton metadata files for APKs that are missing them"))
1722     parser.add_argument("--delete-unknown", action="store_true", default=False,
1723                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1724     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1725                         help=_("Report on build data status"))
1726     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1727                         help=_("Interactively ask about things that need updating."))
1728     parser.add_argument("-I", "--icons", action="store_true", default=False,
1729                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1730     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1731                         help=_("Specify editor to use in interactive mode. Default " +
1732                                "is {path}").format(path='/etc/alternatives/editor'))
1733     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1734                         help=_("Update the wiki"))
1735     parser.add_argument("--pretty", action="store_true", default=False,
1736                         help=_("Produce human-readable XML/JSON for index files"))
1737     parser.add_argument("--clean", action="store_true", default=False,
1738                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1739     parser.add_argument("--nosign", action="store_true", default=False,
1740                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1741     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1742                         help=_("Use date from APK instead of current time for newly added APKs"))
1743     parser.add_argument("--rename-apks", action="store_true", default=False,
1744                         help=_("Rename APK files that do not match package.name_123.apk"))
1745     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1746                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1747     metadata.add_metadata_arguments(parser)
1748     options = parser.parse_args()
1749     metadata.warnings_action = options.W
1750
1751     config = common.read_config(options)
1752
1753     if not ('jarsigner' in config and 'keytool' in config):
1754         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1755
1756     repodirs = ['repo']
1757     if config['archive_older'] != 0:
1758         repodirs.append('archive')
1759         if not os.path.exists('archive'):
1760             os.mkdir('archive')
1761
1762     if options.icons:
1763         resize_all_icons(repodirs)
1764         sys.exit(0)
1765
1766     if options.rename_apks:
1767         options.clean = True
1768
1769     # check that icons exist now, rather than fail at the end of `fdroid update`
1770     for k in ['repo_icon', 'archive_icon']:
1771         if k in config:
1772             if not os.path.exists(config[k]):
1773                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1774                                  .format(name=k, path=config[k]))
1775                 sys.exit(1)
1776
1777     # if the user asks to create a keystore, do it now, reusing whatever it can
1778     if options.create_key:
1779         if os.path.exists(config['keystore']):
1780             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1781             logging.critical("\t'" + config['keystore'] + "'")
1782             sys.exit(1)
1783
1784         if 'repo_keyalias' not in config:
1785             config['repo_keyalias'] = socket.getfqdn()
1786             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1787         if 'keydname' not in config:
1788             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1789             common.write_to_config(config, 'keydname', config['keydname'])
1790         if 'keystore' not in config:
1791             config['keystore'] = common.default_config['keystore']
1792             common.write_to_config(config, 'keystore', config['keystore'])
1793
1794         password = common.genpassword()
1795         if 'keystorepass' not in config:
1796             config['keystorepass'] = password
1797             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1798         if 'keypass' not in config:
1799             config['keypass'] = password
1800             common.write_to_config(config, 'keypass', config['keypass'])
1801         common.genkeystore(config)
1802
1803     # Get all apps...
1804     apps = metadata.read_metadata()
1805
1806     # Generate a list of categories...
1807     categories = set()
1808     for app in apps.values():
1809         categories.update(app.Categories)
1810
1811     # Read known apks data (will be updated and written back when we've finished)
1812     knownapks = common.KnownApks()
1813
1814     # Get APK cache
1815     apkcache = get_cache()
1816
1817     # Delete builds for disabled apps
1818     delete_disabled_builds(apps, apkcache, repodirs)
1819
1820     # Scan all apks in the main repo
1821     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1822
1823     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1824                                            options.use_date_from_apk)
1825     cachechanged = cachechanged or fcachechanged
1826     apks += files
1827     for apk in apks:
1828         if apk['packageName'] not in apps:
1829             if options.create_metadata:
1830                 create_metadata_from_template(apk)
1831                 apps = metadata.read_metadata()
1832             else:
1833                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1834                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1835                 if options.delete_unknown:
1836                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1837                                  .format(apkfilename=apk['apkName']))
1838                     rmf = os.path.join(repodirs[0], apk['apkName'])
1839                     if not os.path.exists(rmf):
1840                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1841                     else:
1842                         os.remove(rmf)
1843                 else:
1844                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1845
1846     copy_triple_t_store_metadata(apps)
1847     insert_obbs(repodirs[0], apps, apks)
1848     insert_localized_app_metadata(apps)
1849     translate_per_build_anti_features(apps, apks)
1850
1851     # Scan the archive repo for apks as well
1852     if len(repodirs) > 1:
1853         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1854         if cc:
1855             cachechanged = True
1856     else:
1857         archapks = []
1858
1859     # Apply information from latest apks to the application and update dates
1860     apply_info_from_latest_apk(apps, apks + archapks)
1861
1862     # Sort the app list by name, then the web site doesn't have to by default.
1863     # (we had to wait until we'd scanned the apks to do this, because mostly the
1864     # name comes from there!)
1865     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1866
1867     # APKs are placed into multiple repos based on the app package, providing
1868     # per-app subscription feeds for nightly builds and things like it
1869     if config['per_app_repos']:
1870         add_apks_to_per_app_repos(repodirs[0], apks)
1871         for appid, app in apps.items():
1872             repodir = os.path.join(appid, 'fdroid', 'repo')
1873             appdict = dict()
1874             appdict[appid] = app
1875             if os.path.isdir(repodir):
1876                 index.make(appdict, [appid], apks, repodir, False)
1877             else:
1878                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1879         return
1880
1881     if len(repodirs) > 1:
1882         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1883
1884     # Make the index for the main repo...
1885     index.make(apps, sortedids, apks, repodirs[0], False)
1886     make_categories_txt(repodirs[0], categories)
1887
1888     # If there's an archive repo,  make the index for it. We already scanned it
1889     # earlier on.
1890     if len(repodirs) > 1:
1891         index.make(apps, sortedids, archapks, repodirs[1], True)
1892
1893     git_remote = config.get('binary_transparency_remote')
1894     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1895         from . import btlog
1896         btlog.make_binary_transparency_log(repodirs)
1897
1898     if config['update_stats']:
1899         # Update known apks info...
1900         knownapks.writeifchanged()
1901
1902         # Generate latest apps data for widget
1903         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1904             data = ''
1905             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1906                 for line in f:
1907                     appid = line.rstrip()
1908                     data += appid + "\t"
1909                     app = apps[appid]
1910                     data += app.Name + "\t"
1911                     if app.icon is not None:
1912                         data += app.icon + "\t"
1913                     data += app.License + "\n"
1914             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1915                 f.write(data)
1916
1917     if cachechanged:
1918         write_cache(apkcache)
1919
1920     # Update the wiki...
1921     if options.wiki:
1922         update_wiki(apps, sortedids, apks + archapks)
1923
1924     logging.info(_("Finished"))
1925
1926
1927 if __name__ == "__main__":
1928     main()