fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in sorted(app.AntiFeatures):
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.Bitcoin,
 153             app.Litecoin,
 154             app.License,
 155             requiresroot,
 156             app.AuthorName,
 157             app.AuthorEmail)
 158
 159         if app.Provides:
 160             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 161
 162         wikidata += app.Summary
 163         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 164
 165         wikidata += "=Description=\n"
 166         wikidata += metadata.description_wiki(app.Description) + "\n"
 167
 168         wikidata += "=Maintainer Notes=\n"
 169         if app.MaintainerNotes:
 170             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 171         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 172
 173         # Get a list of all packages for this application...
 174         apklist = []
 175         gotcurrentver = False
 176         cantupdate = False
 177         buildfails = False
 178         for apk in apks:
 179             if apk['packageName'] == appid:
 180                 if str(apk['versionCode']) == app.CurrentVersionCode:
 181                     gotcurrentver = True
 182                 apklist.append(apk)
 183         # Include ones we can't build, as a special case...
 184         for build in app.builds:
 185             if build.disable:
 186                 if build.versionCode == app.CurrentVersionCode:
 187                     cantupdate = True
 188                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 189                 apklist.append({'versionCode': int(build.versionCode),
 190                                 'versionName': build.versionName,
 191                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 192                                 })
 193             else:
 194                 builtit = False
 195                 for apk in apklist:
 196                     if apk['versionCode'] == int(build.versionCode):
 197                         builtit = True
 198                         break
 199                 if not builtit:
 200                     buildfails = True
 201                     apklist.append({'versionCode': int(build.versionCode),
 202                                     'versionName': build.versionName,
 203                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 204                                     })
 205         if app.CurrentVersionCode == '0':
 206             cantupdate = True
 207         # Sort with most recent first...
 208         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 209
 210         wikidata += "=Versions=\n"
 211         if len(apklist) == 0:
 212             wikidata += "We currently have no versions of this app available."
 213         elif not gotcurrentver:
 214             wikidata += "We don't have the current version of this app."
 215         else:
 216             wikidata += "We have the current version of this app."
 217         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 218         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 219         if len(app.NoSourceSince) > 0:
 220             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 221         if len(app.CurrentVersion) > 0:
 222             wikidata += "The current (recommended) version is " + app.CurrentVersion
 223             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 224         validapks = 0
 225         for apk in apklist:
 226             wikidata += "==" + apk['versionName'] + "==\n"
 227
 228             if 'buildproblem' in apk:
 229                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 230             else:
 231                 validapks += 1
 232                 wikidata += "This version is built and signed by "
 233                 if 'srcname' in apk:
 234                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 235                 else:
 236                     wikidata += "the original developer.\n\n"
 237             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 238
 239         wikidata += '\n[[Category:' + wikicat + ']]\n'
 240         if len(app.NoSourceSince) > 0:
 241             wikidata += '\n[[Category:Apps missing source code]]\n'
 242         if validapks == 0 and not app.Disabled:
 243             wikidata += '\n[[Category:Apps with no packages]]\n'
 244         if cantupdate and not app.Disabled:
 245             wikidata += "\n[[Category:Apps we cannot update]]\n"
 246         if buildfails and not app.Disabled:
 247             wikidata += "\n[[Category:Apps with failing builds]]\n"
 248         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 249             wikidata += '\n[[Category:Apps to Update]]\n'
 250         if app.Disabled:
 251             wikidata += '\n[[Category:Apps that are disabled]]\n'
 252         if app.UpdateCheckMode == 'None' and not app.Disabled:
 253             wikidata += '\n[[Category:Apps with no update check]]\n'
 254         for appcat in app.Categories:
 255             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 256
 257         # We can't have underscores in the page name, even if they're in
 258         # the package ID, because MediaWiki messes with them...
 259         pagename = appid.replace('_', ' ')
 260
 261         # Drop a trailing newline, because mediawiki is going to drop it anyway
 262         # and it we don't we'll think the page has changed when it hasn't...
 263         if wikidata.endswith('\n'):
 264             wikidata = wikidata[:-1]
 265
 266         generated_pages[pagename] = wikidata
 267
 268         # Make a redirect from the name to the ID too, unless there's
 269         # already an existing page with the name and it isn't a redirect.
 270         noclobber = False
 271         apppagename = app.Name.replace('_', ' ')
 272         apppagename = apppagename.replace('{', '')
 273         apppagename = apppagename.replace('}', ' ')
 274         apppagename = apppagename.replace(':', ' ')
 275         apppagename = apppagename.replace('[', ' ')
 276         apppagename = apppagename.replace(']', ' ')
 277         # Drop double spaces caused mostly by replacing ':' above
 278         apppagename = apppagename.replace('  ', ' ')
 279         for expagename in site.allpages(prefix=apppagename,
 280                                         filterredir='nonredirects',
 281                                         generator=False):
 282             if expagename == apppagename:
 283                 noclobber = True
 284         # Another reason not to make the redirect page is if the app name
 285         # is the same as it's ID, because that will overwrite the real page
 286         # with an redirect to itself! (Although it seems like an odd
 287         # scenario this happens a lot, e.g. where there is metadata but no
 288         # builds or binaries to extract a name from.
 289         if apppagename == pagename:
 290             noclobber = True
 291         if not noclobber:
 292             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 293
 294     for tcat, genp in [(wikicat, generated_pages),
 295                        (wikiredircat, generated_redirects)]:
 296         catpages = site.Pages['Category:' + tcat]
 297         existingpages = []
 298         for page in catpages:
 299             existingpages.append(page.name)
 300             if page.name in genp:
 301                 pagetxt = page.edit()
 302                 if pagetxt != genp[page.name]:
 303                     logging.debug("Updating modified page " + page.name)
 304                     page.save(genp[page.name], summary='Auto-updated')
 305                 else:
 306                     logging.debug("Page " + page.name + " is unchanged")
 307             else:
 308                 logging.warn("Deleting page " + page.name)
 309                 page.delete('No longer published')
 310         for pagename, text in genp.items():
 311             logging.debug("Checking " + pagename)
 312             if pagename not in existingpages:
 313                 logging.debug("Creating page " + pagename)
 314                 try:
 315                     newpage = site.Pages[pagename]
 316                     newpage.save(text, summary='Auto-created')
 317                 except Exception as e:
 318                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 319
 320     # Purge server cache to ensure counts are up to date
 321     site.pages['Repository Maintenance'].purge()
 322
 323
 324 def delete_disabled_builds(apps, apkcache, repodirs):
 325     """Delete disabled build outputs.
 326
 327     :param apps: list of all applications, as per metadata.read_metadata
 328     :param apkcache: current apk cache information
 329     :param repodirs: the repo directories to process
 330     """
 331     for appid, app in apps.items():
 332         for build in app['builds']:
 333             if not build.disable:
 334                 continue
 335             apkfilename = common.get_release_filename(app, build)
 336             iconfilename = "%s.%s.png" % (
 337                 appid,
 338                 build.versionCode)
 339             for repodir in repodirs:
 340                 files = [
 341                     os.path.join(repodir, apkfilename),
 342                     os.path.join(repodir, apkfilename + '.asc'),
 343                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 344                 ]
 345                 for density in all_screen_densities:
 346                     repo_dir = get_icon_dir(repodir, density)
 347                     files.append(os.path.join(repo_dir, iconfilename))
 348
 349                 for f in files:
 350                     if os.path.exists(f):
 351                         logging.info("Deleting disabled build output " + f)
 352                         os.remove(f)
 353             if apkfilename in apkcache:
 354                 del apkcache[apkfilename]
 355
 356
 357 def resize_icon(iconpath, density):
 358
 359     if not os.path.isfile(iconpath):
 360         return
 361
 362     fp = None
 363     try:
 364         fp = open(iconpath, 'rb')
 365         im = Image.open(fp)
 366         size = dpi_to_px(density)
 367
 368         if any(length > size for length in im.size):
 369             oldsize = im.size
 370             im.thumbnail((size, size), Image.ANTIALIAS)
 371             logging.debug("%s was too large at %s - new size is %s" % (
 372                 iconpath, oldsize, im.size))
 373             im.save(iconpath, "PNG")
 374
 375     except Exception as e:
 376         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 377
 378     finally:
 379         if fp:
 380             fp.close()
 381
 382
 383 def resize_all_icons(repodirs):
 384     """Resize all icons that exceed the max size
 385
 386     :param repodirs: the repo directories to process
 387     """
 388     for repodir in repodirs:
 389         for density in screen_densities:
 390             icon_dir = get_icon_dir(repodir, density)
 391             icon_glob = os.path.join(icon_dir, '*.png')
 392             for iconpath in glob.glob(icon_glob):
 393                 resize_icon(iconpath, density)
 394
 395
 396 def getsig(apkpath):
 397     """ Get the signing certificate of an apk. To get the same md5 has that
 398     Android gets, we encode the .RSA certificate in a specific format and pass
 399     it hex-encoded to the md5 digest algorithm.
 400
 401     :param apkpath: path to the apk
 402     :returns: A string containing the md5 of the signature of the apk or None
 403               if an error occurred.
 404     """
 405
 406     with zipfile.ZipFile(apkpath, 'r') as apk:
 407         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 408
 409         if len(certs) < 1:
 410             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 411             return None
 412         if len(certs) > 1:
 413             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 414             return None
 415
 416         cert = apk.read(certs[0])
 417
 418     cert_encoded = common.get_certificate(cert)
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_cache_file():
 424     return os.path.join('tmp', 'apkcache')
 425
 426
 427 def get_cache():
 428     """Get the cached dict of the APK index
 429
 430     Gather information about all the apk files in the repo directory,
 431     using cached data if possible. Some of the index operations take a
 432     long time, like calculating the SHA-256 and verifying the APK
 433     signature.
 434
 435     The cache is invalidated if the metadata version is different, or
 436     the 'allow_disabled_algorithms' config/option is different.  In
 437     those cases, there is no easy way to know what has changed from
 438     the cache, so just rerun the whole thing.
 439
 440     :return: apkcache
 441
 442     """
 443     apkcachefile = get_cache_file()
 444     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 445     if not options.clean and os.path.exists(apkcachefile):
 446         with open(apkcachefile, 'rb') as cf:
 447             apkcache = pickle.load(cf, encoding='utf-8')
 448         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 449            or apkcache.get('allow_disabled_algorithms') != ada:
 450             apkcache = {}
 451     else:
 452         apkcache = {}
 453
 454     apkcache["METADATA_VERSION"] = METADATA_VERSION
 455     apkcache['allow_disabled_algorithms'] = ada
 456
 457     return apkcache
 458
 459
 460 def write_cache(apkcache):
 461     apkcachefile = get_cache_file()
 462     cache_path = os.path.dirname(apkcachefile)
 463     if not os.path.exists(cache_path):
 464         os.makedirs(cache_path)
 465     with open(apkcachefile, 'wb') as cf:
 466         pickle.dump(apkcache, cf)
 467
 468
 469 def get_icon_bytes(apkzip, iconsrc):
 470     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 471     try:
 472         return apkzip.read(iconsrc)
 473     except KeyError:
 474         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 475
 476
 477 def sha256sum(filename):
 478     '''Calculate the sha256 of the given file'''
 479     sha = hashlib.sha256()
 480     with open(filename, 'rb') as f:
 481         while True:
 482             t = f.read(16384)
 483             if len(t) == 0:
 484                 break
 485             sha.update(t)
 486     return sha.hexdigest()
 487
 488
 489 def has_known_vulnerability(filename):
 490     """checks for known vulnerabilities in the APK
 491
 492     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 493     version.  Google also enforces this:
 494     https://support.google.com/faqs/answer/6376725?hl=en
 495
 496     Checks whether there are more than one classes.dex or AndroidManifest.xml
 497     files, which is invalid and an essential part of the "Master Key" attack.
 498
 499     http://www.saurik.com/id/17
 500     """
 501
 502     # statically load this pattern
 503     if not hasattr(has_known_vulnerability, "pattern"):
 504         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 505
 506     files_in_apk = set()
 507     with zipfile.ZipFile(filename) as zf:
 508         for name in zf.namelist():
 509             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 510                 lib = zf.open(name)
 511                 while True:
 512                     chunk = lib.read(4096)
 513                     if chunk == b'':
 514                         break
 515                     m = has_known_vulnerability.pattern.search(chunk)
 516                     if m:
 517                         version = m.group(1).decode('ascii')
 518                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 519                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 520                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 521                             logging.debug(_('"{path}" contains recent {name} ({version})')
 522                                           .format(path=filename, name=name, version=version))
 523                         else:
 524                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 525                                             .format(path=filename, name=name, version=version))
 526                             return True
 527                         break
 528             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 529                 if name in files_in_apk:
 530                     return True
 531                 files_in_apk.add(name)
 532
 533     return False
 534
 535
 536 def insert_obbs(repodir, apps, apks):
 537     """Scans the .obb files in a given repo directory and adds them to the
 538     relevant APK instances.  OBB files have versionCodes like APK
 539     files, and they are loosely associated.  If there is an OBB file
 540     present, then any APK with the same or higher versionCode will use
 541     that OBB file.  There are two OBB types: main and patch, each APK
 542     can only have only have one of each.
 543
 544     https://developer.android.com/google/play/expansion-files.html
 545
 546     :param repodir: repo directory to scan
 547     :param apps: list of current, valid apps
 548     :param apks: current information on all APKs
 549
 550     """
 551
 552     def obbWarnDelete(f, msg):
 553         logging.warning(msg + ' ' + f)
 554         if options.delete_unknown:
 555             logging.error(_("Deleting unknown file: {path}").format(path=f))
 556             os.remove(f)
 557
 558     obbs = []
 559     java_Integer_MIN_VALUE = -pow(2, 31)
 560     currentPackageNames = apps.keys()
 561     for f in glob.glob(os.path.join(repodir, '*.obb')):
 562         obbfile = os.path.basename(f)
 563         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 564         chunks = obbfile.split('.')
 565         if chunks[0] != 'main' and chunks[0] != 'patch':
 566             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 567             continue
 568         if not re.match(r'^-?[0-9]+$', chunks[1]):
 569             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 570                           .format(name=chunks[0]))
 571             continue
 572         versionCode = int(chunks[1])
 573         packagename = ".".join(chunks[2:-1])
 574
 575         highestVersionCode = java_Integer_MIN_VALUE
 576         if packagename not in currentPackageNames:
 577             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 578             continue
 579         for apk in apks:
 580             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 581                 highestVersionCode = apk['versionCode']
 582         if versionCode > highestVersionCode:
 583             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 584                           .format(integer=str(versionCode)))
 585             continue
 586         obbsha256 = sha256sum(f)
 587         obbs.append((packagename, versionCode, obbfile, obbsha256))
 588
 589     for apk in apks:
 590         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 591             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 592                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 593                     apk['obbMainFile'] = obbfile
 594                     apk['obbMainFileSha256'] = obbsha256
 595                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 596                     apk['obbPatchFile'] = obbfile
 597                     apk['obbPatchFileSha256'] = obbsha256
 598             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 599                 break
 600
 601
 602 def translate_per_build_anti_features(apps, apks):
 603     """Grab the anti-features list from the build metadata
 604
 605     For most Anti-Features, they are really most applicable per-APK,
 606     not for an app.  An app can fix a vulnerability, add/remove
 607     tracking, etc.  This reads the 'antifeatures' list from the Build
 608     entries in the fdroiddata metadata file, then transforms it into
 609     the 'antiFeatures' list of unique items for the index.
 610
 611     The field key is all lower case in the metadata file to match the
 612     rest of the Build fields.  It is 'antiFeatures' camel case in the
 613     implementation, index, and fdroidclient since it is translated
 614     from the build 'antifeatures' field, not directly included.
 615
 616     """
 617
 618     antiFeatures = dict()
 619     for packageName, app in apps.items():
 620         d = dict()
 621         for build in app['builds']:
 622             afl = build.get('antifeatures')
 623             if afl:
 624                 d[int(build.versionCode)] = afl
 625         if len(d) > 0:
 626             antiFeatures[packageName] = d
 627
 628     for apk in apks:
 629         d = antiFeatures.get(apk['packageName'])
 630         if d:
 631             afl = d.get(apk['versionCode'])
 632             if afl:
 633                 apk['antiFeatures'].update(afl)
 634
 635
 636 def _get_localized_dict(app, locale):
 637     '''get the dict to add localized store metadata to'''
 638     if 'localized' not in app:
 639         app['localized'] = collections.OrderedDict()
 640     if locale not in app['localized']:
 641         app['localized'][locale] = collections.OrderedDict()
 642     return app['localized'][locale]
 643
 644
 645 def _set_localized_text_entry(app, locale, key, f):
 646     limit = config['char_limits'][key]
 647     localized = _get_localized_dict(app, locale)
 648     with open(f) as fp:
 649         text = fp.read()[:limit]
 650         if len(text) > 0:
 651             localized[key] = text
 652
 653
 654 def _set_author_entry(app, key, f):
 655     limit = config['char_limits']['author']
 656     with open(f) as fp:
 657         text = fp.read()[:limit]
 658         if len(text) > 0:
 659             app[key] = text
 660
 661
 662 def copy_triple_t_store_metadata(apps):
 663     """Include store metadata from the app's source repo
 664
 665     The Triple-T Gradle Play Publisher is a plugin that has a standard
 666     file layout for all of the metadata and graphics that the Google
 667     Play Store accepts.  Since F-Droid has the git repo, it can just
 668     pluck those files directly.  This method reads any text files into
 669     the app dict, then copies any graphics into the fdroid repo
 670     directory structure.
 671
 672     This needs to be run before insert_localized_app_metadata() so that
 673     the graphics files that are copied into the fdroid repo get
 674     properly indexed.
 675
 676     https://github.com/Triple-T/gradle-play-publisher#upload-images
 677     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 678
 679     """
 680
 681     if not os.path.isdir('build'):
 682         return  # nothing to do
 683
 684     for packageName, app in apps.items():
 685         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 686             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 687             for root, dirs, files in os.walk(d):
 688                 segments = root.split('/')
 689                 locale = segments[-2]
 690                 for f in files:
 691                     if f == 'fulldescription':
 692                         _set_localized_text_entry(app, locale, 'description',
 693                                                   os.path.join(root, f))
 694                         continue
 695                     elif f == 'shortdescription':
 696                         _set_localized_text_entry(app, locale, 'summary',
 697                                                   os.path.join(root, f))
 698                         continue
 699                     elif f == 'title':
 700                         _set_localized_text_entry(app, locale, 'name',
 701                                                   os.path.join(root, f))
 702                         continue
 703                     elif f == 'video':
 704                         _set_localized_text_entry(app, locale, 'video',
 705                                                   os.path.join(root, f))
 706                         continue
 707                     elif f == 'whatsnew':
 708                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 709                                                   os.path.join(root, f))
 710                         continue
 711                     elif f == 'contactEmail':
 712                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 713                         continue
 714                     elif f == 'contactPhone':
 715                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 716                         continue
 717                     elif f == 'contactWebsite':
 718                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 719                         continue
 720
 721                     base, extension = common.get_extension(f)
 722                     dirname = os.path.basename(root)
 723                     if extension in ALLOWED_EXTENSIONS \
 724                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 725                         if segments[-2] == 'listing':
 726                             locale = segments[-3]
 727                         else:
 728                             locale = segments[-2]
 729                         destdir = os.path.join('repo', packageName, locale, dirname)
 730                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 731                         sourcefile = os.path.join(root, f)
 732                         destfile = os.path.join(destdir, os.path.basename(f))
 733                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 734                         shutil.copy(sourcefile, destfile)
 735
 736
 737 def insert_localized_app_metadata(apps):
 738     """scans standard locations for graphics and localized text
 739
 740     Scans for localized description files, store graphics, and
 741     screenshot PNG files in statically defined screenshots directory
 742     and adds them to the app metadata.  The screenshots and graphic
 743     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 744     and must be in the following layout:
 745     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 746
 747     repo/packageName/locale/featureGraphic.png
 748     repo/packageName/locale/phoneScreenshots/1.png
 749     repo/packageName/locale/phoneScreenshots/2.png
 750
 751     The changelog files must be text files named with the versionCode
 752     ending with ".txt" and must be in the following layout:
 753     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 754
 755     repo/packageName/locale/changelogs/12345.txt
 756
 757     This will scan the each app's source repo then the metadata/ dir
 758     for these standard locations of changelog files.  If it finds
 759     them, they will be added to the dict of all packages, with the
 760     versions in the metadata/ folder taking precendence over the what
 761     is in the app's source repo.
 762
 763     Where "packageName" is the app's packageName and "locale" is the locale
 764     of the graphics, e.g. what language they are in, using the IETF RFC5646
 765     format (en-US, fr-CA, es-MX, etc).
 766
 767     This will also scan the app's git for a fastlane folder, and the
 768     metadata/ folder and the apps' source repos for standard locations
 769     of graphic and screenshot files.  If it finds them, it will copy
 770     them into the repo.  The fastlane files follow this pattern:
 771     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 772
 773     """
 774
 775     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 776     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 777     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 778
 779     for srcd in sorted(sourcedirs):
 780         if not os.path.isdir(srcd):
 781             continue
 782         for root, dirs, files in os.walk(srcd):
 783             segments = root.split('/')
 784             packageName = segments[1]
 785             if packageName not in apps:
 786                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 787                 continue
 788             locale = segments[-1]
 789             destdir = os.path.join('repo', packageName, locale)
 790             for f in files:
 791                 if f in ('description.txt', 'full_description.txt'):
 792                     _set_localized_text_entry(apps[packageName], locale, 'description',
 793                                               os.path.join(root, f))
 794                     continue
 795                 elif f in ('summary.txt', 'short_description.txt'):
 796                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 797                                               os.path.join(root, f))
 798                     continue
 799                 elif f in ('name.txt', 'title.txt'):
 800                     _set_localized_text_entry(apps[packageName], locale, 'name',
 801                                               os.path.join(root, f))
 802                     continue
 803                 elif f == 'video.txt':
 804                     _set_localized_text_entry(apps[packageName], locale, 'video',
 805                                               os.path.join(root, f))
 806                     continue
 807                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 808                     locale = segments[-2]
 809                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 810                                               os.path.join(root, f))
 811                     continue
 812
 813                 base, extension = common.get_extension(f)
 814                 if locale == 'images':
 815                     locale = segments[-2]
 816                     destdir = os.path.join('repo', packageName, locale)
 817                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 818                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 819                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 820                     shutil.copy(os.path.join(root, f), destdir)
 821             for d in dirs:
 822                 if d in SCREENSHOT_DIRS:
 823                     if locale == 'images':
 824                         locale = segments[-2]
 825                         destdir = os.path.join('repo', packageName, locale)
 826                     for f in glob.glob(os.path.join(root, d, '*.*')):
 827                         _ignored, extension = common.get_extension(f)
 828                         if extension in ALLOWED_EXTENSIONS:
 829                             screenshotdestdir = os.path.join(destdir, d)
 830                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 831                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 832                             shutil.copy(f, screenshotdestdir)
 833
 834     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 835     for d in repofiles:
 836         if not os.path.isdir(d):
 837             continue
 838         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 839             if not os.path.isfile(f):
 840                 continue
 841             segments = f.split('/')
 842             packageName = segments[1]
 843             locale = segments[2]
 844             screenshotdir = segments[3]
 845             filename = os.path.basename(f)
 846             base, extension = common.get_extension(filename)
 847
 848             if packageName not in apps:
 849                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 850                                 .format(path=filename, name=packageName))
 851                 continue
 852             graphics = _get_localized_dict(apps[packageName], locale)
 853
 854             if extension not in ALLOWED_EXTENSIONS:
 855                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 856             elif base in GRAPHIC_NAMES:
 857                 # there can only be zero or one of these per locale
 858                 graphics[base] = filename
 859             elif screenshotdir in SCREENSHOT_DIRS:
 860                 # there can any number of these per locale
 861                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 862                 if screenshotdir not in graphics:
 863                     graphics[screenshotdir] = []
 864                 graphics[screenshotdir].append(filename)
 865             else:
 866                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 867
 868
 869 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 870     """Scan a repo for all files with an extension except APK/OBB
 871
 872     :param apkcache: current cached info about all repo files
 873     :param repodir: repo directory to scan
 874     :param knownapks: list of all known files, as per metadata.read_metadata
 875     :param use_date_from_file: use date from file (instead of current date)
 876                                for newly added files
 877     """
 878
 879     cachechanged = False
 880     repo_files = []
 881     repodir = repodir.encode('utf-8')
 882     for name in os.listdir(repodir):
 883         file_extension = common.get_file_extension(name)
 884         if file_extension == 'apk' or file_extension == 'obb':
 885             continue
 886         filename = os.path.join(repodir, name)
 887         name_utf8 = name.decode('utf-8')
 888         if filename.endswith(b'_src.tar.gz'):
 889             logging.debug(_('skipping source tarball: {path}')
 890                           .format(path=filename.decode('utf-8')))
 891             continue
 892         if not common.is_repo_file(filename):
 893             continue
 894         stat = os.stat(filename)
 895         if stat.st_size == 0:
 896             raise FDroidException(_('{path} is zero size!')
 897                                   .format(path=filename))
 898
 899         shasum = sha256sum(filename)
 900         usecache = False
 901         if name in apkcache:
 902             repo_file = apkcache[name]
 903             # added time is cached as tuple but used here as datetime instance
 904             if 'added' in repo_file:
 905                 a = repo_file['added']
 906                 if isinstance(a, datetime):
 907                     repo_file['added'] = a
 908                 else:
 909                     repo_file['added'] = datetime(*a[:6])
 910             if repo_file.get('hash') == shasum:
 911                 logging.debug(_("Reading {apkfilename} from cache")
 912                               .format(apkfilename=name_utf8))
 913                 usecache = True
 914             else:
 915                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 916                               .format(apkfilename=name_utf8))
 917
 918         if not usecache:
 919             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 920             repo_file = collections.OrderedDict()
 921             repo_file['name'] = os.path.splitext(name_utf8)[0]
 922             # TODO rename apkname globally to something more generic
 923             repo_file['apkName'] = name_utf8
 924             repo_file['hash'] = shasum
 925             repo_file['hashType'] = 'sha256'
 926             repo_file['versionCode'] = 0
 927             repo_file['versionName'] = shasum
 928             # the static ID is the SHA256 unless it is set in the metadata
 929             repo_file['packageName'] = shasum
 930
 931             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 932             if m:
 933                 repo_file['packageName'] = m.group(1)
 934                 repo_file['versionCode'] = int(m.group(2))
 935             srcfilename = name + b'_src.tar.gz'
 936             if os.path.exists(os.path.join(repodir, srcfilename)):
 937                 repo_file['srcname'] = srcfilename.decode('utf-8')
 938             repo_file['size'] = stat.st_size
 939
 940             apkcache[name] = repo_file
 941             cachechanged = True
 942
 943         if use_date_from_file:
 944             timestamp = stat.st_ctime
 945             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 946         else:
 947             default_date_param = None
 948
 949         # Record in knownapks, getting the added date at the same time..
 950         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 951                                     default_date=default_date_param)
 952         if added:
 953             repo_file['added'] = added
 954
 955         repo_files.append(repo_file)
 956
 957     return repo_files, cachechanged
 958
 959
 960 def scan_apk(apk_file):
 961     """
 962     Scans an APK file and returns dictionary with metadata of the APK.
 963
 964     Attention: This does *not* verify that the APK signature is correct.
 965
 966     :param apk_file: The (ideally absolute) path to the APK file
 967     :raises BuildException
 968     :return A dict containing APK metadata
 969     """
 970     apk = {
 971         'hash': sha256sum(apk_file),
 972         'hashType': 'sha256',
 973         'uses-permission': [],
 974         'uses-permission-sdk-23': [],
 975         'features': [],
 976         'icons_src': {},
 977         'icons': {},
 978         'antiFeatures': set(),
 979     }
 980
 981     if SdkToolsPopen(['aapt', 'version'], output=False):
 982         scan_apk_aapt(apk, apk_file)
 983     else:
 984         scan_apk_androguard(apk, apk_file)
 985
 986     # Get the signature, or rather the signing key fingerprints
 987     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 988     apk['sig'] = getsig(apk_file)
 989     if not apk['sig']:
 990         raise BuildException("Failed to get apk signature")
 991     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
 992                                                                apk_file))
 993     if not apk.get('signer'):
 994         raise BuildException("Failed to get apk signing key fingerprint")
 995
 996     # Get size of the APK
 997     apk['size'] = os.path.getsize(apk_file)
 998
 999     if 'minSdkVersion' not in apk:
1000         logging.warning("No SDK version information found in {0}".format(apk_file))
1001         apk['minSdkVersion'] = 1
1002
1003     # Check for known vulnerabilities
1004     if has_known_vulnerability(apk_file):
1005         apk['antiFeatures'].add('KnownVuln')
1006
1007     return apk
1008
1009
1010 def scan_apk_aapt(apk, apkfile):
1011     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1012     if p.returncode != 0:
1013         if options.delete_unknown:
1014             if os.path.exists(apkfile):
1015                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1016                 os.remove(apkfile)
1017             else:
1018                 logging.error("Could not find {0} to remove it".format(apkfile))
1019         else:
1020             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1021         raise BuildException(_("Invalid APK"))
1022     for line in p.output.splitlines():
1023         if line.startswith("package:"):
1024             try:
1025                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1026                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1027                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1028             except Exception as e:
1029                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1030         elif line.startswith("application:"):
1031             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1032             # Keep path to non-dpi icon in case we need it
1033             match = re.match(APK_ICON_PAT_NODPI, line)
1034             if match:
1035                 apk['icons_src']['-1'] = match.group(1)
1036         elif line.startswith("launchable-activity:"):
1037             # Only use launchable-activity as fallback to application
1038             if not apk['name']:
1039                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1040             if '-1' not in apk['icons_src']:
1041                 match = re.match(APK_ICON_PAT_NODPI, line)
1042                 if match:
1043                     apk['icons_src']['-1'] = match.group(1)
1044         elif line.startswith("application-icon-"):
1045             match = re.match(APK_ICON_PAT, line)
1046             if match:
1047                 density = match.group(1)
1048                 path = match.group(2)
1049                 apk['icons_src'][density] = path
1050         elif line.startswith("sdkVersion:"):
1051             m = re.match(APK_SDK_VERSION_PAT, line)
1052             if m is None:
1053                 logging.error(line.replace('sdkVersion:', '')
1054                               + ' is not a valid minSdkVersion!')
1055             else:
1056                 apk['minSdkVersion'] = m.group(1)
1057                 # if target not set, default to min
1058                 if 'targetSdkVersion' not in apk:
1059                     apk['targetSdkVersion'] = m.group(1)
1060         elif line.startswith("targetSdkVersion:"):
1061             m = re.match(APK_SDK_VERSION_PAT, line)
1062             if m is None:
1063                 logging.error(line.replace('targetSdkVersion:', '')
1064                               + ' is not a valid targetSdkVersion!')
1065             else:
1066                 apk['targetSdkVersion'] = m.group(1)
1067         elif line.startswith("maxSdkVersion:"):
1068             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1069         elif line.startswith("native-code:"):
1070             apk['nativecode'] = []
1071             for arch in line[13:].split(' '):
1072                 apk['nativecode'].append(arch[1:-1])
1073         elif line.startswith('uses-permission:'):
1074             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1075             if perm_match['maxSdkVersion']:
1076                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1077             permission = UsesPermission(
1078                 perm_match['name'],
1079                 perm_match['maxSdkVersion']
1080             )
1081
1082             apk['uses-permission'].append(permission)
1083         elif line.startswith('uses-permission-sdk-23:'):
1084             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1085             if perm_match['maxSdkVersion']:
1086                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1087             permission_sdk_23 = UsesPermissionSdk23(
1088                 perm_match['name'],
1089                 perm_match['maxSdkVersion']
1090             )
1091
1092             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1093
1094         elif line.startswith('uses-feature:'):
1095             feature = re.match(APK_FEATURE_PAT, line).group(1)
1096             # Filter out this, it's only added with the latest SDK tools and
1097             # causes problems for lots of apps.
1098             if feature != "android.hardware.screen.portrait" \
1099                     and feature != "android.hardware.screen.landscape":
1100                 if feature.startswith("android.feature."):
1101                     feature = feature[16:]
1102                 apk['features'].add(feature)
1103
1104
1105 def scan_apk_androguard(apk, apkfile):
1106     try:
1107         from androguard.core.bytecodes.apk import APK
1108         apkobject = APK(apkfile)
1109         if apkobject.is_valid_APK():
1110             arsc = apkobject.get_android_resources()
1111         else:
1112             if options.delete_unknown:
1113                 if os.path.exists(apkfile):
1114                     logging.error(_("Failed to get apk information, deleting {path}")
1115                                   .format(path=apkfile))
1116                     os.remove(apkfile)
1117                 else:
1118                     logging.error(_("Could not find {path} to remove it")
1119                                   .format(path=apkfile))
1120             else:
1121                 logging.error(_("Failed to get apk information, skipping {path}")
1122                               .format(path=apkfile))
1123             raise BuildException(_("Invalid APK"))
1124     except ImportError:
1125         raise FDroidException("androguard library is not installed and aapt not present")
1126     except FileNotFoundError:
1127         logging.error(_("Could not open apk file for analysis"))
1128         raise BuildException(_("Invalid APK"))
1129
1130     apk['packageName'] = apkobject.get_package()
1131     apk['versionCode'] = int(apkobject.get_androidversion_code())
1132     apk['versionName'] = apkobject.get_androidversion_name()
1133     if apk['versionName'][0] == "@":
1134         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1135         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1136         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1137     apk['name'] = apkobject.get_app_name()
1138
1139     if apkobject.get_max_sdk_version() is not None:
1140         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1141     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1142     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1143
1144     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1145     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1146
1147     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1148
1149     for file in apkobject.get_files():
1150         d_re = density_re.match(file)
1151         if d_re:
1152             folder = d_re.group(1).split('-')
1153             if len(folder) > 1:
1154                 resolution = folder[1]
1155             else:
1156                 resolution = 'mdpi'
1157             density = screen_resolutions[resolution]
1158             apk['icons_src'][density] = d_re.group(0)
1159
1160     if apk['icons_src'].get('-1') is None:
1161         apk['icons_src']['-1'] = apk['icons_src']['160']
1162
1163     arch_re = re.compile("^lib/(.*)/.*$")
1164     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1165     if len(arch) >= 1:
1166         apk['nativecode'] = []
1167         apk['nativecode'].extend(sorted(list(arch)))
1168
1169     xml = apkobject.get_android_manifest_xml()
1170
1171     for item in xml.getElementsByTagName('uses-permission'):
1172         name = str(item.getAttribute("android:name"))
1173         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1174         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1175         permission = UsesPermission(
1176             name,
1177             maxSdkVersion
1178         )
1179         apk['uses-permission'].append(permission)
1180
1181     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1182         name = str(item.getAttribute("android:name"))
1183         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1184         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1185         permission_sdk_23 = UsesPermissionSdk23(
1186             name,
1187             maxSdkVersion
1188         )
1189         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1190
1191     for item in xml.getElementsByTagName('uses-feature'):
1192         feature = str(item.getAttribute("android:name"))
1193         if feature != "android.hardware.screen.portrait" \
1194                 and feature != "android.hardware.screen.landscape":
1195             if feature.startswith("android.feature."):
1196                 feature = feature[16:]
1197         apk['features'].append(feature)
1198
1199
1200 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1201                 allow_disabled_algorithms=False, archive_bad_sig=False):
1202     """Processes the apk with the given filename in the given repo directory.
1203
1204     This also extracts the icons.
1205
1206     :param apkcache: current apk cache information
1207     :param apkfilename: the filename of the apk to scan
1208     :param repodir: repo directory to scan
1209     :param knownapks: known apks info
1210     :param use_date_from_apk: use date from APK (instead of current date)
1211                               for newly added APKs
1212     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1213                                       disabled algorithms in the signature (e.g. MD5)
1214     :param archive_bad_sig: move APKs with a bad signature to the archive
1215     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1216      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1217     """
1218
1219     apk = {}
1220     apkfile = os.path.join(repodir, apkfilename)
1221
1222     cachechanged = False
1223     usecache = False
1224     if apkfilename in apkcache:
1225         apk = apkcache[apkfilename]
1226         if apk.get('hash') == sha256sum(apkfile):
1227             logging.debug(_("Reading {apkfilename} from cache")
1228                           .format(apkfilename=apkfilename))
1229             usecache = True
1230         else:
1231             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1232                           .format(apkfilename=apkfilename))
1233
1234     if not usecache:
1235         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1236
1237         try:
1238             apk = scan_apk(apkfile)
1239         except BuildException:
1240             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1241                             .format(apkfilename=apkfilename))
1242             return True, None, False
1243
1244         # Check for debuggable apks...
1245         if common.isApkAndDebuggable(apkfile):
1246             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1247
1248         if options.rename_apks:
1249             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1250             std_short_name = os.path.join(repodir, n)
1251             if apkfile != std_short_name:
1252                 if os.path.exists(std_short_name):
1253                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1254                     if apkfile != std_long_name:
1255                         if os.path.exists(std_long_name):
1256                             dupdir = os.path.join('duplicates', repodir)
1257                             if not os.path.isdir(dupdir):
1258                                 os.makedirs(dupdir, exist_ok=True)
1259                             dupfile = os.path.join('duplicates', std_long_name)
1260                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1261                             os.rename(apkfile, dupfile)
1262                             return True, None, False
1263                         else:
1264                             os.rename(apkfile, std_long_name)
1265                     apkfile = std_long_name
1266                 else:
1267                     os.rename(apkfile, std_short_name)
1268                     apkfile = std_short_name
1269                 apkfilename = apkfile[len(repodir) + 1:]
1270
1271         apk['apkName'] = apkfilename
1272         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1273         if os.path.exists(os.path.join(repodir, srcfilename)):
1274             apk['srcname'] = srcfilename
1275
1276         # verify the jar signature is correct, allow deprecated
1277         # algorithms only if the APK is in the archive.
1278         skipapk = False
1279         if not common.verify_apk_signature(apkfile):
1280             if repodir == 'archive' or allow_disabled_algorithms:
1281                 if common.verify_old_apk_signature(apkfile):
1282                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1283                 else:
1284                     skipapk = True
1285             else:
1286                 skipapk = True
1287
1288         if skipapk:
1289             if archive_bad_sig:
1290                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1291                                 .format(apkfilename=apkfilename))
1292                 move_apk_between_sections(repodir, 'archive', apk)
1293             else:
1294                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1295                                 .format(apkfilename=apkfilename))
1296             return True, None, False
1297
1298         apkzip = zipfile.ZipFile(apkfile, 'r')
1299
1300         manifest = apkzip.getinfo('AndroidManifest.xml')
1301         if manifest.date_time[1] == 0:  # month can't be zero
1302             logging.debug(_('AndroidManifest.xml has no date'))
1303         else:
1304             common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1305
1306         # extract icons from APK zip file
1307         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1308         try:
1309             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1310         finally:
1311             apkzip.close()  # ensure that APK zip file gets closed
1312
1313         # resize existing icons for densities missing in the APK
1314         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1315
1316         if use_date_from_apk and manifest.date_time[1] != 0:
1317             default_date_param = datetime(*manifest.date_time)
1318         else:
1319             default_date_param = None
1320
1321         # Record in known apks, getting the added date at the same time..
1322         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1323                                     default_date=default_date_param)
1324         if added:
1325             apk['added'] = added
1326
1327         apkcache[apkfilename] = apk
1328         cachechanged = True
1329
1330     return False, apk, cachechanged
1331
1332
1333 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1334     """Processes the apks in the given repo directory.
1335
1336     This also extracts the icons.
1337
1338     :param apkcache: current apk cache information
1339     :param repodir: repo directory to scan
1340     :param knownapks: known apks info
1341     :param use_date_from_apk: use date from APK (instead of current date)
1342                               for newly added APKs
1343     :returns: (apks, cachechanged) where apks is a list of apk information,
1344               and cachechanged is True if the apkcache got changed.
1345     """
1346
1347     cachechanged = False
1348
1349     for icon_dir in get_all_icon_dirs(repodir):
1350         if os.path.exists(icon_dir):
1351             if options.clean:
1352                 shutil.rmtree(icon_dir)
1353                 os.makedirs(icon_dir)
1354         else:
1355             os.makedirs(icon_dir)
1356
1357     apks = []
1358     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1359         apkfilename = apkfile[len(repodir) + 1:]
1360         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1361         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1362                                              use_date_from_apk, ada, True)
1363         if skip:
1364             continue
1365         apks.append(apk)
1366         cachechanged = cachechanged or cachethis
1367
1368     return apks, cachechanged
1369
1370
1371 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1372     """
1373     Extracts icons from the given APK zip in various densities,
1374     saves them into given repo directory
1375     and stores their names in the APK metadata dictionary.
1376
1377     :param icon_filename: A string representing the icon's file name
1378     :param apk: A populated dictionary containing APK metadata.
1379                 Needs to have 'icons_src' key
1380     :param apkzip: An opened zipfile.ZipFile of the APK file
1381     :param repo_dir: The directory of the APK's repository
1382     :return: A list of icon densities that are missing
1383     """
1384     empty_densities = []
1385     for density in screen_densities:
1386         if density not in apk['icons_src']:
1387             empty_densities.append(density)
1388             continue
1389         icon_src = apk['icons_src'][density]
1390         icon_dir = get_icon_dir(repo_dir, density)
1391         icon_dest = os.path.join(icon_dir, icon_filename)
1392
1393         # Extract the icon files per density
1394         if icon_src.endswith('.xml'):
1395             png = os.path.basename(icon_src)[:-4] + '.png'
1396             for f in apkzip.namelist():
1397                 if f.endswith(png):
1398                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1399                     if m and screen_resolutions[m.group(2)] == density:
1400                         icon_src = f
1401             if icon_src.endswith('.xml'):
1402                 empty_densities.append(density)
1403                 continue
1404         try:
1405             with open(icon_dest, 'wb') as f:
1406                 f.write(get_icon_bytes(apkzip, icon_src))
1407             apk['icons'][density] = icon_filename
1408         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1409             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1410             del apk['icons_src'][density]
1411             empty_densities.append(density)
1412
1413     if '-1' in apk['icons_src']:
1414         icon_src = apk['icons_src']['-1']
1415         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1416         with open(icon_path, 'wb') as f:
1417             f.write(get_icon_bytes(apkzip, icon_src))
1418         try:
1419             im = Image.open(icon_path)
1420             dpi = px_to_dpi(im.size[0])
1421             for density in screen_densities:
1422                 if density in apk['icons']:
1423                     break
1424                 if density == screen_densities[-1] or dpi >= int(density):
1425                     apk['icons'][density] = icon_filename
1426                     shutil.move(icon_path,
1427                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1428                     empty_densities.remove(density)
1429                     break
1430         except Exception as e:
1431             logging.warning(_("Failed reading {path}: {error}")
1432                             .format(path=icon_path, error=e))
1433
1434     if apk['icons']:
1435         apk['icon'] = icon_filename
1436
1437     return empty_densities
1438
1439
1440 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1441     """
1442     Resize existing icons for densities missing in the APK to ensure all densities are available
1443
1444     :param empty_densities: A list of icon densities that are missing
1445     :param icon_filename: A string representing the icon's file name
1446     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1447     :param repo_dir: The directory of the APK's repository
1448     """
1449     # First try resizing down to not lose quality
1450     last_density = None
1451     for density in screen_densities:
1452         if density not in empty_densities:
1453             last_density = density
1454             continue
1455         if last_density is None:
1456             continue
1457         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1458
1459         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1460         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1461         fp = None
1462         try:
1463             fp = open(last_icon_path, 'rb')
1464             im = Image.open(fp)
1465
1466             size = dpi_to_px(density)
1467
1468             im.thumbnail((size, size), Image.ANTIALIAS)
1469             im.save(icon_path, "PNG")
1470             empty_densities.remove(density)
1471         except Exception as e:
1472             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1473         finally:
1474             if fp:
1475                 fp.close()
1476
1477     # Then just copy from the highest resolution available
1478     last_density = None
1479     for density in reversed(screen_densities):
1480         if density not in empty_densities:
1481             last_density = density
1482             continue
1483
1484         if last_density is None:
1485             continue
1486
1487         shutil.copyfile(
1488             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1489             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1490         )
1491         empty_densities.remove(density)
1492
1493     for density in screen_densities:
1494         icon_dir = get_icon_dir(repo_dir, density)
1495         icon_dest = os.path.join(icon_dir, icon_filename)
1496         resize_icon(icon_dest, density)
1497
1498     # Copy from icons-mdpi to icons since mdpi is the baseline density
1499     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1500     if os.path.isfile(baseline):
1501         apk['icons']['0'] = icon_filename
1502         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1503
1504
1505 def apply_info_from_latest_apk(apps, apks):
1506     """
1507     Some information from the apks needs to be applied up to the application level.
1508     When doing this, we use the info from the most recent version's apk.
1509     We deal with figuring out when the app was added and last updated at the same time.
1510     """
1511     for appid, app in apps.items():
1512         bestver = UNSET_VERSION_CODE
1513         for apk in apks:
1514             if apk['packageName'] == appid:
1515                 if apk['versionCode'] > bestver:
1516                     bestver = apk['versionCode']
1517                     bestapk = apk
1518
1519                 if 'added' in apk:
1520                     if not app.added or apk['added'] < app.added:
1521                         app.added = apk['added']
1522                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1523                         app.lastUpdated = apk['added']
1524
1525         if not app.added:
1526             logging.debug("Don't know when " + appid + " was added")
1527         if not app.lastUpdated:
1528             logging.debug("Don't know when " + appid + " was last updated")
1529
1530         if bestver == UNSET_VERSION_CODE:
1531
1532             if app.Name is None:
1533                 app.Name = app.AutoName or appid
1534             app.icon = None
1535             logging.debug("Application " + appid + " has no packages")
1536         else:
1537             if app.Name is None:
1538                 app.Name = bestapk['name']
1539             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1540             if app.CurrentVersionCode is None:
1541                 app.CurrentVersionCode = str(bestver)
1542
1543
1544 def make_categories_txt(repodir, categories):
1545     '''Write a category list in the repo to allow quick access'''
1546     catdata = ''
1547     for cat in sorted(categories):
1548         catdata += cat + '\n'
1549     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1550         f.write(catdata)
1551
1552
1553 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1554
1555     def filter_apk_list_sorted(apk_list):
1556         res = []
1557         for apk in apk_list:
1558             if apk['packageName'] == appid:
1559                 res.append(apk)
1560
1561         # Sort the apk list by version code. First is highest/newest.
1562         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1563
1564     for appid, app in apps.items():
1565
1566         if app.ArchivePolicy:
1567             keepversions = int(app.ArchivePolicy[:-9])
1568         else:
1569             keepversions = defaultkeepversions
1570
1571         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1572                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1573
1574         current_app_apks = filter_apk_list_sorted(apks)
1575         if len(current_app_apks) > keepversions:
1576             # Move back the ones we don't want.
1577             for apk in current_app_apks[keepversions:]:
1578                 move_apk_between_sections(repodir, archivedir, apk)
1579                 archapks.append(apk)
1580                 apks.remove(apk)
1581
1582         current_app_archapks = filter_apk_list_sorted(archapks)
1583         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1584             kept = 0
1585             # Move forward the ones we want again, except DisableAlgorithm
1586             for apk in current_app_archapks:
1587                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1588                     move_apk_between_sections(archivedir, repodir, apk)
1589                     archapks.remove(apk)
1590                     apks.append(apk)
1591                     kept += 1
1592                 if kept == keepversions:
1593                     break
1594
1595
1596 def move_apk_between_sections(from_dir, to_dir, apk):
1597     """move an APK from repo to archive or vice versa"""
1598
1599     def _move_file(from_dir, to_dir, filename, ignore_missing):
1600         from_path = os.path.join(from_dir, filename)
1601         if ignore_missing and not os.path.exists(from_path):
1602             return
1603         to_path = os.path.join(to_dir, filename)
1604         if not os.path.exists(to_dir):
1605             os.mkdir(to_dir)
1606         shutil.move(from_path, to_path)
1607
1608     if from_dir == to_dir:
1609         return
1610
1611     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1612     _move_file(from_dir, to_dir, apk['apkName'], False)
1613     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1614     for density in all_screen_densities:
1615         from_icon_dir = get_icon_dir(from_dir, density)
1616         to_icon_dir = get_icon_dir(to_dir, density)
1617         if density not in apk.get('icons', []):
1618             continue
1619         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1620     if 'srcname' in apk:
1621         _move_file(from_dir, to_dir, apk['srcname'], False)
1622
1623
1624 def add_apks_to_per_app_repos(repodir, apks):
1625     apks_per_app = dict()
1626     for apk in apks:
1627         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1628         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1629         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1630         apks_per_app[apk['packageName']] = apk
1631
1632         if not os.path.exists(apk['per_app_icons']):
1633             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1634             os.makedirs(apk['per_app_icons'])
1635
1636         apkpath = os.path.join(repodir, apk['apkName'])
1637         shutil.copy(apkpath, apk['per_app_repo'])
1638         apksigpath = apkpath + '.sig'
1639         if os.path.exists(apksigpath):
1640             shutil.copy(apksigpath, apk['per_app_repo'])
1641         apkascpath = apkpath + '.asc'
1642         if os.path.exists(apkascpath):
1643             shutil.copy(apkascpath, apk['per_app_repo'])
1644
1645
1646 def create_metadata_from_template(apk):
1647     '''create a new metadata file using internal or external template
1648
1649     Generate warnings for apk's with no metadata (or create skeleton
1650     metadata files, if requested on the command line).  Though the
1651     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1652     since those impose things on the metadata file made from the
1653     template: field sort order, empty field value, formatting, etc.
1654     '''
1655
1656     import yaml
1657     if os.path.exists('template.yml'):
1658         with open('template.yml') as f:
1659             metatxt = f.read()
1660         if 'name' in apk and apk['name'] != '':
1661             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1662                              r'\1 ' + apk['name'],
1663                              metatxt,
1664                              flags=re.IGNORECASE | re.MULTILINE)
1665         else:
1666             logging.warning(_('{appid} does not have a name! Using package name instead.')
1667                             .format(appid=apk['packageName']))
1668             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1669                              r'\1 ' + apk['packageName'],
1670                              metatxt,
1671                              flags=re.IGNORECASE | re.MULTILINE)
1672         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1673             f.write(metatxt)
1674     else:
1675         app = dict()
1676         app['Categories'] = [os.path.basename(os.getcwd())]
1677         # include some blanks as part of the template
1678         app['AuthorName'] = ''
1679         app['Summary'] = ''
1680         app['WebSite'] = ''
1681         app['IssueTracker'] = ''
1682         app['SourceCode'] = ''
1683         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1684         if 'name' in apk and apk['name'] != '':
1685             app['Name'] = apk['name']
1686         else:
1687             logging.warning(_('{appid} does not have a name! Using package name instead.')
1688                             .format(appid=apk['packageName']))
1689             app['Name'] = apk['packageName']
1690         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1691             yaml.dump(app, f, default_flow_style=False)
1692     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1693
1694
1695 config = None
1696 options = None
1697
1698
1699 def main():
1700
1701     global config, options
1702
1703     # Parse command line...
1704     parser = ArgumentParser()
1705     common.setup_global_opts(parser)
1706     parser.add_argument("--create-key", action="store_true", default=False,
1707                         help=_("Add a repo signing key to an unsigned repo"))
1708     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1709                         help=_("Add skeleton metadata files for APKs that are missing them"))
1710     parser.add_argument("--delete-unknown", action="store_true", default=False,
1711                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1712     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1713                         help=_("Report on build data status"))
1714     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1715                         help=_("Interactively ask about things that need updating."))
1716     parser.add_argument("-I", "--icons", action="store_true", default=False,
1717                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1718     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1719                         help=_("Specify editor to use in interactive mode. Default " +
1720                                "is {path}").format(path='/etc/alternatives/editor'))
1721     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1722                         help=_("Update the wiki"))
1723     parser.add_argument("--pretty", action="store_true", default=False,
1724                         help=_("Produce human-readable XML/JSON for index files"))
1725     parser.add_argument("--clean", action="store_true", default=False,
1726                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1727     parser.add_argument("--nosign", action="store_true", default=False,
1728                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1729     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1730                         help=_("Use date from APK instead of current time for newly added APKs"))
1731     parser.add_argument("--rename-apks", action="store_true", default=False,
1732                         help=_("Rename APK files that do not match package.name_123.apk"))
1733     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1734                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1735     metadata.add_metadata_arguments(parser)
1736     options = parser.parse_args()
1737     metadata.warnings_action = options.W
1738
1739     config = common.read_config(options)
1740
1741     if not ('jarsigner' in config and 'keytool' in config):
1742         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1743
1744     repodirs = ['repo']
1745     if config['archive_older'] != 0:
1746         repodirs.append('archive')
1747         if not os.path.exists('archive'):
1748             os.mkdir('archive')
1749
1750     if options.icons:
1751         resize_all_icons(repodirs)
1752         sys.exit(0)
1753
1754     if options.rename_apks:
1755         options.clean = True
1756
1757     # check that icons exist now, rather than fail at the end of `fdroid update`
1758     for k in ['repo_icon', 'archive_icon']:
1759         if k in config:
1760             if not os.path.exists(config[k]):
1761                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1762                                  .format(name=k, path=config[k]))
1763                 sys.exit(1)
1764
1765     # if the user asks to create a keystore, do it now, reusing whatever it can
1766     if options.create_key:
1767         if os.path.exists(config['keystore']):
1768             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1769             logging.critical("\t'" + config['keystore'] + "'")
1770             sys.exit(1)
1771
1772         if 'repo_keyalias' not in config:
1773             config['repo_keyalias'] = socket.getfqdn()
1774             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1775         if 'keydname' not in config:
1776             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1777             common.write_to_config(config, 'keydname', config['keydname'])
1778         if 'keystore' not in config:
1779             config['keystore'] = common.default_config['keystore']
1780             common.write_to_config(config, 'keystore', config['keystore'])
1781
1782         password = common.genpassword()
1783         if 'keystorepass' not in config:
1784             config['keystorepass'] = password
1785             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1786         if 'keypass' not in config:
1787             config['keypass'] = password
1788             common.write_to_config(config, 'keypass', config['keypass'])
1789         common.genkeystore(config)
1790
1791     # Get all apps...
1792     apps = metadata.read_metadata()
1793
1794     # Generate a list of categories...
1795     categories = set()
1796     for app in apps.values():
1797         categories.update(app.Categories)
1798
1799     # Read known apks data (will be updated and written back when we've finished)
1800     knownapks = common.KnownApks()
1801
1802     # Get APK cache
1803     apkcache = get_cache()
1804
1805     # Delete builds for disabled apps
1806     delete_disabled_builds(apps, apkcache, repodirs)
1807
1808     # Scan all apks in the main repo
1809     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1810
1811     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1812                                            options.use_date_from_apk)
1813     cachechanged = cachechanged or fcachechanged
1814     apks += files
1815     for apk in apks:
1816         if apk['packageName'] not in apps:
1817             if options.create_metadata:
1818                 create_metadata_from_template(apk)
1819                 apps = metadata.read_metadata()
1820             else:
1821                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1822                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1823                 if options.delete_unknown:
1824                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1825                                  .format(apkfilename=apk['apkName']))
1826                     rmf = os.path.join(repodirs[0], apk['apkName'])
1827                     if not os.path.exists(rmf):
1828                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1829                     else:
1830                         os.remove(rmf)
1831                 else:
1832                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1833
1834     copy_triple_t_store_metadata(apps)
1835     insert_obbs(repodirs[0], apps, apks)
1836     insert_localized_app_metadata(apps)
1837     translate_per_build_anti_features(apps, apks)
1838
1839     # Scan the archive repo for apks as well
1840     if len(repodirs) > 1:
1841         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1842         if cc:
1843             cachechanged = True
1844     else:
1845         archapks = []
1846
1847     # Apply information from latest apks to the application and update dates
1848     apply_info_from_latest_apk(apps, apks + archapks)
1849
1850     # Sort the app list by name, then the web site doesn't have to by default.
1851     # (we had to wait until we'd scanned the apks to do this, because mostly the
1852     # name comes from there!)
1853     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1854
1855     # APKs are placed into multiple repos based on the app package, providing
1856     # per-app subscription feeds for nightly builds and things like it
1857     if config['per_app_repos']:
1858         add_apks_to_per_app_repos(repodirs[0], apks)
1859         for appid, app in apps.items():
1860             repodir = os.path.join(appid, 'fdroid', 'repo')
1861             appdict = dict()
1862             appdict[appid] = app
1863             if os.path.isdir(repodir):
1864                 index.make(appdict, [appid], apks, repodir, False)
1865             else:
1866                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1867         return
1868
1869     if len(repodirs) > 1:
1870         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1871
1872     # Make the index for the main repo...
1873     index.make(apps, sortedids, apks, repodirs[0], False)
1874     make_categories_txt(repodirs[0], categories)
1875
1876     # If there's an archive repo,  make the index for it. We already scanned it
1877     # earlier on.
1878     if len(repodirs) > 1:
1879         index.make(apps, sortedids, archapks, repodirs[1], True)
1880
1881     git_remote = config.get('binary_transparency_remote')
1882     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1883         from . import btlog
1884         btlog.make_binary_transparency_log(repodirs)
1885
1886     if config['update_stats']:
1887         # Update known apks info...
1888         knownapks.writeifchanged()
1889
1890         # Generate latest apps data for widget
1891         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1892             data = ''
1893             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1894                 for line in f:
1895                     appid = line.rstrip()
1896                     data += appid + "\t"
1897                     app = apps[appid]
1898                     data += app.Name + "\t"
1899                     if app.icon is not None:
1900                         data += app.icon + "\t"
1901                     data += app.License + "\n"
1902             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1903                 f.write(data)
1904
1905     if cachechanged:
1906         write_cache(apkcache)
1907
1908     # Update the wiki...
1909     if options.wiki:
1910         update_wiki(apps, sortedids, apks + archapks)
1911
1912     logging.info(_("Finished"))
1913
1914
1915 if __name__ == "__main__":
1916     main()