fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime, timedelta
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in app.AntiFeatures:
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.Bitcoin,
 153             app.Litecoin,
 154             app.License,
 155             requiresroot,
 156             app.AuthorName,
 157             app.AuthorEmail)
 158
 159         if app.Provides:
 160             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 161
 162         wikidata += app.Summary
 163         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 164
 165         wikidata += "=Description=\n"
 166         wikidata += metadata.description_wiki(app.Description) + "\n"
 167
 168         wikidata += "=Maintainer Notes=\n"
 169         if app.MaintainerNotes:
 170             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 171         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 172
 173         # Get a list of all packages for this application...
 174         apklist = []
 175         gotcurrentver = False
 176         cantupdate = False
 177         buildfails = False
 178         for apk in apks:
 179             if apk['packageName'] == appid:
 180                 if str(apk['versionCode']) == app.CurrentVersionCode:
 181                     gotcurrentver = True
 182                 apklist.append(apk)
 183         # Include ones we can't build, as a special case...
 184         for build in app.builds:
 185             if build.disable:
 186                 if build.versionCode == app.CurrentVersionCode:
 187                     cantupdate = True
 188                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 189                 apklist.append({'versionCode': int(build.versionCode),
 190                                 'versionName': build.versionName,
 191                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 192                                 })
 193             else:
 194                 builtit = False
 195                 for apk in apklist:
 196                     if apk['versionCode'] == int(build.versionCode):
 197                         builtit = True
 198                         break
 199                 if not builtit:
 200                     buildfails = True
 201                     apklist.append({'versionCode': int(build.versionCode),
 202                                     'versionName': build.versionName,
 203                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 204                                     })
 205         if app.CurrentVersionCode == '0':
 206             cantupdate = True
 207         # Sort with most recent first...
 208         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 209
 210         wikidata += "=Versions=\n"
 211         if len(apklist) == 0:
 212             wikidata += "We currently have no versions of this app available."
 213         elif not gotcurrentver:
 214             wikidata += "We don't have the current version of this app."
 215         else:
 216             wikidata += "We have the current version of this app."
 217         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 218         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 219         if len(app.NoSourceSince) > 0:
 220             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 221         if len(app.CurrentVersion) > 0:
 222             wikidata += "The current (recommended) version is " + app.CurrentVersion
 223             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 224         validapks = 0
 225         for apk in apklist:
 226             wikidata += "==" + apk['versionName'] + "==\n"
 227
 228             if 'buildproblem' in apk:
 229                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 230             else:
 231                 validapks += 1
 232                 wikidata += "This version is built and signed by "
 233                 if 'srcname' in apk:
 234                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 235                 else:
 236                     wikidata += "the original developer.\n\n"
 237             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 238
 239         wikidata += '\n[[Category:' + wikicat + ']]\n'
 240         if len(app.NoSourceSince) > 0:
 241             wikidata += '\n[[Category:Apps missing source code]]\n'
 242         if validapks == 0 and not app.Disabled:
 243             wikidata += '\n[[Category:Apps with no packages]]\n'
 244         if cantupdate and not app.Disabled:
 245             wikidata += "\n[[Category:Apps we cannot update]]\n"
 246         if buildfails and not app.Disabled:
 247             wikidata += "\n[[Category:Apps with failing builds]]\n"
 248         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 249             wikidata += '\n[[Category:Apps to Update]]\n'
 250         if app.Disabled:
 251             wikidata += '\n[[Category:Apps that are disabled]]\n'
 252         if app.UpdateCheckMode == 'None' and not app.Disabled:
 253             wikidata += '\n[[Category:Apps with no update check]]\n'
 254         for appcat in app.Categories:
 255             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 256
 257         # We can't have underscores in the page name, even if they're in
 258         # the package ID, because MediaWiki messes with them...
 259         pagename = appid.replace('_', ' ')
 260
 261         # Drop a trailing newline, because mediawiki is going to drop it anyway
 262         # and it we don't we'll think the page has changed when it hasn't...
 263         if wikidata.endswith('\n'):
 264             wikidata = wikidata[:-1]
 265
 266         generated_pages[pagename] = wikidata
 267
 268         # Make a redirect from the name to the ID too, unless there's
 269         # already an existing page with the name and it isn't a redirect.
 270         noclobber = False
 271         apppagename = app.Name.replace('_', ' ')
 272         apppagename = apppagename.replace('{', '')
 273         apppagename = apppagename.replace('}', ' ')
 274         apppagename = apppagename.replace(':', ' ')
 275         apppagename = apppagename.replace('[', ' ')
 276         apppagename = apppagename.replace(']', ' ')
 277         # Drop double spaces caused mostly by replacing ':' above
 278         apppagename = apppagename.replace('  ', ' ')
 279         for expagename in site.allpages(prefix=apppagename,
 280                                         filterredir='nonredirects',
 281                                         generator=False):
 282             if expagename == apppagename:
 283                 noclobber = True
 284         # Another reason not to make the redirect page is if the app name
 285         # is the same as it's ID, because that will overwrite the real page
 286         # with an redirect to itself! (Although it seems like an odd
 287         # scenario this happens a lot, e.g. where there is metadata but no
 288         # builds or binaries to extract a name from.
 289         if apppagename == pagename:
 290             noclobber = True
 291         if not noclobber:
 292             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 293
 294     for tcat, genp in [(wikicat, generated_pages),
 295                        (wikiredircat, generated_redirects)]:
 296         catpages = site.Pages['Category:' + tcat]
 297         existingpages = []
 298         for page in catpages:
 299             existingpages.append(page.name)
 300             if page.name in genp:
 301                 pagetxt = page.edit()
 302                 if pagetxt != genp[page.name]:
 303                     logging.debug("Updating modified page " + page.name)
 304                     page.save(genp[page.name], summary='Auto-updated')
 305                 else:
 306                     logging.debug("Page " + page.name + " is unchanged")
 307             else:
 308                 logging.warn("Deleting page " + page.name)
 309                 page.delete('No longer published')
 310         for pagename, text in genp.items():
 311             logging.debug("Checking " + pagename)
 312             if pagename not in existingpages:
 313                 logging.debug("Creating page " + pagename)
 314                 try:
 315                     newpage = site.Pages[pagename]
 316                     newpage.save(text, summary='Auto-created')
 317                 except Exception as e:
 318                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 319
 320     # Purge server cache to ensure counts are up to date
 321     site.pages['Repository Maintenance'].purge()
 322
 323
 324 def delete_disabled_builds(apps, apkcache, repodirs):
 325     """Delete disabled build outputs.
 326
 327     :param apps: list of all applications, as per metadata.read_metadata
 328     :param apkcache: current apk cache information
 329     :param repodirs: the repo directories to process
 330     """
 331     for appid, app in apps.items():
 332         for build in app['builds']:
 333             if not build.disable:
 334                 continue
 335             apkfilename = common.get_release_filename(app, build)
 336             iconfilename = "%s.%s.png" % (
 337                 appid,
 338                 build.versionCode)
 339             for repodir in repodirs:
 340                 files = [
 341                     os.path.join(repodir, apkfilename),
 342                     os.path.join(repodir, apkfilename + '.asc'),
 343                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 344                 ]
 345                 for density in all_screen_densities:
 346                     repo_dir = get_icon_dir(repodir, density)
 347                     files.append(os.path.join(repo_dir, iconfilename))
 348
 349                 for f in files:
 350                     if os.path.exists(f):
 351                         logging.info("Deleting disabled build output " + f)
 352                         os.remove(f)
 353             if apkfilename in apkcache:
 354                 del apkcache[apkfilename]
 355
 356
 357 def resize_icon(iconpath, density):
 358
 359     if not os.path.isfile(iconpath):
 360         return
 361
 362     fp = None
 363     try:
 364         fp = open(iconpath, 'rb')
 365         im = Image.open(fp)
 366         size = dpi_to_px(density)
 367
 368         if any(length > size for length in im.size):
 369             oldsize = im.size
 370             im.thumbnail((size, size), Image.ANTIALIAS)
 371             logging.debug("%s was too large at %s - new size is %s" % (
 372                 iconpath, oldsize, im.size))
 373             im.save(iconpath, "PNG")
 374
 375     except Exception as e:
 376         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 377
 378     finally:
 379         if fp:
 380             fp.close()
 381
 382
 383 def resize_all_icons(repodirs):
 384     """Resize all icons that exceed the max size
 385
 386     :param repodirs: the repo directories to process
 387     """
 388     for repodir in repodirs:
 389         for density in screen_densities:
 390             icon_dir = get_icon_dir(repodir, density)
 391             icon_glob = os.path.join(icon_dir, '*.png')
 392             for iconpath in glob.glob(icon_glob):
 393                 resize_icon(iconpath, density)
 394
 395
 396 def getsig(apkpath):
 397     """ Get the signing certificate of an apk. To get the same md5 has that
 398     Android gets, we encode the .RSA certificate in a specific format and pass
 399     it hex-encoded to the md5 digest algorithm.
 400
 401     :param apkpath: path to the apk
 402     :returns: A string containing the md5 of the signature of the apk or None
 403               if an error occurred.
 404     """
 405
 406     with zipfile.ZipFile(apkpath, 'r') as apk:
 407         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 408
 409         if len(certs) < 1:
 410             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 411             return None
 412         if len(certs) > 1:
 413             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 414             return None
 415
 416         cert = apk.read(certs[0])
 417
 418     cert_encoded = common.get_certificate(cert)
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_cache_file():
 424     return os.path.join('tmp', 'apkcache')
 425
 426
 427 def get_cache():
 428     """Get the cached dict of the APK index
 429
 430     Gather information about all the apk files in the repo directory,
 431     using cached data if possible. Some of the index operations take a
 432     long time, like calculating the SHA-256 and verifying the APK
 433     signature.
 434
 435     The cache is invalidated if the metadata version is different, or
 436     the 'allow_disabled_algorithms' config/option is different.  In
 437     those cases, there is no easy way to know what has changed from
 438     the cache, so just rerun the whole thing.
 439
 440     :return: apkcache
 441
 442     """
 443     apkcachefile = get_cache_file()
 444     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 445     if not options.clean and os.path.exists(apkcachefile):
 446         with open(apkcachefile, 'rb') as cf:
 447             apkcache = pickle.load(cf, encoding='utf-8')
 448         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 449            or apkcache.get('allow_disabled_algorithms') != ada:
 450             apkcache = {}
 451     else:
 452         apkcache = {}
 453
 454     apkcache["METADATA_VERSION"] = METADATA_VERSION
 455     apkcache['allow_disabled_algorithms'] = ada
 456
 457     return apkcache
 458
 459
 460 def write_cache(apkcache):
 461     apkcachefile = get_cache_file()
 462     cache_path = os.path.dirname(apkcachefile)
 463     if not os.path.exists(cache_path):
 464         os.makedirs(cache_path)
 465     with open(apkcachefile, 'wb') as cf:
 466         pickle.dump(apkcache, cf)
 467
 468
 469 def get_icon_bytes(apkzip, iconsrc):
 470     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 471     try:
 472         return apkzip.read(iconsrc)
 473     except KeyError:
 474         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 475
 476
 477 def sha256sum(filename):
 478     '''Calculate the sha256 of the given file'''
 479     sha = hashlib.sha256()
 480     with open(filename, 'rb') as f:
 481         while True:
 482             t = f.read(16384)
 483             if len(t) == 0:
 484                 break
 485             sha.update(t)
 486     return sha.hexdigest()
 487
 488
 489 def has_known_vulnerability(filename):
 490     """checks for known vulnerabilities in the APK
 491
 492     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 493     version.  Google also enforces this:
 494     https://support.google.com/faqs/answer/6376725?hl=en
 495
 496     Checks whether there are more than one classes.dex or AndroidManifest.xml
 497     files, which is invalid and an essential part of the "Master Key" attack.
 498
 499     http://www.saurik.com/id/17
 500     """
 501
 502     # statically load this pattern
 503     if not hasattr(has_known_vulnerability, "pattern"):
 504         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 505
 506     files_in_apk = set()
 507     with zipfile.ZipFile(filename) as zf:
 508         for name in zf.namelist():
 509             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 510                 lib = zf.open(name)
 511                 while True:
 512                     chunk = lib.read(4096)
 513                     if chunk == b'':
 514                         break
 515                     m = has_known_vulnerability.pattern.search(chunk)
 516                     if m:
 517                         version = m.group(1).decode('ascii')
 518                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 519                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 520                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 521                             logging.debug(_('"{path}" contains recent {name} ({version})')
 522                                           .format(path=filename, name=name, version=version))
 523                         else:
 524                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 525                                             .format(path=filename, name=name, version=version))
 526                             return True
 527                         break
 528             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 529                 if name in files_in_apk:
 530                     return True
 531                 files_in_apk.add(name)
 532
 533     return False
 534
 535
 536 def insert_obbs(repodir, apps, apks):
 537     """Scans the .obb files in a given repo directory and adds them to the
 538     relevant APK instances.  OBB files have versionCodes like APK
 539     files, and they are loosely associated.  If there is an OBB file
 540     present, then any APK with the same or higher versionCode will use
 541     that OBB file.  There are two OBB types: main and patch, each APK
 542     can only have only have one of each.
 543
 544     https://developer.android.com/google/play/expansion-files.html
 545
 546     :param repodir: repo directory to scan
 547     :param apps: list of current, valid apps
 548     :param apks: current information on all APKs
 549
 550     """
 551
 552     def obbWarnDelete(f, msg):
 553         logging.warning(msg + ' ' + f)
 554         if options.delete_unknown:
 555             logging.error(_("Deleting unknown file: {path}").format(path=f))
 556             os.remove(f)
 557
 558     obbs = []
 559     java_Integer_MIN_VALUE = -pow(2, 31)
 560     currentPackageNames = apps.keys()
 561     for f in glob.glob(os.path.join(repodir, '*.obb')):
 562         obbfile = os.path.basename(f)
 563         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 564         chunks = obbfile.split('.')
 565         if chunks[0] != 'main' and chunks[0] != 'patch':
 566             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 567             continue
 568         if not re.match(r'^-?[0-9]+$', chunks[1]):
 569             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 570                           .format(name=chunks[0]))
 571             continue
 572         versionCode = int(chunks[1])
 573         packagename = ".".join(chunks[2:-1])
 574
 575         highestVersionCode = java_Integer_MIN_VALUE
 576         if packagename not in currentPackageNames:
 577             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 578             continue
 579         for apk in apks:
 580             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 581                 highestVersionCode = apk['versionCode']
 582         if versionCode > highestVersionCode:
 583             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 584                           .format(integer=str(versionCode)))
 585             continue
 586         obbsha256 = sha256sum(f)
 587         obbs.append((packagename, versionCode, obbfile, obbsha256))
 588
 589     for apk in apks:
 590         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 591             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 592                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 593                     apk['obbMainFile'] = obbfile
 594                     apk['obbMainFileSha256'] = obbsha256
 595                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 596                     apk['obbPatchFile'] = obbfile
 597                     apk['obbPatchFileSha256'] = obbsha256
 598             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 599                 break
 600
 601
 602 def translate_per_build_anti_features(apps, apks):
 603     """Grab the anti-features list from the build metadata
 604
 605     For most Anti-Features, they are really most applicable per-APK,
 606     not for an app.  An app can fix a vulnerability, add/remove
 607     tracking, etc.  This reads the 'antifeatures' list from the Build
 608     entries in the fdroiddata metadata file, then transforms it into
 609     the 'antiFeatures' list of unique items for the index.
 610
 611     The field key is all lower case in the metadata file to match the
 612     rest of the Build fields.  It is 'antiFeatures' camel case in the
 613     implementation, index, and fdroidclient since it is translated
 614     from the build 'antifeatures' field, not directly included.
 615
 616     """
 617
 618     antiFeatures = dict()
 619     for packageName, app in apps.items():
 620         d = dict()
 621         for build in app['builds']:
 622             afl = build.get('antifeatures')
 623             if afl:
 624                 d[int(build.versionCode)] = afl
 625         if len(d) > 0:
 626             antiFeatures[packageName] = d
 627
 628     for apk in apks:
 629         d = antiFeatures.get(apk['packageName'])
 630         if d:
 631             afl = d.get(apk['versionCode'])
 632             if afl:
 633                 apk['antiFeatures'].update(afl)
 634
 635
 636 def _get_localized_dict(app, locale):
 637     '''get the dict to add localized store metadata to'''
 638     if 'localized' not in app:
 639         app['localized'] = collections.OrderedDict()
 640     if locale not in app['localized']:
 641         app['localized'][locale] = collections.OrderedDict()
 642     return app['localized'][locale]
 643
 644
 645 def _set_localized_text_entry(app, locale, key, f):
 646     limit = config['char_limits'][key]
 647     localized = _get_localized_dict(app, locale)
 648     with open(f) as fp:
 649         text = fp.read()[:limit]
 650         if len(text) > 0:
 651             localized[key] = text
 652
 653
 654 def _set_author_entry(app, key, f):
 655     limit = config['char_limits']['author']
 656     with open(f) as fp:
 657         text = fp.read()[:limit]
 658         if len(text) > 0:
 659             app[key] = text
 660
 661
 662 def copy_triple_t_store_metadata(apps):
 663     """Include store metadata from the app's source repo
 664
 665     The Triple-T Gradle Play Publisher is a plugin that has a standard
 666     file layout for all of the metadata and graphics that the Google
 667     Play Store accepts.  Since F-Droid has the git repo, it can just
 668     pluck those files directly.  This method reads any text files into
 669     the app dict, then copies any graphics into the fdroid repo
 670     directory structure.
 671
 672     This needs to be run before insert_localized_app_metadata() so that
 673     the graphics files that are copied into the fdroid repo get
 674     properly indexed.
 675
 676     https://github.com/Triple-T/gradle-play-publisher#upload-images
 677     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 678
 679     """
 680
 681     if not os.path.isdir('build'):
 682         return  # nothing to do
 683
 684     for packageName, app in apps.items():
 685         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 686             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 687             for root, dirs, files in os.walk(d):
 688                 segments = root.split('/')
 689                 locale = segments[-2]
 690                 for f in files:
 691                     if f == 'fulldescription':
 692                         _set_localized_text_entry(app, locale, 'description',
 693                                                   os.path.join(root, f))
 694                         continue
 695                     elif f == 'shortdescription':
 696                         _set_localized_text_entry(app, locale, 'summary',
 697                                                   os.path.join(root, f))
 698                         continue
 699                     elif f == 'title':
 700                         _set_localized_text_entry(app, locale, 'name',
 701                                                   os.path.join(root, f))
 702                         continue
 703                     elif f == 'video':
 704                         _set_localized_text_entry(app, locale, 'video',
 705                                                   os.path.join(root, f))
 706                         continue
 707                     elif f == 'whatsnew':
 708                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 709                                                   os.path.join(root, f))
 710                         continue
 711                     elif f == 'contactEmail':
 712                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 713                         continue
 714                     elif f == 'contactPhone':
 715                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 716                         continue
 717                     elif f == 'contactWebsite':
 718                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 719                         continue
 720
 721                     base, extension = common.get_extension(f)
 722                     dirname = os.path.basename(root)
 723                     if extension in ALLOWED_EXTENSIONS \
 724                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 725                         if segments[-2] == 'listing':
 726                             locale = segments[-3]
 727                         else:
 728                             locale = segments[-2]
 729                         destdir = os.path.join('repo', packageName, locale, dirname)
 730                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 731                         sourcefile = os.path.join(root, f)
 732                         destfile = os.path.join(destdir, os.path.basename(f))
 733                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 734                         shutil.copy(sourcefile, destfile)
 735
 736
 737 def insert_localized_app_metadata(apps):
 738     """scans standard locations for graphics and localized text
 739
 740     Scans for localized description files, store graphics, and
 741     screenshot PNG files in statically defined screenshots directory
 742     and adds them to the app metadata.  The screenshots and graphic
 743     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 744     and must be in the following layout:
 745     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 746
 747     repo/packageName/locale/featureGraphic.png
 748     repo/packageName/locale/phoneScreenshots/1.png
 749     repo/packageName/locale/phoneScreenshots/2.png
 750
 751     The changelog files must be text files named with the versionCode
 752     ending with ".txt" and must be in the following layout:
 753     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 754
 755     repo/packageName/locale/changelogs/12345.txt
 756
 757     This will scan the each app's source repo then the metadata/ dir
 758     for these standard locations of changelog files.  If it finds
 759     them, they will be added to the dict of all packages, with the
 760     versions in the metadata/ folder taking precendence over the what
 761     is in the app's source repo.
 762
 763     Where "packageName" is the app's packageName and "locale" is the locale
 764     of the graphics, e.g. what language they are in, using the IETF RFC5646
 765     format (en-US, fr-CA, es-MX, etc).
 766
 767     This will also scan the app's git for a fastlane folder, and the
 768     metadata/ folder and the apps' source repos for standard locations
 769     of graphic and screenshot files.  If it finds them, it will copy
 770     them into the repo.  The fastlane files follow this pattern:
 771     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 772
 773     """
 774
 775     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 776     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 777     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 778
 779     for srcd in sorted(sourcedirs):
 780         if not os.path.isdir(srcd):
 781             continue
 782         for root, dirs, files in os.walk(srcd):
 783             segments = root.split('/')
 784             packageName = segments[1]
 785             if packageName not in apps:
 786                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 787                 continue
 788             locale = segments[-1]
 789             destdir = os.path.join('repo', packageName, locale)
 790             for f in files:
 791                 if f in ('description.txt', 'full_description.txt'):
 792                     _set_localized_text_entry(apps[packageName], locale, 'description',
 793                                               os.path.join(root, f))
 794                     continue
 795                 elif f in ('summary.txt', 'short_description.txt'):
 796                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 797                                               os.path.join(root, f))
 798                     continue
 799                 elif f in ('name.txt', 'title.txt'):
 800                     _set_localized_text_entry(apps[packageName], locale, 'name',
 801                                               os.path.join(root, f))
 802                     continue
 803                 elif f == 'video.txt':
 804                     _set_localized_text_entry(apps[packageName], locale, 'video',
 805                                               os.path.join(root, f))
 806                     continue
 807                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 808                     locale = segments[-2]
 809                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 810                                               os.path.join(root, f))
 811                     continue
 812
 813                 base, extension = common.get_extension(f)
 814                 if locale == 'images':
 815                     locale = segments[-2]
 816                     destdir = os.path.join('repo', packageName, locale)
 817                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 818                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 819                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 820                     shutil.copy(os.path.join(root, f), destdir)
 821             for d in dirs:
 822                 if d in SCREENSHOT_DIRS:
 823                     if locale == 'images':
 824                         locale = segments[-2]
 825                         destdir = os.path.join('repo', packageName, locale)
 826                     for f in glob.glob(os.path.join(root, d, '*.*')):
 827                         _, extension = common.get_extension(f)
 828                         if extension in ALLOWED_EXTENSIONS:
 829                             screenshotdestdir = os.path.join(destdir, d)
 830                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 831                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 832                             shutil.copy(f, screenshotdestdir)
 833
 834     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 835     for d in repofiles:
 836         if not os.path.isdir(d):
 837             continue
 838         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 839             if not os.path.isfile(f):
 840                 continue
 841             segments = f.split('/')
 842             packageName = segments[1]
 843             locale = segments[2]
 844             screenshotdir = segments[3]
 845             filename = os.path.basename(f)
 846             base, extension = common.get_extension(filename)
 847
 848             if packageName not in apps:
 849                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 850                                 % (filename, packageName))
 851                 continue
 852             graphics = _get_localized_dict(apps[packageName], locale)
 853
 854             if extension not in ALLOWED_EXTENSIONS:
 855                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 856             elif base in GRAPHIC_NAMES:
 857                 # there can only be zero or one of these per locale
 858                 graphics[base] = filename
 859             elif screenshotdir in SCREENSHOT_DIRS:
 860                 # there can any number of these per locale
 861                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 862                 if screenshotdir not in graphics:
 863                     graphics[screenshotdir] = []
 864                 graphics[screenshotdir].append(filename)
 865             else:
 866                 logging.warning('Unsupported graphics file found: ' + f)
 867
 868
 869 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 870     """Scan a repo for all files with an extension except APK/OBB
 871
 872     :param apkcache: current cached info about all repo files
 873     :param repodir: repo directory to scan
 874     :param knownapks: list of all known files, as per metadata.read_metadata
 875     :param use_date_from_file: use date from file (instead of current date)
 876                                for newly added files
 877     """
 878
 879     cachechanged = False
 880     repo_files = []
 881     repodir = repodir.encode('utf-8')
 882     for name in os.listdir(repodir):
 883         file_extension = common.get_file_extension(name)
 884         if file_extension == 'apk' or file_extension == 'obb':
 885             continue
 886         filename = os.path.join(repodir, name)
 887         name_utf8 = name.decode('utf-8')
 888         if filename.endswith(b'_src.tar.gz'):
 889             logging.debug(_('skipping source tarball: {path}')
 890                           .format(path=filename.decode('utf-8')))
 891             continue
 892         if not common.is_repo_file(filename):
 893             continue
 894         stat = os.stat(filename)
 895         if stat.st_size == 0:
 896             raise FDroidException(_('{path} is zero size!')
 897                                   .format(path=filename))
 898
 899         shasum = sha256sum(filename)
 900         usecache = False
 901         if name in apkcache:
 902             repo_file = apkcache[name]
 903             # added time is cached as tuple but used here as datetime instance
 904             if 'added' in repo_file:
 905                 a = repo_file['added']
 906                 if isinstance(a, datetime):
 907                     repo_file['added'] = a
 908                 else:
 909                     repo_file['added'] = datetime(*a[:6])
 910             if repo_file.get('hash') == shasum:
 911                 logging.debug(_("Reading {apkfilename} from cache")
 912                               .format(apkfilename=name_utf8))
 913                 usecache = True
 914             else:
 915                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 916                               .format(apkfilename=name_utf8))
 917
 918         if not usecache:
 919             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 920             repo_file = collections.OrderedDict()
 921             repo_file['name'] = os.path.splitext(name_utf8)[0]
 922             # TODO rename apkname globally to something more generic
 923             repo_file['apkName'] = name_utf8
 924             repo_file['hash'] = shasum
 925             repo_file['hashType'] = 'sha256'
 926             repo_file['versionCode'] = 0
 927             repo_file['versionName'] = shasum
 928             # the static ID is the SHA256 unless it is set in the metadata
 929             repo_file['packageName'] = shasum
 930
 931             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 932             if m:
 933                 repo_file['packageName'] = m.group(1)
 934                 repo_file['versionCode'] = int(m.group(2))
 935             srcfilename = name + b'_src.tar.gz'
 936             if os.path.exists(os.path.join(repodir, srcfilename)):
 937                 repo_file['srcname'] = srcfilename.decode('utf-8')
 938             repo_file['size'] = stat.st_size
 939
 940             apkcache[name] = repo_file
 941             cachechanged = True
 942
 943         if use_date_from_file:
 944             timestamp = stat.st_ctime
 945             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 946         else:
 947             default_date_param = None
 948
 949         # Record in knownapks, getting the added date at the same time..
 950         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 951                                     default_date=default_date_param)
 952         if added:
 953             repo_file['added'] = added
 954
 955         repo_files.append(repo_file)
 956
 957     return repo_files, cachechanged
 958
 959
 960 def scan_apk(apk_file):
 961     """
 962     Scans an APK file and returns dictionary with metadata of the APK.
 963
 964     Attention: This does *not* verify that the APK signature is correct.
 965
 966     :param apk_file: The (ideally absolute) path to the APK file
 967     :raises BuildException
 968     :return A dict containing APK metadata
 969     """
 970     apk = {
 971         'hash': sha256sum(apk_file),
 972         'hashType': 'sha256',
 973         'uses-permission': [],
 974         'uses-permission-sdk-23': [],
 975         'features': [],
 976         'icons_src': {},
 977         'icons': {},
 978         'antiFeatures': set(),
 979     }
 980
 981     if SdkToolsPopen(['aapt', 'version'], output=False):
 982         scan_apk_aapt(apk, apk_file)
 983     else:
 984         scan_apk_androguard(apk, apk_file)
 985
 986     # Get the signature, or rather the signing key fingerprints
 987     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 988     apk['sig'] = getsig(apk_file)
 989     if not apk['sig']:
 990         raise BuildException("Failed to get apk signature")
 991     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
 992                                                                apk_file))
 993     if not apk.get('signer'):
 994         raise BuildException("Failed to get apk signing key fingerprint")
 995
 996     # Get size of the APK
 997     apk['size'] = os.path.getsize(apk_file)
 998
 999     if 'minSdkVersion' not in apk:
1000         logging.warning("No SDK version information found in {0}".format(apk_file))
1001         apk['minSdkVersion'] = 1
1002
1003     # Check for known vulnerabilities
1004     if has_known_vulnerability(apk_file):
1005         apk['antiFeatures'].add('KnownVuln')
1006
1007     return apk
1008
1009
1010 def scan_apk_aapt(apk, apkfile):
1011     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1012     if p.returncode != 0:
1013         if options.delete_unknown:
1014             if os.path.exists(apkfile):
1015                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1016                 os.remove(apkfile)
1017             else:
1018                 logging.error("Could not find {0} to remove it".format(apkfile))
1019         else:
1020             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1021         raise BuildException(_("Invalid APK"))
1022     for line in p.output.splitlines():
1023         if line.startswith("package:"):
1024             try:
1025                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1026                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1027                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1028             except Exception as e:
1029                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1030         elif line.startswith("application:"):
1031             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1032             # Keep path to non-dpi icon in case we need it
1033             match = re.match(APK_ICON_PAT_NODPI, line)
1034             if match:
1035                 apk['icons_src']['-1'] = match.group(1)
1036         elif line.startswith("launchable-activity:"):
1037             # Only use launchable-activity as fallback to application
1038             if not apk['name']:
1039                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1040             if '-1' not in apk['icons_src']:
1041                 match = re.match(APK_ICON_PAT_NODPI, line)
1042                 if match:
1043                     apk['icons_src']['-1'] = match.group(1)
1044         elif line.startswith("application-icon-"):
1045             match = re.match(APK_ICON_PAT, line)
1046             if match:
1047                 density = match.group(1)
1048                 path = match.group(2)
1049                 apk['icons_src'][density] = path
1050         elif line.startswith("sdkVersion:"):
1051             m = re.match(APK_SDK_VERSION_PAT, line)
1052             if m is None:
1053                 logging.error(line.replace('sdkVersion:', '')
1054                               + ' is not a valid minSdkVersion!')
1055             else:
1056                 apk['minSdkVersion'] = m.group(1)
1057                 # if target not set, default to min
1058                 if 'targetSdkVersion' not in apk:
1059                     apk['targetSdkVersion'] = m.group(1)
1060         elif line.startswith("targetSdkVersion:"):
1061             m = re.match(APK_SDK_VERSION_PAT, line)
1062             if m is None:
1063                 logging.error(line.replace('targetSdkVersion:', '')
1064                               + ' is not a valid targetSdkVersion!')
1065             else:
1066                 apk['targetSdkVersion'] = m.group(1)
1067         elif line.startswith("maxSdkVersion:"):
1068             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1069         elif line.startswith("native-code:"):
1070             apk['nativecode'] = []
1071             for arch in line[13:].split(' '):
1072                 apk['nativecode'].append(arch[1:-1])
1073         elif line.startswith('uses-permission:'):
1074             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1075             if perm_match['maxSdkVersion']:
1076                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1077             permission = UsesPermission(
1078                 perm_match['name'],
1079                 perm_match['maxSdkVersion']
1080             )
1081
1082             apk['uses-permission'].append(permission)
1083         elif line.startswith('uses-permission-sdk-23:'):
1084             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1085             if perm_match['maxSdkVersion']:
1086                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1087             permission_sdk_23 = UsesPermissionSdk23(
1088                 perm_match['name'],
1089                 perm_match['maxSdkVersion']
1090             )
1091
1092             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1093
1094         elif line.startswith('uses-feature:'):
1095             feature = re.match(APK_FEATURE_PAT, line).group(1)
1096             # Filter out this, it's only added with the latest SDK tools and
1097             # causes problems for lots of apps.
1098             if feature != "android.hardware.screen.portrait" \
1099                     and feature != "android.hardware.screen.landscape":
1100                 if feature.startswith("android.feature."):
1101                     feature = feature[16:]
1102                 apk['features'].add(feature)
1103
1104
1105 def scan_apk_androguard(apk, apkfile):
1106     try:
1107         from androguard.core.bytecodes.apk import APK
1108         apkobject = APK(apkfile)
1109         if apkobject.is_valid_APK():
1110             arsc = apkobject.get_android_resources()
1111         else:
1112             if options.delete_unknown:
1113                 if os.path.exists(apkfile):
1114                     logging.error(_("Failed to get apk information, deleting {path}")
1115                                   .format(path=apkfile))
1116                     os.remove(apkfile)
1117                 else:
1118                     logging.error(_("Could not find {path} to remove it")
1119                                   .format(path=apkfile))
1120             else:
1121                 logging.error(_("Failed to get apk information, skipping {path}")
1122                               .format(path=apkfile))
1123             raise BuildException(_("Invalid APK"))
1124     except ImportError:
1125         raise FDroidException("androguard library is not installed and aapt not present")
1126     except FileNotFoundError:
1127         logging.error(_("Could not open apk file for analysis"))
1128         raise BuildException(_("Invalid APK"))
1129
1130     apk['packageName'] = apkobject.get_package()
1131     apk['versionCode'] = int(apkobject.get_androidversion_code())
1132     apk['versionName'] = apkobject.get_androidversion_name()
1133     if apk['versionName'][0] == "@":
1134         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1135         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1136         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1137     apk['name'] = apkobject.get_app_name()
1138
1139     if apkobject.get_max_sdk_version() is not None:
1140         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1141     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1142     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1143
1144     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1145     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1146
1147     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1148
1149     for file in apkobject.get_files():
1150         d_re = density_re.match(file)
1151         if d_re:
1152             folder = d_re.group(1).split('-')
1153             if len(folder) > 1:
1154                 resolution = folder[1]
1155             else:
1156                 resolution = 'mdpi'
1157             density = screen_resolutions[resolution]
1158             apk['icons_src'][density] = d_re.group(0)
1159
1160     if apk['icons_src'].get('-1') is None:
1161         apk['icons_src']['-1'] = apk['icons_src']['160']
1162
1163     arch_re = re.compile("^lib/(.*)/.*$")
1164     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1165     if len(arch) >= 1:
1166         apk['nativecode'] = []
1167         apk['nativecode'].extend(sorted(list(arch)))
1168
1169     xml = apkobject.get_android_manifest_xml()
1170
1171     for item in xml.getElementsByTagName('uses-permission'):
1172         name = str(item.getAttribute("android:name"))
1173         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1174         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1175         permission = UsesPermission(
1176             name,
1177             maxSdkVersion
1178         )
1179         apk['uses-permission'].append(permission)
1180
1181     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1182         name = str(item.getAttribute("android:name"))
1183         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1184         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1185         permission_sdk_23 = UsesPermissionSdk23(
1186             name,
1187             maxSdkVersion
1188         )
1189         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1190
1191     for item in xml.getElementsByTagName('uses-feature'):
1192         feature = str(item.getAttribute("android:name"))
1193         if feature != "android.hardware.screen.portrait" \
1194                 and feature != "android.hardware.screen.landscape":
1195             if feature.startswith("android.feature."):
1196                 feature = feature[16:]
1197         apk['features'].append(feature)
1198
1199
1200 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1201                 allow_disabled_algorithms=False, archive_bad_sig=False):
1202     """Processes the apk with the given filename in the given repo directory.
1203
1204     This also extracts the icons.
1205
1206     :param apkcache: current apk cache information
1207     :param apkfilename: the filename of the apk to scan
1208     :param repodir: repo directory to scan
1209     :param knownapks: known apks info
1210     :param use_date_from_apk: use date from APK (instead of current date)
1211                               for newly added APKs
1212     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1213                                       disabled algorithms in the signature (e.g. MD5)
1214     :param archive_bad_sig: move APKs with a bad signature to the archive
1215     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1216      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1217     """
1218
1219     apk = {}
1220     apkfile = os.path.join(repodir, apkfilename)
1221
1222     cachechanged = False
1223     usecache = False
1224     if apkfilename in apkcache:
1225         apk = apkcache[apkfilename]
1226         if apk.get('hash') == sha256sum(apkfile):
1227             logging.debug(_("Reading {apkfilename} from cache")
1228                           .format(apkfilename=apkfilename))
1229             usecache = True
1230         else:
1231             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1232                           .format(apkfilename=apkfilename))
1233
1234     if not usecache:
1235         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1236
1237         try:
1238             apk = scan_apk(apkfile)
1239         except BuildException:
1240             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1241                             .format(apkfilename=apkfilename))
1242             return True, None, False
1243
1244         # Check for debuggable apks...
1245         if common.isApkAndDebuggable(apkfile):
1246             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1247
1248         if options.rename_apks:
1249             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1250             std_short_name = os.path.join(repodir, n)
1251             if apkfile != std_short_name:
1252                 if os.path.exists(std_short_name):
1253                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1254                     if apkfile != std_long_name:
1255                         if os.path.exists(std_long_name):
1256                             dupdir = os.path.join('duplicates', repodir)
1257                             if not os.path.isdir(dupdir):
1258                                 os.makedirs(dupdir, exist_ok=True)
1259                             dupfile = os.path.join('duplicates', std_long_name)
1260                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1261                             os.rename(apkfile, dupfile)
1262                             return True, None, False
1263                         else:
1264                             os.rename(apkfile, std_long_name)
1265                     apkfile = std_long_name
1266                 else:
1267                     os.rename(apkfile, std_short_name)
1268                     apkfile = std_short_name
1269                 apkfilename = apkfile[len(repodir) + 1:]
1270
1271         apk['apkName'] = apkfilename
1272         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1273         if os.path.exists(os.path.join(repodir, srcfilename)):
1274             apk['srcname'] = srcfilename
1275
1276         # verify the jar signature is correct, allow deprecated
1277         # algorithms only if the APK is in the archive.
1278         skipapk = False
1279         if not common.verify_apk_signature(apkfile):
1280             if repodir == 'archive' or allow_disabled_algorithms:
1281                 if common.verify_old_apk_signature(apkfile):
1282                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1283                 else:
1284                     skipapk = True
1285             else:
1286                 skipapk = True
1287
1288         if skipapk:
1289             if archive_bad_sig:
1290                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1291                                 .format(apkfilename=apkfilename))
1292                 move_apk_between_sections(repodir, 'archive', apk)
1293             else:
1294                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1295                                 .format(apkfilename=apkfilename))
1296             return True, None, False
1297
1298         apkzip = zipfile.ZipFile(apkfile, 'r')
1299
1300         # if an APK has files newer than the system time, suggest updating
1301         # the system clock.  This is useful for offline systems, used for
1302         # signing, which do not have another source of clock sync info. It
1303         # has to be more than 24 hours newer because ZIP/APK files do not
1304         # store timezone info
1305         manifest = apkzip.getinfo('AndroidManifest.xml')
1306         if manifest.date_time[1] == 0:  # month can't be zero
1307             logging.debug(_('AndroidManifest.xml has no date'))
1308         else:
1309             dt_obj = datetime(*manifest.date_time)
1310             checkdt = dt_obj - timedelta(1)
1311             if datetime.today() < checkdt:
1312                 logging.warning('System clock is older than manifest in: '
1313                                 + apkfilename
1314                                 + '\nSet clock to that time using:\n'
1315                                 + 'sudo date -s "' + str(dt_obj) + '"')
1316
1317         # extract icons from APK zip file
1318         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1319         try:
1320             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1321         finally:
1322             apkzip.close()  # ensure that APK zip file gets closed
1323
1324         # resize existing icons for densities missing in the APK
1325         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1326
1327         if use_date_from_apk and manifest.date_time[1] != 0:
1328             default_date_param = datetime(*manifest.date_time)
1329         else:
1330             default_date_param = None
1331
1332         # Record in known apks, getting the added date at the same time..
1333         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1334                                     default_date=default_date_param)
1335         if added:
1336             apk['added'] = added
1337
1338         apkcache[apkfilename] = apk
1339         cachechanged = True
1340
1341     return False, apk, cachechanged
1342
1343
1344 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1345     """Processes the apks in the given repo directory.
1346
1347     This also extracts the icons.
1348
1349     :param apkcache: current apk cache information
1350     :param repodir: repo directory to scan
1351     :param knownapks: known apks info
1352     :param use_date_from_apk: use date from APK (instead of current date)
1353                               for newly added APKs
1354     :returns: (apks, cachechanged) where apks is a list of apk information,
1355               and cachechanged is True if the apkcache got changed.
1356     """
1357
1358     cachechanged = False
1359
1360     for icon_dir in get_all_icon_dirs(repodir):
1361         if os.path.exists(icon_dir):
1362             if options.clean:
1363                 shutil.rmtree(icon_dir)
1364                 os.makedirs(icon_dir)
1365         else:
1366             os.makedirs(icon_dir)
1367
1368     apks = []
1369     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1370         apkfilename = apkfile[len(repodir) + 1:]
1371         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1372         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1373                                              use_date_from_apk, ada, True)
1374         if skip:
1375             continue
1376         apks.append(apk)
1377         cachechanged = cachechanged or cachethis
1378
1379     return apks, cachechanged
1380
1381
1382 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1383     """
1384     Extracts icons from the given APK zip in various densities,
1385     saves them into given repo directory
1386     and stores their names in the APK metadata dictionary.
1387
1388     :param icon_filename: A string representing the icon's file name
1389     :param apk: A populated dictionary containing APK metadata.
1390                 Needs to have 'icons_src' key
1391     :param apkzip: An opened zipfile.ZipFile of the APK file
1392     :param repo_dir: The directory of the APK's repository
1393     :return: A list of icon densities that are missing
1394     """
1395     empty_densities = []
1396     for density in screen_densities:
1397         if density not in apk['icons_src']:
1398             empty_densities.append(density)
1399             continue
1400         icon_src = apk['icons_src'][density]
1401         icon_dir = get_icon_dir(repo_dir, density)
1402         icon_dest = os.path.join(icon_dir, icon_filename)
1403
1404         # Extract the icon files per density
1405         if icon_src.endswith('.xml'):
1406             png = os.path.basename(icon_src)[:-4] + '.png'
1407             for f in apkzip.namelist():
1408                 if f.endswith(png):
1409                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1410                     if m and screen_resolutions[m.group(2)] == density:
1411                         icon_src = f
1412             if icon_src.endswith('.xml'):
1413                 empty_densities.append(density)
1414                 continue
1415         try:
1416             with open(icon_dest, 'wb') as f:
1417                 f.write(get_icon_bytes(apkzip, icon_src))
1418             apk['icons'][density] = icon_filename
1419         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1420             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1421             del apk['icons_src'][density]
1422             empty_densities.append(density)
1423
1424     if '-1' in apk['icons_src']:
1425         icon_src = apk['icons_src']['-1']
1426         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1427         with open(icon_path, 'wb') as f:
1428             f.write(get_icon_bytes(apkzip, icon_src))
1429         try:
1430             im = Image.open(icon_path)
1431             dpi = px_to_dpi(im.size[0])
1432             for density in screen_densities:
1433                 if density in apk['icons']:
1434                     break
1435                 if density == screen_densities[-1] or dpi >= int(density):
1436                     apk['icons'][density] = icon_filename
1437                     shutil.move(icon_path,
1438                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1439                     empty_densities.remove(density)
1440                     break
1441         except Exception as e:
1442             logging.warning(_("Failed reading {path}: {error}")
1443                             .format(path=icon_path, error=e))
1444
1445     if apk['icons']:
1446         apk['icon'] = icon_filename
1447
1448     return empty_densities
1449
1450
1451 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1452     """
1453     Resize existing icons for densities missing in the APK to ensure all densities are available
1454
1455     :param empty_densities: A list of icon densities that are missing
1456     :param icon_filename: A string representing the icon's file name
1457     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1458     :param repo_dir: The directory of the APK's repository
1459     """
1460     # First try resizing down to not lose quality
1461     last_density = None
1462     for density in screen_densities:
1463         if density not in empty_densities:
1464             last_density = density
1465             continue
1466         if last_density is None:
1467             continue
1468         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1469
1470         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1471         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1472         fp = None
1473         try:
1474             fp = open(last_icon_path, 'rb')
1475             im = Image.open(fp)
1476
1477             size = dpi_to_px(density)
1478
1479             im.thumbnail((size, size), Image.ANTIALIAS)
1480             im.save(icon_path, "PNG")
1481             empty_densities.remove(density)
1482         except Exception as e:
1483             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1484         finally:
1485             if fp:
1486                 fp.close()
1487
1488     # Then just copy from the highest resolution available
1489     last_density = None
1490     for density in reversed(screen_densities):
1491         if density not in empty_densities:
1492             last_density = density
1493             continue
1494
1495         if last_density is None:
1496             continue
1497
1498         shutil.copyfile(
1499             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1500             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1501         )
1502         empty_densities.remove(density)
1503
1504     for density in screen_densities:
1505         icon_dir = get_icon_dir(repo_dir, density)
1506         icon_dest = os.path.join(icon_dir, icon_filename)
1507         resize_icon(icon_dest, density)
1508
1509     # Copy from icons-mdpi to icons since mdpi is the baseline density
1510     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1511     if os.path.isfile(baseline):
1512         apk['icons']['0'] = icon_filename
1513         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1514
1515
1516 def apply_info_from_latest_apk(apps, apks):
1517     """
1518     Some information from the apks needs to be applied up to the application level.
1519     When doing this, we use the info from the most recent version's apk.
1520     We deal with figuring out when the app was added and last updated at the same time.
1521     """
1522     for appid, app in apps.items():
1523         bestver = UNSET_VERSION_CODE
1524         for apk in apks:
1525             if apk['packageName'] == appid:
1526                 if apk['versionCode'] > bestver:
1527                     bestver = apk['versionCode']
1528                     bestapk = apk
1529
1530                 if 'added' in apk:
1531                     if not app.added or apk['added'] < app.added:
1532                         app.added = apk['added']
1533                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1534                         app.lastUpdated = apk['added']
1535
1536         if not app.added:
1537             logging.debug("Don't know when " + appid + " was added")
1538         if not app.lastUpdated:
1539             logging.debug("Don't know when " + appid + " was last updated")
1540
1541         if bestver == UNSET_VERSION_CODE:
1542
1543             if app.Name is None:
1544                 app.Name = app.AutoName or appid
1545             app.icon = None
1546             logging.debug("Application " + appid + " has no packages")
1547         else:
1548             if app.Name is None:
1549                 app.Name = bestapk['name']
1550             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1551             if app.CurrentVersionCode is None:
1552                 app.CurrentVersionCode = str(bestver)
1553
1554
1555 def make_categories_txt(repodir, categories):
1556     '''Write a category list in the repo to allow quick access'''
1557     catdata = ''
1558     for cat in sorted(categories):
1559         catdata += cat + '\n'
1560     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1561         f.write(catdata)
1562
1563
1564 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1565
1566     def filter_apk_list_sorted(apk_list):
1567         res = []
1568         for apk in apk_list:
1569             if apk['packageName'] == appid:
1570                 res.append(apk)
1571
1572         # Sort the apk list by version code. First is highest/newest.
1573         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1574
1575     for appid, app in apps.items():
1576
1577         if app.ArchivePolicy:
1578             keepversions = int(app.ArchivePolicy[:-9])
1579         else:
1580             keepversions = defaultkeepversions
1581
1582         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1583                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1584
1585         current_app_apks = filter_apk_list_sorted(apks)
1586         if len(current_app_apks) > keepversions:
1587             # Move back the ones we don't want.
1588             for apk in current_app_apks[keepversions:]:
1589                 move_apk_between_sections(repodir, archivedir, apk)
1590                 archapks.append(apk)
1591                 apks.remove(apk)
1592
1593         current_app_archapks = filter_apk_list_sorted(archapks)
1594         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1595             kept = 0
1596             # Move forward the ones we want again, except DisableAlgorithm
1597             for apk in current_app_archapks:
1598                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1599                     move_apk_between_sections(archivedir, repodir, apk)
1600                     archapks.remove(apk)
1601                     apks.append(apk)
1602                     kept += 1
1603                 if kept == keepversions:
1604                     break
1605
1606
1607 def move_apk_between_sections(from_dir, to_dir, apk):
1608     """move an APK from repo to archive or vice versa"""
1609
1610     def _move_file(from_dir, to_dir, filename, ignore_missing):
1611         from_path = os.path.join(from_dir, filename)
1612         if ignore_missing and not os.path.exists(from_path):
1613             return
1614         to_path = os.path.join(to_dir, filename)
1615         if not os.path.exists(to_dir):
1616             os.mkdir(to_dir)
1617         shutil.move(from_path, to_path)
1618
1619     if from_dir == to_dir:
1620         return
1621
1622     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1623     _move_file(from_dir, to_dir, apk['apkName'], False)
1624     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1625     for density in all_screen_densities:
1626         from_icon_dir = get_icon_dir(from_dir, density)
1627         to_icon_dir = get_icon_dir(to_dir, density)
1628         if density not in apk['icons']:
1629             continue
1630         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1631     if 'srcname' in apk:
1632         _move_file(from_dir, to_dir, apk['srcname'], False)
1633
1634
1635 def add_apks_to_per_app_repos(repodir, apks):
1636     apks_per_app = dict()
1637     for apk in apks:
1638         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1639         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1640         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1641         apks_per_app[apk['packageName']] = apk
1642
1643         if not os.path.exists(apk['per_app_icons']):
1644             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1645             os.makedirs(apk['per_app_icons'])
1646
1647         apkpath = os.path.join(repodir, apk['apkName'])
1648         shutil.copy(apkpath, apk['per_app_repo'])
1649         apksigpath = apkpath + '.sig'
1650         if os.path.exists(apksigpath):
1651             shutil.copy(apksigpath, apk['per_app_repo'])
1652         apkascpath = apkpath + '.asc'
1653         if os.path.exists(apkascpath):
1654             shutil.copy(apkascpath, apk['per_app_repo'])
1655
1656
1657 def create_metadata_from_template(apk):
1658     '''create a new metadata file using internal or external template
1659
1660     Generate warnings for apk's with no metadata (or create skeleton
1661     metadata files, if requested on the command line).  Though the
1662     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1663     since those impose things on the metadata file made from the
1664     template: field sort order, empty field value, formatting, etc.
1665     '''
1666
1667     import yaml
1668     if os.path.exists('template.yml'):
1669         with open('template.yml') as f:
1670             metatxt = f.read()
1671         if 'name' in apk and apk['name'] != '':
1672             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1673                              r'\1 ' + apk['name'],
1674                              metatxt,
1675                              flags=re.IGNORECASE | re.MULTILINE)
1676         else:
1677             logging.warning(_('{appid} does not have a name! Using package name instead.')
1678                             .format(appid=apk['packageName']))
1679             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1680                              r'\1 ' + apk['packageName'],
1681                              metatxt,
1682                              flags=re.IGNORECASE | re.MULTILINE)
1683         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1684             f.write(metatxt)
1685     else:
1686         app = dict()
1687         app['Categories'] = [os.path.basename(os.getcwd())]
1688         # include some blanks as part of the template
1689         app['AuthorName'] = ''
1690         app['Summary'] = ''
1691         app['WebSite'] = ''
1692         app['IssueTracker'] = ''
1693         app['SourceCode'] = ''
1694         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1695         if 'name' in apk and apk['name'] != '':
1696             app['Name'] = apk['name']
1697         else:
1698             logging.warning(_('{appid} does not have a name! Using package name instead.')
1699                             .format(appid=apk['packageName']))
1700             app['Name'] = apk['packageName']
1701         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1702             yaml.dump(app, f, default_flow_style=False)
1703     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1704
1705
1706 config = None
1707 options = None
1708
1709
1710 def main():
1711
1712     global config, options
1713
1714     # Parse command line...
1715     parser = ArgumentParser()
1716     common.setup_global_opts(parser)
1717     parser.add_argument("--create-key", action="store_true", default=False,
1718                         help=_("Create a repo signing key in a keystore"))
1719     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1720                         help=_("Create skeleton metadata files that are missing"))
1721     parser.add_argument("--delete-unknown", action="store_true", default=False,
1722                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1723     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1724                         help=_("Report on build data status"))
1725     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1726                         help=_("Interactively ask about things that need updating."))
1727     parser.add_argument("-I", "--icons", action="store_true", default=False,
1728                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1729     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1730                         help=_("Specify editor to use in interactive mode. Default " +
1731                                "is {path}").format(path='/etc/alternatives/editor'))
1732     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1733                         help=_("Update the wiki"))
1734     parser.add_argument("--pretty", action="store_true", default=False,
1735                         help=_("Produce human-readable index.xml"))
1736     parser.add_argument("--clean", action="store_true", default=False,
1737                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1738     parser.add_argument("--nosign", action="store_true", default=False,
1739                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1740     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1741                         help=_("Use date from APK instead of current time for newly added APKs"))
1742     parser.add_argument("--rename-apks", action="store_true", default=False,
1743                         help=_("Rename APK files that do not match package.name_123.apk"))
1744     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1745                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1746     metadata.add_metadata_arguments(parser)
1747     options = parser.parse_args()
1748     metadata.warnings_action = options.W
1749
1750     config = common.read_config(options)
1751
1752     if not ('jarsigner' in config and 'keytool' in config):
1753         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1754
1755     repodirs = ['repo']
1756     if config['archive_older'] != 0:
1757         repodirs.append('archive')
1758         if not os.path.exists('archive'):
1759             os.mkdir('archive')
1760
1761     if options.icons:
1762         resize_all_icons(repodirs)
1763         sys.exit(0)
1764
1765     if options.rename_apks:
1766         options.clean = True
1767
1768     # check that icons exist now, rather than fail at the end of `fdroid update`
1769     for k in ['repo_icon', 'archive_icon']:
1770         if k in config:
1771             if not os.path.exists(config[k]):
1772                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1773                                  .format(name=k, path=config[k]))
1774                 sys.exit(1)
1775
1776     # if the user asks to create a keystore, do it now, reusing whatever it can
1777     if options.create_key:
1778         if os.path.exists(config['keystore']):
1779             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1780             logging.critical("\t'" + config['keystore'] + "'")
1781             sys.exit(1)
1782
1783         if 'repo_keyalias' not in config:
1784             config['repo_keyalias'] = socket.getfqdn()
1785             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1786         if 'keydname' not in config:
1787             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1788             common.write_to_config(config, 'keydname', config['keydname'])
1789         if 'keystore' not in config:
1790             config['keystore'] = common.default_config['keystore']
1791             common.write_to_config(config, 'keystore', config['keystore'])
1792
1793         password = common.genpassword()
1794         if 'keystorepass' not in config:
1795             config['keystorepass'] = password
1796             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1797         if 'keypass' not in config:
1798             config['keypass'] = password
1799             common.write_to_config(config, 'keypass', config['keypass'])
1800         common.genkeystore(config)
1801
1802     # Get all apps...
1803     apps = metadata.read_metadata()
1804
1805     # Generate a list of categories...
1806     categories = set()
1807     for app in apps.values():
1808         categories.update(app.Categories)
1809
1810     # Read known apks data (will be updated and written back when we've finished)
1811     knownapks = common.KnownApks()
1812
1813     # Get APK cache
1814     apkcache = get_cache()
1815
1816     # Delete builds for disabled apps
1817     delete_disabled_builds(apps, apkcache, repodirs)
1818
1819     # Scan all apks in the main repo
1820     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1821
1822     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1823                                            options.use_date_from_apk)
1824     cachechanged = cachechanged or fcachechanged
1825     apks += files
1826     for apk in apks:
1827         if apk['packageName'] not in apps:
1828             if options.create_metadata:
1829                 create_metadata_from_template(apk)
1830                 apps = metadata.read_metadata()
1831             else:
1832                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1833                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1834                 if options.delete_unknown:
1835                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1836                                  .format(apkfilename=apk['apkName']))
1837                     rmf = os.path.join(repodirs[0], apk['apkName'])
1838                     if not os.path.exists(rmf):
1839                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1840                     else:
1841                         os.remove(rmf)
1842                 else:
1843                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1844
1845     copy_triple_t_store_metadata(apps)
1846     insert_obbs(repodirs[0], apps, apks)
1847     insert_localized_app_metadata(apps)
1848     translate_per_build_anti_features(apps, apks)
1849
1850     # Scan the archive repo for apks as well
1851     if len(repodirs) > 1:
1852         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1853         if cc:
1854             cachechanged = True
1855     else:
1856         archapks = []
1857
1858     # Apply information from latest apks to the application and update dates
1859     apply_info_from_latest_apk(apps, apks + archapks)
1860
1861     # Sort the app list by name, then the web site doesn't have to by default.
1862     # (we had to wait until we'd scanned the apks to do this, because mostly the
1863     # name comes from there!)
1864     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1865
1866     # APKs are placed into multiple repos based on the app package, providing
1867     # per-app subscription feeds for nightly builds and things like it
1868     if config['per_app_repos']:
1869         add_apks_to_per_app_repos(repodirs[0], apks)
1870         for appid, app in apps.items():
1871             repodir = os.path.join(appid, 'fdroid', 'repo')
1872             appdict = dict()
1873             appdict[appid] = app
1874             if os.path.isdir(repodir):
1875                 index.make(appdict, [appid], apks, repodir, False)
1876             else:
1877                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1878         return
1879
1880     if len(repodirs) > 1:
1881         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1882
1883     # Make the index for the main repo...
1884     index.make(apps, sortedids, apks, repodirs[0], False)
1885     make_categories_txt(repodirs[0], categories)
1886
1887     # If there's an archive repo,  make the index for it. We already scanned it
1888     # earlier on.
1889     if len(repodirs) > 1:
1890         index.make(apps, sortedids, archapks, repodirs[1], True)
1891
1892     git_remote = config.get('binary_transparency_remote')
1893     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1894         from . import btlog
1895         btlog.make_binary_transparency_log(repodirs)
1896
1897     if config['update_stats']:
1898         # Update known apks info...
1899         knownapks.writeifchanged()
1900
1901         # Generate latest apps data for widget
1902         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1903             data = ''
1904             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1905                 for line in f:
1906                     appid = line.rstrip()
1907                     data += appid + "\t"
1908                     app = apps[appid]
1909                     data += app.Name + "\t"
1910                     if app.icon is not None:
1911                         data += app.icon + "\t"
1912                     data += app.License + "\n"
1913             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1914                 f.write(data)
1915
1916     if cachechanged:
1917         write_cache(apkcache)
1918
1919     # Update the wiki...
1920     if options.wiki:
1921         update_wiki(apps, sortedids, apks + archapks)
1922
1923     logging.info(_("Finished"))
1924
1925
1926 if __name__ == "__main__":
1927     main()