fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime, timedelta
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in app.AntiFeatures:
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.Bitcoin,
 153             app.Litecoin,
 154             app.License,
 155             requiresroot,
 156             app.AuthorName,
 157             app.AuthorEmail)
 158
 159         if app.Provides:
 160             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 161
 162         wikidata += app.Summary
 163         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 164
 165         wikidata += "=Description=\n"
 166         wikidata += metadata.description_wiki(app.Description) + "\n"
 167
 168         wikidata += "=Maintainer Notes=\n"
 169         if app.MaintainerNotes:
 170             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 171         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 172
 173         # Get a list of all packages for this application...
 174         apklist = []
 175         gotcurrentver = False
 176         cantupdate = False
 177         buildfails = False
 178         for apk in apks:
 179             if apk['packageName'] == appid:
 180                 if str(apk['versionCode']) == app.CurrentVersionCode:
 181                     gotcurrentver = True
 182                 apklist.append(apk)
 183         # Include ones we can't build, as a special case...
 184         for build in app.builds:
 185             if build.disable:
 186                 if build.versionCode == app.CurrentVersionCode:
 187                     cantupdate = True
 188                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 189                 apklist.append({'versionCode': int(build.versionCode),
 190                                 'versionName': build.versionName,
 191                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 192                                 })
 193             else:
 194                 builtit = False
 195                 for apk in apklist:
 196                     if apk['versionCode'] == int(build.versionCode):
 197                         builtit = True
 198                         break
 199                 if not builtit:
 200                     buildfails = True
 201                     apklist.append({'versionCode': int(build.versionCode),
 202                                     'versionName': build.versionName,
 203                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 204                                     })
 205         if app.CurrentVersionCode == '0':
 206             cantupdate = True
 207         # Sort with most recent first...
 208         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 209
 210         wikidata += "=Versions=\n"
 211         if len(apklist) == 0:
 212             wikidata += "We currently have no versions of this app available."
 213         elif not gotcurrentver:
 214             wikidata += "We don't have the current version of this app."
 215         else:
 216             wikidata += "We have the current version of this app."
 217         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 218         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 219         if len(app.NoSourceSince) > 0:
 220             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 221         if len(app.CurrentVersion) > 0:
 222             wikidata += "The current (recommended) version is " + app.CurrentVersion
 223             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 224         validapks = 0
 225         for apk in apklist:
 226             wikidata += "==" + apk['versionName'] + "==\n"
 227
 228             if 'buildproblem' in apk:
 229                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 230             else:
 231                 validapks += 1
 232                 wikidata += "This version is built and signed by "
 233                 if 'srcname' in apk:
 234                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 235                 else:
 236                     wikidata += "the original developer.\n\n"
 237             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 238
 239         wikidata += '\n[[Category:' + wikicat + ']]\n'
 240         if len(app.NoSourceSince) > 0:
 241             wikidata += '\n[[Category:Apps missing source code]]\n'
 242         if validapks == 0 and not app.Disabled:
 243             wikidata += '\n[[Category:Apps with no packages]]\n'
 244         if cantupdate and not app.Disabled:
 245             wikidata += "\n[[Category:Apps we cannot update]]\n"
 246         if buildfails and not app.Disabled:
 247             wikidata += "\n[[Category:Apps with failing builds]]\n"
 248         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 249             wikidata += '\n[[Category:Apps to Update]]\n'
 250         if app.Disabled:
 251             wikidata += '\n[[Category:Apps that are disabled]]\n'
 252         if app.UpdateCheckMode == 'None' and not app.Disabled:
 253             wikidata += '\n[[Category:Apps with no update check]]\n'
 254         for appcat in app.Categories:
 255             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 256
 257         # We can't have underscores in the page name, even if they're in
 258         # the package ID, because MediaWiki messes with them...
 259         pagename = appid.replace('_', ' ')
 260
 261         # Drop a trailing newline, because mediawiki is going to drop it anyway
 262         # and it we don't we'll think the page has changed when it hasn't...
 263         if wikidata.endswith('\n'):
 264             wikidata = wikidata[:-1]
 265
 266         generated_pages[pagename] = wikidata
 267
 268         # Make a redirect from the name to the ID too, unless there's
 269         # already an existing page with the name and it isn't a redirect.
 270         noclobber = False
 271         apppagename = app.Name.replace('_', ' ')
 272         apppagename = apppagename.replace('{', '')
 273         apppagename = apppagename.replace('}', ' ')
 274         apppagename = apppagename.replace(':', ' ')
 275         apppagename = apppagename.replace('[', ' ')
 276         apppagename = apppagename.replace(']', ' ')
 277         # Drop double spaces caused mostly by replacing ':' above
 278         apppagename = apppagename.replace('  ', ' ')
 279         for expagename in site.allpages(prefix=apppagename,
 280                                         filterredir='nonredirects',
 281                                         generator=False):
 282             if expagename == apppagename:
 283                 noclobber = True
 284         # Another reason not to make the redirect page is if the app name
 285         # is the same as it's ID, because that will overwrite the real page
 286         # with an redirect to itself! (Although it seems like an odd
 287         # scenario this happens a lot, e.g. where there is metadata but no
 288         # builds or binaries to extract a name from.
 289         if apppagename == pagename:
 290             noclobber = True
 291         if not noclobber:
 292             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 293
 294     for tcat, genp in [(wikicat, generated_pages),
 295                        (wikiredircat, generated_redirects)]:
 296         catpages = site.Pages['Category:' + tcat]
 297         existingpages = []
 298         for page in catpages:
 299             existingpages.append(page.name)
 300             if page.name in genp:
 301                 pagetxt = page.edit()
 302                 if pagetxt != genp[page.name]:
 303                     logging.debug("Updating modified page " + page.name)
 304                     page.save(genp[page.name], summary='Auto-updated')
 305                 else:
 306                     logging.debug("Page " + page.name + " is unchanged")
 307             else:
 308                 logging.warn("Deleting page " + page.name)
 309                 page.delete('No longer published')
 310         for pagename, text in genp.items():
 311             logging.debug("Checking " + pagename)
 312             if pagename not in existingpages:
 313                 logging.debug("Creating page " + pagename)
 314                 try:
 315                     newpage = site.Pages[pagename]
 316                     newpage.save(text, summary='Auto-created')
 317                 except Exception as e:
 318                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 319
 320     # Purge server cache to ensure counts are up to date
 321     site.pages['Repository Maintenance'].purge()
 322
 323
 324 def delete_disabled_builds(apps, apkcache, repodirs):
 325     """Delete disabled build outputs.
 326
 327     :param apps: list of all applications, as per metadata.read_metadata
 328     :param apkcache: current apk cache information
 329     :param repodirs: the repo directories to process
 330     """
 331     for appid, app in apps.items():
 332         for build in app['builds']:
 333             if not build.disable:
 334                 continue
 335             apkfilename = common.get_release_filename(app, build)
 336             iconfilename = "%s.%s.png" % (
 337                 appid,
 338                 build.versionCode)
 339             for repodir in repodirs:
 340                 files = [
 341                     os.path.join(repodir, apkfilename),
 342                     os.path.join(repodir, apkfilename + '.asc'),
 343                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 344                 ]
 345                 for density in all_screen_densities:
 346                     repo_dir = get_icon_dir(repodir, density)
 347                     files.append(os.path.join(repo_dir, iconfilename))
 348
 349                 for f in files:
 350                     if os.path.exists(f):
 351                         logging.info("Deleting disabled build output " + f)
 352                         os.remove(f)
 353             if apkfilename in apkcache:
 354                 del apkcache[apkfilename]
 355
 356
 357 def resize_icon(iconpath, density):
 358
 359     if not os.path.isfile(iconpath):
 360         return
 361
 362     fp = None
 363     try:
 364         fp = open(iconpath, 'rb')
 365         im = Image.open(fp)
 366         size = dpi_to_px(density)
 367
 368         if any(length > size for length in im.size):
 369             oldsize = im.size
 370             im.thumbnail((size, size), Image.ANTIALIAS)
 371             logging.debug("%s was too large at %s - new size is %s" % (
 372                 iconpath, oldsize, im.size))
 373             im.save(iconpath, "PNG")
 374
 375     except Exception as e:
 376         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 377
 378     finally:
 379         if fp:
 380             fp.close()
 381
 382
 383 def resize_all_icons(repodirs):
 384     """Resize all icons that exceed the max size
 385
 386     :param repodirs: the repo directories to process
 387     """
 388     for repodir in repodirs:
 389         for density in screen_densities:
 390             icon_dir = get_icon_dir(repodir, density)
 391             icon_glob = os.path.join(icon_dir, '*.png')
 392             for iconpath in glob.glob(icon_glob):
 393                 resize_icon(iconpath, density)
 394
 395
 396 def getsig(apkpath):
 397     """ Get the signing certificate of an apk. To get the same md5 has that
 398     Android gets, we encode the .RSA certificate in a specific format and pass
 399     it hex-encoded to the md5 digest algorithm.
 400
 401     :param apkpath: path to the apk
 402     :returns: A string containing the md5 of the signature of the apk or None
 403               if an error occurred.
 404     """
 405
 406     with zipfile.ZipFile(apkpath, 'r') as apk:
 407         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 408
 409         if len(certs) < 1:
 410             logging.error("Found no signing certificates on %s" % apkpath)
 411             return None
 412         if len(certs) > 1:
 413             logging.error("Found multiple signing certificates on %s" % apkpath)
 414             return None
 415
 416         cert = apk.read(certs[0])
 417
 418     cert_encoded = common.get_certificate(cert)
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_cache_file():
 424     return os.path.join('tmp', 'apkcache')
 425
 426
 427 def get_cache():
 428     """Get the cached dict of the APK index
 429
 430     Gather information about all the apk files in the repo directory,
 431     using cached data if possible. Some of the index operations take a
 432     long time, like calculating the SHA-256 and verifying the APK
 433     signature.
 434
 435     The cache is invalidated if the metadata version is different, or
 436     the 'allow_disabled_algorithms' config/option is different.  In
 437     those cases, there is no easy way to know what has changed from
 438     the cache, so just rerun the whole thing.
 439
 440     :return: apkcache
 441
 442     """
 443     apkcachefile = get_cache_file()
 444     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 445     if not options.clean and os.path.exists(apkcachefile):
 446         with open(apkcachefile, 'rb') as cf:
 447             apkcache = pickle.load(cf, encoding='utf-8')
 448         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 449            or apkcache.get('allow_disabled_algorithms') != ada:
 450             apkcache = {}
 451     else:
 452         apkcache = {}
 453
 454     apkcache["METADATA_VERSION"] = METADATA_VERSION
 455     apkcache['allow_disabled_algorithms'] = ada
 456
 457     return apkcache
 458
 459
 460 def write_cache(apkcache):
 461     apkcachefile = get_cache_file()
 462     cache_path = os.path.dirname(apkcachefile)
 463     if not os.path.exists(cache_path):
 464         os.makedirs(cache_path)
 465     with open(apkcachefile, 'wb') as cf:
 466         pickle.dump(apkcache, cf)
 467
 468
 469 def get_icon_bytes(apkzip, iconsrc):
 470     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 471     try:
 472         return apkzip.read(iconsrc)
 473     except KeyError:
 474         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 475
 476
 477 def sha256sum(filename):
 478     '''Calculate the sha256 of the given file'''
 479     sha = hashlib.sha256()
 480     with open(filename, 'rb') as f:
 481         while True:
 482             t = f.read(16384)
 483             if len(t) == 0:
 484                 break
 485             sha.update(t)
 486     return sha.hexdigest()
 487
 488
 489 def has_known_vulnerability(filename):
 490     """checks for known vulnerabilities in the APK
 491
 492     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 493     version.  Google also enforces this:
 494     https://support.google.com/faqs/answer/6376725?hl=en
 495
 496     Checks whether there are more than one classes.dex or AndroidManifest.xml
 497     files, which is invalid and an essential part of the "Master Key" attack.
 498
 499     http://www.saurik.com/id/17
 500     """
 501
 502     # statically load this pattern
 503     if not hasattr(has_known_vulnerability, "pattern"):
 504         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 505
 506     files_in_apk = set()
 507     with zipfile.ZipFile(filename) as zf:
 508         for name in zf.namelist():
 509             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 510                 lib = zf.open(name)
 511                 while True:
 512                     chunk = lib.read(4096)
 513                     if chunk == b'':
 514                         break
 515                     m = has_known_vulnerability.pattern.search(chunk)
 516                     if m:
 517                         version = m.group(1).decode('ascii')
 518                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 519                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 520                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 521                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 522                         else:
 523                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 524                             return True
 525                         break
 526             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 527                 if name in files_in_apk:
 528                     return True
 529                 files_in_apk.add(name)
 530
 531     return False
 532
 533
 534 def insert_obbs(repodir, apps, apks):
 535     """Scans the .obb files in a given repo directory and adds them to the
 536     relevant APK instances.  OBB files have versionCodes like APK
 537     files, and they are loosely associated.  If there is an OBB file
 538     present, then any APK with the same or higher versionCode will use
 539     that OBB file.  There are two OBB types: main and patch, each APK
 540     can only have only have one of each.
 541
 542     https://developer.android.com/google/play/expansion-files.html
 543
 544     :param repodir: repo directory to scan
 545     :param apps: list of current, valid apps
 546     :param apks: current information on all APKs
 547
 548     """
 549
 550     def obbWarnDelete(f, msg):
 551         logging.warning(msg + f)
 552         if options.delete_unknown:
 553             logging.error("Deleting unknown file: " + f)
 554             os.remove(f)
 555
 556     obbs = []
 557     java_Integer_MIN_VALUE = -pow(2, 31)
 558     currentPackageNames = apps.keys()
 559     for f in glob.glob(os.path.join(repodir, '*.obb')):
 560         obbfile = os.path.basename(f)
 561         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 562         chunks = obbfile.split('.')
 563         if chunks[0] != 'main' and chunks[0] != 'patch':
 564             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 565             continue
 566         if not re.match(r'^-?[0-9]+$', chunks[1]):
 567             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 568             continue
 569         versionCode = int(chunks[1])
 570         packagename = ".".join(chunks[2:-1])
 571
 572         highestVersionCode = java_Integer_MIN_VALUE
 573         if packagename not in currentPackageNames:
 574             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 575             continue
 576         for apk in apks:
 577             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 578                 highestVersionCode = apk['versionCode']
 579         if versionCode > highestVersionCode:
 580             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 581                           + ') than any APK: ')
 582             continue
 583         obbsha256 = sha256sum(f)
 584         obbs.append((packagename, versionCode, obbfile, obbsha256))
 585
 586     for apk in apks:
 587         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 588             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 589                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 590                     apk['obbMainFile'] = obbfile
 591                     apk['obbMainFileSha256'] = obbsha256
 592                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 593                     apk['obbPatchFile'] = obbfile
 594                     apk['obbPatchFileSha256'] = obbsha256
 595             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 596                 break
 597
 598
 599 def translate_per_build_anti_features(apps, apks):
 600     """Grab the anti-features list from the build metadata
 601
 602     For most Anti-Features, they are really most applicable per-APK,
 603     not for an app.  An app can fix a vulnerability, add/remove
 604     tracking, etc.  This reads the 'antifeatures' list from the Build
 605     entries in the fdroiddata metadata file, then transforms it into
 606     the 'antiFeatures' list of unique items for the index.
 607
 608     The field key is all lower case in the metadata file to match the
 609     rest of the Build fields.  It is 'antiFeatures' camel case in the
 610     implementation, index, and fdroidclient since it is translated
 611     from the build 'antifeatures' field, not directly included.
 612
 613     """
 614
 615     antiFeatures = dict()
 616     for packageName, app in apps.items():
 617         d = dict()
 618         for build in app['builds']:
 619             afl = build.get('antifeatures')
 620             if afl:
 621                 d[int(build.versionCode)] = afl
 622         if len(d) > 0:
 623             antiFeatures[packageName] = d
 624
 625     for apk in apks:
 626         d = antiFeatures.get(apk['packageName'])
 627         if d:
 628             afl = d.get(apk['versionCode'])
 629             if afl:
 630                 apk['antiFeatures'].update(afl)
 631
 632
 633 def _get_localized_dict(app, locale):
 634     '''get the dict to add localized store metadata to'''
 635     if 'localized' not in app:
 636         app['localized'] = collections.OrderedDict()
 637     if locale not in app['localized']:
 638         app['localized'][locale] = collections.OrderedDict()
 639     return app['localized'][locale]
 640
 641
 642 def _set_localized_text_entry(app, locale, key, f):
 643     limit = config['char_limits'][key]
 644     localized = _get_localized_dict(app, locale)
 645     with open(f) as fp:
 646         text = fp.read()[:limit]
 647         if len(text) > 0:
 648             localized[key] = text
 649
 650
 651 def _set_author_entry(app, key, f):
 652     limit = config['char_limits']['author']
 653     with open(f) as fp:
 654         text = fp.read()[:limit]
 655         if len(text) > 0:
 656             app[key] = text
 657
 658
 659 def copy_triple_t_store_metadata(apps):
 660     """Include store metadata from the app's source repo
 661
 662     The Triple-T Gradle Play Publisher is a plugin that has a standard
 663     file layout for all of the metadata and graphics that the Google
 664     Play Store accepts.  Since F-Droid has the git repo, it can just
 665     pluck those files directly.  This method reads any text files into
 666     the app dict, then copies any graphics into the fdroid repo
 667     directory structure.
 668
 669     This needs to be run before insert_localized_app_metadata() so that
 670     the graphics files that are copied into the fdroid repo get
 671     properly indexed.
 672
 673     https://github.com/Triple-T/gradle-play-publisher#upload-images
 674     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 675
 676     """
 677
 678     if not os.path.isdir('build'):
 679         return  # nothing to do
 680
 681     for packageName, app in apps.items():
 682         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 683             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 684             for root, dirs, files in os.walk(d):
 685                 segments = root.split('/')
 686                 locale = segments[-2]
 687                 for f in files:
 688                     if f == 'fulldescription':
 689                         _set_localized_text_entry(app, locale, 'description',
 690                                                   os.path.join(root, f))
 691                         continue
 692                     elif f == 'shortdescription':
 693                         _set_localized_text_entry(app, locale, 'summary',
 694                                                   os.path.join(root, f))
 695                         continue
 696                     elif f == 'title':
 697                         _set_localized_text_entry(app, locale, 'name',
 698                                                   os.path.join(root, f))
 699                         continue
 700                     elif f == 'video':
 701                         _set_localized_text_entry(app, locale, 'video',
 702                                                   os.path.join(root, f))
 703                         continue
 704                     elif f == 'whatsnew':
 705                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 706                                                   os.path.join(root, f))
 707                         continue
 708                     elif f == 'contactEmail':
 709                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 710                         continue
 711                     elif f == 'contactPhone':
 712                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 713                         continue
 714                     elif f == 'contactWebsite':
 715                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 716                         continue
 717
 718                     base, extension = common.get_extension(f)
 719                     dirname = os.path.basename(root)
 720                     if extension in ALLOWED_EXTENSIONS \
 721                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 722                         if segments[-2] == 'listing':
 723                             locale = segments[-3]
 724                         else:
 725                             locale = segments[-2]
 726                         destdir = os.path.join('repo', packageName, locale, dirname)
 727                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 728                         sourcefile = os.path.join(root, f)
 729                         destfile = os.path.join(destdir, os.path.basename(f))
 730                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 731                         shutil.copy(sourcefile, destfile)
 732
 733
 734 def insert_localized_app_metadata(apps):
 735     """scans standard locations for graphics and localized text
 736
 737     Scans for localized description files, store graphics, and
 738     screenshot PNG files in statically defined screenshots directory
 739     and adds them to the app metadata.  The screenshots and graphic
 740     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 741     and must be in the following layout:
 742     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 743
 744     repo/packageName/locale/featureGraphic.png
 745     repo/packageName/locale/phoneScreenshots/1.png
 746     repo/packageName/locale/phoneScreenshots/2.png
 747
 748     The changelog files must be text files named with the versionCode
 749     ending with ".txt" and must be in the following layout:
 750     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 751
 752     repo/packageName/locale/changelogs/12345.txt
 753
 754     This will scan the each app's source repo then the metadata/ dir
 755     for these standard locations of changelog files.  If it finds
 756     them, they will be added to the dict of all packages, with the
 757     versions in the metadata/ folder taking precendence over the what
 758     is in the app's source repo.
 759
 760     Where "packageName" is the app's packageName and "locale" is the locale
 761     of the graphics, e.g. what language they are in, using the IETF RFC5646
 762     format (en-US, fr-CA, es-MX, etc).
 763
 764     This will also scan the app's git for a fastlane folder, and the
 765     metadata/ folder and the apps' source repos for standard locations
 766     of graphic and screenshot files.  If it finds them, it will copy
 767     them into the repo.  The fastlane files follow this pattern:
 768     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 769
 770     """
 771
 772     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 773     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 774     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 775
 776     for srcd in sorted(sourcedirs):
 777         if not os.path.isdir(srcd):
 778             continue
 779         for root, dirs, files in os.walk(srcd):
 780             segments = root.split('/')
 781             packageName = segments[1]
 782             if packageName not in apps:
 783                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 784                 continue
 785             locale = segments[-1]
 786             destdir = os.path.join('repo', packageName, locale)
 787             for f in files:
 788                 if f in ('description.txt', 'full_description.txt'):
 789                     _set_localized_text_entry(apps[packageName], locale, 'description',
 790                                               os.path.join(root, f))
 791                     continue
 792                 elif f in ('summary.txt', 'short_description.txt'):
 793                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 794                                               os.path.join(root, f))
 795                     continue
 796                 elif f in ('name.txt', 'title.txt'):
 797                     _set_localized_text_entry(apps[packageName], locale, 'name',
 798                                               os.path.join(root, f))
 799                     continue
 800                 elif f == 'video.txt':
 801                     _set_localized_text_entry(apps[packageName], locale, 'video',
 802                                               os.path.join(root, f))
 803                     continue
 804                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 805                     locale = segments[-2]
 806                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 807                                               os.path.join(root, f))
 808                     continue
 809
 810                 base, extension = common.get_extension(f)
 811                 if locale == 'images':
 812                     locale = segments[-2]
 813                     destdir = os.path.join('repo', packageName, locale)
 814                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 815                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 816                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 817                     shutil.copy(os.path.join(root, f), destdir)
 818             for d in dirs:
 819                 if d in SCREENSHOT_DIRS:
 820                     for f in glob.glob(os.path.join(root, d, '*.*')):
 821                         _, extension = common.get_extension(f)
 822                         if extension in ALLOWED_EXTENSIONS:
 823                             screenshotdestdir = os.path.join(destdir, d)
 824                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 825                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 826                             shutil.copy(f, screenshotdestdir)
 827
 828     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 829     for d in repofiles:
 830         if not os.path.isdir(d):
 831             continue
 832         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 833             if not os.path.isfile(f):
 834                 continue
 835             segments = f.split('/')
 836             packageName = segments[1]
 837             locale = segments[2]
 838             screenshotdir = segments[3]
 839             filename = os.path.basename(f)
 840             base, extension = common.get_extension(filename)
 841
 842             if packageName not in apps:
 843                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 844                                 % (filename, packageName))
 845                 continue
 846             graphics = _get_localized_dict(apps[packageName], locale)
 847
 848             if extension not in ALLOWED_EXTENSIONS:
 849                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 850             elif base in GRAPHIC_NAMES:
 851                 # there can only be zero or one of these per locale
 852                 graphics[base] = filename
 853             elif screenshotdir in SCREENSHOT_DIRS:
 854                 # there can any number of these per locale
 855                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 856                 if screenshotdir not in graphics:
 857                     graphics[screenshotdir] = []
 858                 graphics[screenshotdir].append(filename)
 859             else:
 860                 logging.warning('Unsupported graphics file found: ' + f)
 861
 862
 863 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 864     """Scan a repo for all files with an extension except APK/OBB
 865
 866     :param apkcache: current cached info about all repo files
 867     :param repodir: repo directory to scan
 868     :param knownapks: list of all known files, as per metadata.read_metadata
 869     :param use_date_from_file: use date from file (instead of current date)
 870                                for newly added files
 871     """
 872
 873     cachechanged = False
 874     repo_files = []
 875     repodir = repodir.encode('utf-8')
 876     for name in os.listdir(repodir):
 877         file_extension = common.get_file_extension(name)
 878         if file_extension == 'apk' or file_extension == 'obb':
 879             continue
 880         filename = os.path.join(repodir, name)
 881         name_utf8 = name.decode('utf-8')
 882         if filename.endswith(b'_src.tar.gz'):
 883             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 884             continue
 885         if not common.is_repo_file(filename):
 886             continue
 887         stat = os.stat(filename)
 888         if stat.st_size == 0:
 889             raise FDroidException(filename + ' is zero size!')
 890
 891         shasum = sha256sum(filename)
 892         usecache = False
 893         if name in apkcache:
 894             repo_file = apkcache[name]
 895             # added time is cached as tuple but used here as datetime instance
 896             if 'added' in repo_file:
 897                 a = repo_file['added']
 898                 if isinstance(a, datetime):
 899                     repo_file['added'] = a
 900                 else:
 901                     repo_file['added'] = datetime(*a[:6])
 902             if repo_file.get('hash') == shasum:
 903                 logging.debug("Reading " + name_utf8 + " from cache")
 904                 usecache = True
 905             else:
 906                 logging.debug("Ignoring stale cache data for " + name_utf8)
 907
 908         if not usecache:
 909             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 910             repo_file = collections.OrderedDict()
 911             repo_file['name'] = os.path.splitext(name_utf8)[0]
 912             # TODO rename apkname globally to something more generic
 913             repo_file['apkName'] = name_utf8
 914             repo_file['hash'] = shasum
 915             repo_file['hashType'] = 'sha256'
 916             repo_file['versionCode'] = 0
 917             repo_file['versionName'] = shasum
 918             # the static ID is the SHA256 unless it is set in the metadata
 919             repo_file['packageName'] = shasum
 920
 921             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 922             if m:
 923                 repo_file['packageName'] = m.group(1)
 924                 repo_file['versionCode'] = int(m.group(2))
 925             srcfilename = name + b'_src.tar.gz'
 926             if os.path.exists(os.path.join(repodir, srcfilename)):
 927                 repo_file['srcname'] = srcfilename.decode('utf-8')
 928             repo_file['size'] = stat.st_size
 929
 930             apkcache[name] = repo_file
 931             cachechanged = True
 932
 933         if use_date_from_file:
 934             timestamp = stat.st_ctime
 935             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 936         else:
 937             default_date_param = None
 938
 939         # Record in knownapks, getting the added date at the same time..
 940         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 941                                     default_date=default_date_param)
 942         if added:
 943             repo_file['added'] = added
 944
 945         repo_files.append(repo_file)
 946
 947     return repo_files, cachechanged
 948
 949
 950 def scan_apk(apk_file):
 951     """
 952     Scans an APK file and returns dictionary with metadata of the APK.
 953
 954     Attention: This does *not* verify that the APK signature is correct.
 955
 956     :param apk_file: The (ideally absolute) path to the APK file
 957     :raises BuildException
 958     :return A dict containing APK metadata
 959     """
 960     apk = {
 961         'hash': sha256sum(apk_file),
 962         'hashType': 'sha256',
 963         'uses-permission': [],
 964         'uses-permission-sdk-23': [],
 965         'features': [],
 966         'icons_src': {},
 967         'icons': {},
 968         'antiFeatures': set(),
 969     }
 970
 971     if SdkToolsPopen(['aapt', 'version'], output=False):
 972         scan_apk_aapt(apk, apk_file)
 973     else:
 974         scan_apk_androguard(apk, apk_file)
 975
 976     # Get the signature, or rather the signing key fingerprints
 977     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 978     apk['sig'] = getsig(apk_file)
 979     if not apk['sig']:
 980         raise BuildException("Failed to get apk signature")
 981     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
 982                                                                apk_file))
 983     if not apk.get('signer'):
 984         raise BuildException("Failed to get apk signing key fingerprint")
 985
 986     # Get size of the APK
 987     apk['size'] = os.path.getsize(apk_file)
 988
 989     if 'minSdkVersion' not in apk:
 990         logging.warning("No SDK version information found in {0}".format(apk_file))
 991         apk['minSdkVersion'] = 1
 992
 993     # Check for known vulnerabilities
 994     if has_known_vulnerability(apk_file):
 995         apk['antiFeatures'].add('KnownVuln')
 996
 997     return apk
 998
 999
1000 def scan_apk_aapt(apk, apkfile):
1001     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1002     if p.returncode != 0:
1003         if options.delete_unknown:
1004             if os.path.exists(apkfile):
1005                 logging.error("Failed to get apk information, deleting " + apkfile)
1006                 os.remove(apkfile)
1007             else:
1008                 logging.error("Could not find {0} to remove it".format(apkfile))
1009         else:
1010             logging.error("Failed to get apk information, skipping " + apkfile)
1011         raise BuildException("Invalid APK")
1012     for line in p.output.splitlines():
1013         if line.startswith("package:"):
1014             try:
1015                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1016                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1017                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1018             except Exception as e:
1019                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1020         elif line.startswith("application:"):
1021             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1022             # Keep path to non-dpi icon in case we need it
1023             match = re.match(APK_ICON_PAT_NODPI, line)
1024             if match:
1025                 apk['icons_src']['-1'] = match.group(1)
1026         elif line.startswith("launchable-activity:"):
1027             # Only use launchable-activity as fallback to application
1028             if not apk['name']:
1029                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1030             if '-1' not in apk['icons_src']:
1031                 match = re.match(APK_ICON_PAT_NODPI, line)
1032                 if match:
1033                     apk['icons_src']['-1'] = match.group(1)
1034         elif line.startswith("application-icon-"):
1035             match = re.match(APK_ICON_PAT, line)
1036             if match:
1037                 density = match.group(1)
1038                 path = match.group(2)
1039                 apk['icons_src'][density] = path
1040         elif line.startswith("sdkVersion:"):
1041             m = re.match(APK_SDK_VERSION_PAT, line)
1042             if m is None:
1043                 logging.error(line.replace('sdkVersion:', '')
1044                               + ' is not a valid minSdkVersion!')
1045             else:
1046                 apk['minSdkVersion'] = m.group(1)
1047                 # if target not set, default to min
1048                 if 'targetSdkVersion' not in apk:
1049                     apk['targetSdkVersion'] = m.group(1)
1050         elif line.startswith("targetSdkVersion:"):
1051             m = re.match(APK_SDK_VERSION_PAT, line)
1052             if m is None:
1053                 logging.error(line.replace('targetSdkVersion:', '')
1054                               + ' is not a valid targetSdkVersion!')
1055             else:
1056                 apk['targetSdkVersion'] = m.group(1)
1057         elif line.startswith("maxSdkVersion:"):
1058             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1059         elif line.startswith("native-code:"):
1060             apk['nativecode'] = []
1061             for arch in line[13:].split(' '):
1062                 apk['nativecode'].append(arch[1:-1])
1063         elif line.startswith('uses-permission:'):
1064             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1065             if perm_match['maxSdkVersion']:
1066                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1067             permission = UsesPermission(
1068                 perm_match['name'],
1069                 perm_match['maxSdkVersion']
1070             )
1071
1072             apk['uses-permission'].append(permission)
1073         elif line.startswith('uses-permission-sdk-23:'):
1074             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1075             if perm_match['maxSdkVersion']:
1076                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1077             permission_sdk_23 = UsesPermissionSdk23(
1078                 perm_match['name'],
1079                 perm_match['maxSdkVersion']
1080             )
1081
1082             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1083
1084         elif line.startswith('uses-feature:'):
1085             feature = re.match(APK_FEATURE_PAT, line).group(1)
1086             # Filter out this, it's only added with the latest SDK tools and
1087             # causes problems for lots of apps.
1088             if feature != "android.hardware.screen.portrait" \
1089                     and feature != "android.hardware.screen.landscape":
1090                 if feature.startswith("android.feature."):
1091                     feature = feature[16:]
1092                 apk['features'].add(feature)
1093
1094
1095 def scan_apk_androguard(apk, apkfile):
1096     try:
1097         from androguard.core.bytecodes.apk import APK
1098         apkobject = APK(apkfile)
1099         if apkobject.is_valid_APK():
1100             arsc = apkobject.get_android_resources()
1101         else:
1102             if options.delete_unknown:
1103                 if os.path.exists(apkfile):
1104                     logging.error("Failed to get apk information, deleting " + apkfile)
1105                     os.remove(apkfile)
1106                 else:
1107                     logging.error("Could not find {0} to remove it".format(apkfile))
1108             else:
1109                 logging.error("Failed to get apk information, skipping " + apkfile)
1110             raise BuildException("Invaild APK")
1111     except ImportError:
1112         raise FDroidException("androguard library is not installed and aapt not present")
1113     except FileNotFoundError:
1114         logging.error("Could not open apk file for analysis")
1115         raise BuildException("Invalid APK")
1116
1117     apk['packageName'] = apkobject.get_package()
1118     apk['versionCode'] = int(apkobject.get_androidversion_code())
1119     apk['versionName'] = apkobject.get_androidversion_name()
1120     if apk['versionName'][0] == "@":
1121         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1122         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1123         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1124     apk['name'] = apkobject.get_app_name()
1125
1126     if apkobject.get_max_sdk_version() is not None:
1127         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1128     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1129     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1130
1131     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1132     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1133
1134     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1135
1136     for file in apkobject.get_files():
1137         d_re = density_re.match(file)
1138         if d_re:
1139             folder = d_re.group(1).split('-')
1140             if len(folder) > 1:
1141                 resolution = folder[1]
1142             else:
1143                 resolution = 'mdpi'
1144             density = screen_resolutions[resolution]
1145             apk['icons_src'][density] = d_re.group(0)
1146
1147     if apk['icons_src'].get('-1') is None:
1148         apk['icons_src']['-1'] = apk['icons_src']['160']
1149
1150     arch_re = re.compile("^lib/(.*)/.*$")
1151     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1152     if len(arch) >= 1:
1153         apk['nativecode'] = []
1154         apk['nativecode'].extend(sorted(list(arch)))
1155
1156     xml = apkobject.get_android_manifest_xml()
1157
1158     for item in xml.getElementsByTagName('uses-permission'):
1159         name = str(item.getAttribute("android:name"))
1160         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1161         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1162         permission = UsesPermission(
1163             name,
1164             maxSdkVersion
1165         )
1166         apk['uses-permission'].append(permission)
1167
1168     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1169         name = str(item.getAttribute("android:name"))
1170         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1171         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1172         permission_sdk_23 = UsesPermissionSdk23(
1173             name,
1174             maxSdkVersion
1175         )
1176         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1177
1178     for item in xml.getElementsByTagName('uses-feature'):
1179         feature = str(item.getAttribute("android:name"))
1180         if feature != "android.hardware.screen.portrait" \
1181                 and feature != "android.hardware.screen.landscape":
1182             if feature.startswith("android.feature."):
1183                 feature = feature[16:]
1184         apk['features'].append(feature)
1185
1186
1187 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1188                 allow_disabled_algorithms=False, archive_bad_sig=False):
1189     """Processes the apk with the given filename in the given repo directory.
1190
1191     This also extracts the icons.
1192
1193     :param apkcache: current apk cache information
1194     :param apkfilename: the filename of the apk to scan
1195     :param repodir: repo directory to scan
1196     :param knownapks: known apks info
1197     :param use_date_from_apk: use date from APK (instead of current date)
1198                               for newly added APKs
1199     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1200                                       disabled algorithms in the signature (e.g. MD5)
1201     :param archive_bad_sig: move APKs with a bad signature to the archive
1202     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1203      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1204     """
1205
1206     apk = {}
1207     apkfile = os.path.join(repodir, apkfilename)
1208
1209     cachechanged = False
1210     usecache = False
1211     if apkfilename in apkcache:
1212         apk = apkcache[apkfilename]
1213         if apk.get('hash') == sha256sum(apkfile):
1214             logging.debug("Reading " + apkfilename + " from cache")
1215             usecache = True
1216         else:
1217             logging.debug("Ignoring stale cache data for " + apkfilename)
1218
1219     if not usecache:
1220         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1221
1222         try:
1223             apk = scan_apk(apkfile)
1224         except BuildException:
1225             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1226                             .format(apkfilename=apkfilename))
1227             return True, None, False
1228
1229         # Check for debuggable apks...
1230         if common.isApkAndDebuggable(apkfile):
1231             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1232
1233         if options.rename_apks:
1234             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1235             std_short_name = os.path.join(repodir, n)
1236             if apkfile != std_short_name:
1237                 if os.path.exists(std_short_name):
1238                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1239                     if apkfile != std_long_name:
1240                         if os.path.exists(std_long_name):
1241                             dupdir = os.path.join('duplicates', repodir)
1242                             if not os.path.isdir(dupdir):
1243                                 os.makedirs(dupdir, exist_ok=True)
1244                             dupfile = os.path.join('duplicates', std_long_name)
1245                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1246                             os.rename(apkfile, dupfile)
1247                             return True, None, False
1248                         else:
1249                             os.rename(apkfile, std_long_name)
1250                     apkfile = std_long_name
1251                 else:
1252                     os.rename(apkfile, std_short_name)
1253                     apkfile = std_short_name
1254                 apkfilename = apkfile[len(repodir) + 1:]
1255
1256         apk['apkName'] = apkfilename
1257         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1258         if os.path.exists(os.path.join(repodir, srcfilename)):
1259             apk['srcname'] = srcfilename
1260
1261         # verify the jar signature is correct, allow deprecated
1262         # algorithms only if the APK is in the archive.
1263         skipapk = False
1264         if not common.verify_apk_signature(apkfile):
1265             if repodir == 'archive' or allow_disabled_algorithms:
1266                 if common.verify_old_apk_signature(apkfile):
1267                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1268                 else:
1269                     skipapk = True
1270             else:
1271                 skipapk = True
1272
1273         if skipapk:
1274             if archive_bad_sig:
1275                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1276                 move_apk_between_sections(repodir, 'archive', apk)
1277             else:
1278                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1279             return True, None, False
1280
1281         apkzip = zipfile.ZipFile(apkfile, 'r')
1282
1283         # if an APK has files newer than the system time, suggest updating
1284         # the system clock.  This is useful for offline systems, used for
1285         # signing, which do not have another source of clock sync info. It
1286         # has to be more than 24 hours newer because ZIP/APK files do not
1287         # store timezone info
1288         manifest = apkzip.getinfo('AndroidManifest.xml')
1289         if manifest.date_time[1] == 0:  # month can't be zero
1290             logging.debug('AndroidManifest.xml has no date')
1291         else:
1292             dt_obj = datetime(*manifest.date_time)
1293             checkdt = dt_obj - timedelta(1)
1294             if datetime.today() < checkdt:
1295                 logging.warning('System clock is older than manifest in: '
1296                                 + apkfilename
1297                                 + '\nSet clock to that time using:\n'
1298                                 + 'sudo date -s "' + str(dt_obj) + '"')
1299
1300         # extract icons from APK zip file
1301         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1302         try:
1303             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1304         finally:
1305             apkzip.close()  # ensure that APK zip file gets closed
1306
1307         # resize existing icons for densities missing in the APK
1308         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1309
1310         if use_date_from_apk and manifest.date_time[1] != 0:
1311             default_date_param = datetime(*manifest.date_time)
1312         else:
1313             default_date_param = None
1314
1315         # Record in known apks, getting the added date at the same time..
1316         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1317                                     default_date=default_date_param)
1318         if added:
1319             apk['added'] = added
1320
1321         apkcache[apkfilename] = apk
1322         cachechanged = True
1323
1324     return False, apk, cachechanged
1325
1326
1327 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1328     """Processes the apks in the given repo directory.
1329
1330     This also extracts the icons.
1331
1332     :param apkcache: current apk cache information
1333     :param repodir: repo directory to scan
1334     :param knownapks: known apks info
1335     :param use_date_from_apk: use date from APK (instead of current date)
1336                               for newly added APKs
1337     :returns: (apks, cachechanged) where apks is a list of apk information,
1338               and cachechanged is True if the apkcache got changed.
1339     """
1340
1341     cachechanged = False
1342
1343     for icon_dir in get_all_icon_dirs(repodir):
1344         if os.path.exists(icon_dir):
1345             if options.clean:
1346                 shutil.rmtree(icon_dir)
1347                 os.makedirs(icon_dir)
1348         else:
1349             os.makedirs(icon_dir)
1350
1351     apks = []
1352     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1353         apkfilename = apkfile[len(repodir) + 1:]
1354         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1355         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1356                                              use_date_from_apk, ada, True)
1357         if skip:
1358             continue
1359         apks.append(apk)
1360         cachechanged = cachechanged or cachethis
1361
1362     return apks, cachechanged
1363
1364
1365 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1366     """
1367     Extracts icons from the given APK zip in various densities,
1368     saves them into given repo directory
1369     and stores their names in the APK metadata dictionary.
1370
1371     :param icon_filename: A string representing the icon's file name
1372     :param apk: A populated dictionary containing APK metadata.
1373                 Needs to have 'icons_src' key
1374     :param apkzip: An opened zipfile.ZipFile of the APK file
1375     :param repo_dir: The directory of the APK's repository
1376     :return: A list of icon densities that are missing
1377     """
1378     empty_densities = []
1379     for density in screen_densities:
1380         if density not in apk['icons_src']:
1381             empty_densities.append(density)
1382             continue
1383         icon_src = apk['icons_src'][density]
1384         icon_dir = get_icon_dir(repo_dir, density)
1385         icon_dest = os.path.join(icon_dir, icon_filename)
1386
1387         # Extract the icon files per density
1388         if icon_src.endswith('.xml'):
1389             png = os.path.basename(icon_src)[:-4] + '.png'
1390             for f in apkzip.namelist():
1391                 if f.endswith(png):
1392                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1393                     if m and screen_resolutions[m.group(2)] == density:
1394                         icon_src = f
1395             if icon_src.endswith('.xml'):
1396                 empty_densities.append(density)
1397                 continue
1398         try:
1399             with open(icon_dest, 'wb') as f:
1400                 f.write(get_icon_bytes(apkzip, icon_src))
1401             apk['icons'][density] = icon_filename
1402         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1403             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1404             del apk['icons_src'][density]
1405             empty_densities.append(density)
1406
1407     if '-1' in apk['icons_src']:
1408         icon_src = apk['icons_src']['-1']
1409         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1410         with open(icon_path, 'wb') as f:
1411             f.write(get_icon_bytes(apkzip, icon_src))
1412         try:
1413             im = Image.open(icon_path)
1414             dpi = px_to_dpi(im.size[0])
1415             for density in screen_densities:
1416                 if density in apk['icons']:
1417                     break
1418                 if density == screen_densities[-1] or dpi >= int(density):
1419                     apk['icons'][density] = icon_filename
1420                     shutil.move(icon_path,
1421                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1422                     empty_densities.remove(density)
1423                     break
1424         except Exception as e:
1425             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1426
1427     if apk['icons']:
1428         apk['icon'] = icon_filename
1429
1430     return empty_densities
1431
1432
1433 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1434     """
1435     Resize existing icons for densities missing in the APK to ensure all densities are available
1436
1437     :param empty_densities: A list of icon densities that are missing
1438     :param icon_filename: A string representing the icon's file name
1439     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1440     :param repo_dir: The directory of the APK's repository
1441     """
1442     # First try resizing down to not lose quality
1443     last_density = None
1444     for density in screen_densities:
1445         if density not in empty_densities:
1446             last_density = density
1447             continue
1448         if last_density is None:
1449             continue
1450         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1451
1452         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1453         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1454         fp = None
1455         try:
1456             fp = open(last_icon_path, 'rb')
1457             im = Image.open(fp)
1458
1459             size = dpi_to_px(density)
1460
1461             im.thumbnail((size, size), Image.ANTIALIAS)
1462             im.save(icon_path, "PNG")
1463             empty_densities.remove(density)
1464         except Exception as e:
1465             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1466         finally:
1467             if fp:
1468                 fp.close()
1469
1470     # Then just copy from the highest resolution available
1471     last_density = None
1472     for density in reversed(screen_densities):
1473         if density not in empty_densities:
1474             last_density = density
1475             continue
1476
1477         if last_density is None:
1478             continue
1479
1480         shutil.copyfile(
1481             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1482             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1483         )
1484         empty_densities.remove(density)
1485
1486     for density in screen_densities:
1487         icon_dir = get_icon_dir(repo_dir, density)
1488         icon_dest = os.path.join(icon_dir, icon_filename)
1489         resize_icon(icon_dest, density)
1490
1491     # Copy from icons-mdpi to icons since mdpi is the baseline density
1492     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1493     if os.path.isfile(baseline):
1494         apk['icons']['0'] = icon_filename
1495         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1496
1497
1498 def apply_info_from_latest_apk(apps, apks):
1499     """
1500     Some information from the apks needs to be applied up to the application level.
1501     When doing this, we use the info from the most recent version's apk.
1502     We deal with figuring out when the app was added and last updated at the same time.
1503     """
1504     for appid, app in apps.items():
1505         bestver = UNSET_VERSION_CODE
1506         for apk in apks:
1507             if apk['packageName'] == appid:
1508                 if apk['versionCode'] > bestver:
1509                     bestver = apk['versionCode']
1510                     bestapk = apk
1511
1512                 if 'added' in apk:
1513                     if not app.added or apk['added'] < app.added:
1514                         app.added = apk['added']
1515                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1516                         app.lastUpdated = apk['added']
1517
1518         if not app.added:
1519             logging.debug("Don't know when " + appid + " was added")
1520         if not app.lastUpdated:
1521             logging.debug("Don't know when " + appid + " was last updated")
1522
1523         if bestver == UNSET_VERSION_CODE:
1524
1525             if app.Name is None:
1526                 app.Name = app.AutoName or appid
1527             app.icon = None
1528             logging.debug("Application " + appid + " has no packages")
1529         else:
1530             if app.Name is None:
1531                 app.Name = bestapk['name']
1532             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1533             if app.CurrentVersionCode is None:
1534                 app.CurrentVersionCode = str(bestver)
1535
1536
1537 def make_categories_txt(repodir, categories):
1538     '''Write a category list in the repo to allow quick access'''
1539     catdata = ''
1540     for cat in sorted(categories):
1541         catdata += cat + '\n'
1542     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1543         f.write(catdata)
1544
1545
1546 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1547
1548     def filter_apk_list_sorted(apk_list):
1549         res = []
1550         for apk in apk_list:
1551             if apk['packageName'] == appid:
1552                 res.append(apk)
1553
1554         # Sort the apk list by version code. First is highest/newest.
1555         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1556
1557     for appid, app in apps.items():
1558
1559         if app.ArchivePolicy:
1560             keepversions = int(app.ArchivePolicy[:-9])
1561         else:
1562             keepversions = defaultkeepversions
1563
1564         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1565                       .format(appid, len(apks), keepversions, len(archapks)))
1566
1567         current_app_apks = filter_apk_list_sorted(apks)
1568         if len(current_app_apks) > keepversions:
1569             # Move back the ones we don't want.
1570             for apk in current_app_apks[keepversions:]:
1571                 move_apk_between_sections(repodir, archivedir, apk)
1572                 archapks.append(apk)
1573                 apks.remove(apk)
1574
1575         current_app_archapks = filter_apk_list_sorted(archapks)
1576         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1577             kept = 0
1578             # Move forward the ones we want again, except DisableAlgorithm
1579             for apk in current_app_archapks:
1580                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1581                     move_apk_between_sections(archivedir, repodir, apk)
1582                     archapks.remove(apk)
1583                     apks.append(apk)
1584                     kept += 1
1585                 if kept == keepversions:
1586                     break
1587
1588
1589 def move_apk_between_sections(from_dir, to_dir, apk):
1590     """move an APK from repo to archive or vice versa"""
1591
1592     def _move_file(from_dir, to_dir, filename, ignore_missing):
1593         from_path = os.path.join(from_dir, filename)
1594         if ignore_missing and not os.path.exists(from_path):
1595             return
1596         to_path = os.path.join(to_dir, filename)
1597         if not os.path.exists(to_dir):
1598             os.mkdir(to_dir)
1599         shutil.move(from_path, to_path)
1600
1601     if from_dir == to_dir:
1602         return
1603
1604     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1605     _move_file(from_dir, to_dir, apk['apkName'], False)
1606     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1607     for density in all_screen_densities:
1608         from_icon_dir = get_icon_dir(from_dir, density)
1609         to_icon_dir = get_icon_dir(to_dir, density)
1610         if density not in apk['icons']:
1611             continue
1612         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1613     if 'srcname' in apk:
1614         _move_file(from_dir, to_dir, apk['srcname'], False)
1615
1616
1617 def add_apks_to_per_app_repos(repodir, apks):
1618     apks_per_app = dict()
1619     for apk in apks:
1620         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1621         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1622         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1623         apks_per_app[apk['packageName']] = apk
1624
1625         if not os.path.exists(apk['per_app_icons']):
1626             logging.info('Adding new repo for only ' + apk['packageName'])
1627             os.makedirs(apk['per_app_icons'])
1628
1629         apkpath = os.path.join(repodir, apk['apkName'])
1630         shutil.copy(apkpath, apk['per_app_repo'])
1631         apksigpath = apkpath + '.sig'
1632         if os.path.exists(apksigpath):
1633             shutil.copy(apksigpath, apk['per_app_repo'])
1634         apkascpath = apkpath + '.asc'
1635         if os.path.exists(apkascpath):
1636             shutil.copy(apkascpath, apk['per_app_repo'])
1637
1638
1639 def create_metadata_from_template(apk):
1640     '''create a new metadata file using internal or external template
1641
1642     Generate warnings for apk's with no metadata (or create skeleton
1643     metadata files, if requested on the command line).  Though the
1644     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1645     since those impose things on the metadata file made from the
1646     template: field sort order, empty field value, formatting, etc.
1647     '''
1648
1649     import yaml
1650     if os.path.exists('template.yml'):
1651         with open('template.yml') as f:
1652             metatxt = f.read()
1653         if 'name' in apk and apk['name'] != '':
1654             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1655                              r'\1 ' + apk['name'],
1656                              metatxt,
1657                              flags=re.IGNORECASE | re.MULTILINE)
1658         else:
1659             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1660             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1661                              r'\1 ' + apk['packageName'],
1662                              metatxt,
1663                              flags=re.IGNORECASE | re.MULTILINE)
1664         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1665             f.write(metatxt)
1666     else:
1667         app = dict()
1668         app['Categories'] = [os.path.basename(os.getcwd())]
1669         # include some blanks as part of the template
1670         app['AuthorName'] = ''
1671         app['Summary'] = ''
1672         app['WebSite'] = ''
1673         app['IssueTracker'] = ''
1674         app['SourceCode'] = ''
1675         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1676         if 'name' in apk and apk['name'] != '':
1677             app['Name'] = apk['name']
1678         else:
1679             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1680             app['Name'] = apk['packageName']
1681         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1682             yaml.dump(app, f, default_flow_style=False)
1683     logging.info("Generated skeleton metadata for " + apk['packageName'])
1684
1685
1686 config = None
1687 options = None
1688
1689
1690 def main():
1691
1692     global config, options
1693
1694     # Parse command line...
1695     parser = ArgumentParser()
1696     common.setup_global_opts(parser)
1697     parser.add_argument("--create-key", action="store_true", default=False,
1698                         help=_("Create a repo signing key in a keystore"))
1699     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1700                         help=_("Create skeleton metadata files that are missing"))
1701     parser.add_argument("--delete-unknown", action="store_true", default=False,
1702                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1703     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1704                         help=_("Report on build data status"))
1705     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1706                         help=_("Interactively ask about things that need updating."))
1707     parser.add_argument("-I", "--icons", action="store_true", default=False,
1708                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1709     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1710                         help=_("Specify editor to use in interactive mode. Default ") +
1711                         "is /etc/alternatives/editor")
1712     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1713                         help=_("Update the wiki"))
1714     parser.add_argument("--pretty", action="store_true", default=False,
1715                         help=_("Produce human-readable index.xml"))
1716     parser.add_argument("--clean", action="store_true", default=False,
1717                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1718     parser.add_argument("--nosign", action="store_true", default=False,
1719                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1720     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1721                         help=_("Use date from APK instead of current time for newly added APKs"))
1722     parser.add_argument("--rename-apks", action="store_true", default=False,
1723                         help=_("Rename APK files that do not match package.name_123.apk"))
1724     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1725                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1726     metadata.add_metadata_arguments(parser)
1727     options = parser.parse_args()
1728     metadata.warnings_action = options.W
1729
1730     config = common.read_config(options)
1731
1732     if not ('jarsigner' in config and 'keytool' in config):
1733         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1734
1735     repodirs = ['repo']
1736     if config['archive_older'] != 0:
1737         repodirs.append('archive')
1738         if not os.path.exists('archive'):
1739             os.mkdir('archive')
1740
1741     if options.icons:
1742         resize_all_icons(repodirs)
1743         sys.exit(0)
1744
1745     if options.rename_apks:
1746         options.clean = True
1747
1748     # check that icons exist now, rather than fail at the end of `fdroid update`
1749     for k in ['repo_icon', 'archive_icon']:
1750         if k in config:
1751             if not os.path.exists(config[k]):
1752                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1753                 sys.exit(1)
1754
1755     # if the user asks to create a keystore, do it now, reusing whatever it can
1756     if options.create_key:
1757         if os.path.exists(config['keystore']):
1758             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1759             logging.critical("\t'" + config['keystore'] + "'")
1760             sys.exit(1)
1761
1762         if 'repo_keyalias' not in config:
1763             config['repo_keyalias'] = socket.getfqdn()
1764             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1765         if 'keydname' not in config:
1766             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1767             common.write_to_config(config, 'keydname', config['keydname'])
1768         if 'keystore' not in config:
1769             config['keystore'] = common.default_config['keystore']
1770             common.write_to_config(config, 'keystore', config['keystore'])
1771
1772         password = common.genpassword()
1773         if 'keystorepass' not in config:
1774             config['keystorepass'] = password
1775             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1776         if 'keypass' not in config:
1777             config['keypass'] = password
1778             common.write_to_config(config, 'keypass', config['keypass'])
1779         common.genkeystore(config)
1780
1781     # Get all apps...
1782     apps = metadata.read_metadata()
1783
1784     # Generate a list of categories...
1785     categories = set()
1786     for app in apps.values():
1787         categories.update(app.Categories)
1788
1789     # Read known apks data (will be updated and written back when we've finished)
1790     knownapks = common.KnownApks()
1791
1792     # Get APK cache
1793     apkcache = get_cache()
1794
1795     # Delete builds for disabled apps
1796     delete_disabled_builds(apps, apkcache, repodirs)
1797
1798     # Scan all apks in the main repo
1799     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1800
1801     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1802                                            options.use_date_from_apk)
1803     cachechanged = cachechanged or fcachechanged
1804     apks += files
1805     for apk in apks:
1806         if apk['packageName'] not in apps:
1807             if options.create_metadata:
1808                 create_metadata_from_template(apk)
1809                 apps = metadata.read_metadata()
1810             else:
1811                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1812                 if options.delete_unknown:
1813                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1814                     rmf = os.path.join(repodirs[0], apk['apkName'])
1815                     if not os.path.exists(rmf):
1816                         logging.error("Could not find {0} to remove it".format(rmf))
1817                     else:
1818                         os.remove(rmf)
1819                 else:
1820                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1821
1822     copy_triple_t_store_metadata(apps)
1823     insert_obbs(repodirs[0], apps, apks)
1824     insert_localized_app_metadata(apps)
1825     translate_per_build_anti_features(apps, apks)
1826
1827     # Scan the archive repo for apks as well
1828     if len(repodirs) > 1:
1829         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1830         if cc:
1831             cachechanged = True
1832     else:
1833         archapks = []
1834
1835     # Apply information from latest apks to the application and update dates
1836     apply_info_from_latest_apk(apps, apks + archapks)
1837
1838     # Sort the app list by name, then the web site doesn't have to by default.
1839     # (we had to wait until we'd scanned the apks to do this, because mostly the
1840     # name comes from there!)
1841     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1842
1843     # APKs are placed into multiple repos based on the app package, providing
1844     # per-app subscription feeds for nightly builds and things like it
1845     if config['per_app_repos']:
1846         add_apks_to_per_app_repos(repodirs[0], apks)
1847         for appid, app in apps.items():
1848             repodir = os.path.join(appid, 'fdroid', 'repo')
1849             appdict = dict()
1850             appdict[appid] = app
1851             if os.path.isdir(repodir):
1852                 index.make(appdict, [appid], apks, repodir, False)
1853             else:
1854                 logging.info('Skipping index generation for ' + appid)
1855         return
1856
1857     if len(repodirs) > 1:
1858         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1859
1860     # Make the index for the main repo...
1861     index.make(apps, sortedids, apks, repodirs[0], False)
1862     make_categories_txt(repodirs[0], categories)
1863
1864     # If there's an archive repo,  make the index for it. We already scanned it
1865     # earlier on.
1866     if len(repodirs) > 1:
1867         index.make(apps, sortedids, archapks, repodirs[1], True)
1868
1869     git_remote = config.get('binary_transparency_remote')
1870     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1871         from . import btlog
1872         btlog.make_binary_transparency_log(repodirs)
1873
1874     if config['update_stats']:
1875         # Update known apks info...
1876         knownapks.writeifchanged()
1877
1878         # Generate latest apps data for widget
1879         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1880             data = ''
1881             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1882                 for line in f:
1883                     appid = line.rstrip()
1884                     data += appid + "\t"
1885                     app = apps[appid]
1886                     data += app.Name + "\t"
1887                     if app.icon is not None:
1888                         data += app.icon + "\t"
1889                     data += app.License + "\n"
1890             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1891                 f.write(data)
1892
1893     if cachechanged:
1894         write_cache(apkcache)
1895
1896     # Update the wiki...
1897     if options.wiki:
1898         update_wiki(apps, sortedids, apks + archapks)
1899
1900     logging.info(_("Finished"))
1901
1902
1903 if __name__ == "__main__":
1904     main()