fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in sorted(app.AntiFeatures):
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.LiberapayID,
 153             app.Bitcoin,
 154             app.Litecoin,
 155             app.License,
 156             requiresroot,
 157             app.AuthorName,
 158             app.AuthorEmail)
 159
 160         if app.Provides:
 161             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 162
 163         wikidata += app.Summary
 164         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 165
 166         wikidata += "=Description=\n"
 167         wikidata += metadata.description_wiki(app.Description) + "\n"
 168
 169         wikidata += "=Maintainer Notes=\n"
 170         if app.MaintainerNotes:
 171             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 172         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 173
 174         # Get a list of all packages for this application...
 175         apklist = []
 176         gotcurrentver = False
 177         cantupdate = False
 178         buildfails = False
 179         for apk in apks:
 180             if apk['packageName'] == appid:
 181                 if str(apk['versionCode']) == app.CurrentVersionCode:
 182                     gotcurrentver = True
 183                 apklist.append(apk)
 184         # Include ones we can't build, as a special case...
 185         for build in app.builds:
 186             if build.disable:
 187                 if build.versionCode == app.CurrentVersionCode:
 188                     cantupdate = True
 189                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 190                 apklist.append({'versionCode': int(build.versionCode),
 191                                 'versionName': build.versionName,
 192                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 193                                 })
 194             else:
 195                 builtit = False
 196                 for apk in apklist:
 197                     if apk['versionCode'] == int(build.versionCode):
 198                         builtit = True
 199                         break
 200                 if not builtit:
 201                     buildfails = True
 202                     apklist.append({'versionCode': int(build.versionCode),
 203                                     'versionName': build.versionName,
 204                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 205                                     })
 206         if app.CurrentVersionCode == '0':
 207             cantupdate = True
 208         # Sort with most recent first...
 209         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 210
 211         wikidata += "=Versions=\n"
 212         if len(apklist) == 0:
 213             wikidata += "We currently have no versions of this app available."
 214         elif not gotcurrentver:
 215             wikidata += "We don't have the current version of this app."
 216         else:
 217             wikidata += "We have the current version of this app."
 218         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 219         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 220         if len(app.NoSourceSince) > 0:
 221             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 222         if len(app.CurrentVersion) > 0:
 223             wikidata += "The current (recommended) version is " + app.CurrentVersion
 224             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 225         validapks = 0
 226         for apk in apklist:
 227             wikidata += "==" + apk['versionName'] + "==\n"
 228
 229             if 'buildproblem' in apk:
 230                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 231             else:
 232                 validapks += 1
 233                 wikidata += "This version is built and signed by "
 234                 if 'srcname' in apk:
 235                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 236                 else:
 237                     wikidata += "the original developer.\n\n"
 238             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 239
 240         wikidata += '\n[[Category:' + wikicat + ']]\n'
 241         if len(app.NoSourceSince) > 0:
 242             wikidata += '\n[[Category:Apps missing source code]]\n'
 243         if validapks == 0 and not app.Disabled:
 244             wikidata += '\n[[Category:Apps with no packages]]\n'
 245         if cantupdate and not app.Disabled:
 246             wikidata += "\n[[Category:Apps we cannot update]]\n"
 247         if buildfails and not app.Disabled:
 248             wikidata += "\n[[Category:Apps with failing builds]]\n"
 249         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 250             wikidata += '\n[[Category:Apps to Update]]\n'
 251         if app.Disabled:
 252             wikidata += '\n[[Category:Apps that are disabled]]\n'
 253         if app.UpdateCheckMode == 'None' and not app.Disabled:
 254             wikidata += '\n[[Category:Apps with no update check]]\n'
 255         for appcat in app.Categories:
 256             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 257
 258         # We can't have underscores in the page name, even if they're in
 259         # the package ID, because MediaWiki messes with them...
 260         pagename = appid.replace('_', ' ')
 261
 262         # Drop a trailing newline, because mediawiki is going to drop it anyway
 263         # and it we don't we'll think the page has changed when it hasn't...
 264         if wikidata.endswith('\n'):
 265             wikidata = wikidata[:-1]
 266
 267         generated_pages[pagename] = wikidata
 268
 269         # Make a redirect from the name to the ID too, unless there's
 270         # already an existing page with the name and it isn't a redirect.
 271         noclobber = False
 272         apppagename = app.Name.replace('_', ' ')
 273         apppagename = apppagename.replace('{', '')
 274         apppagename = apppagename.replace('}', ' ')
 275         apppagename = apppagename.replace(':', ' ')
 276         apppagename = apppagename.replace('[', ' ')
 277         apppagename = apppagename.replace(']', ' ')
 278         # Drop double spaces caused mostly by replacing ':' above
 279         apppagename = apppagename.replace('  ', ' ')
 280         for expagename in site.allpages(prefix=apppagename,
 281                                         filterredir='nonredirects',
 282                                         generator=False):
 283             if expagename == apppagename:
 284                 noclobber = True
 285         # Another reason not to make the redirect page is if the app name
 286         # is the same as it's ID, because that will overwrite the real page
 287         # with an redirect to itself! (Although it seems like an odd
 288         # scenario this happens a lot, e.g. where there is metadata but no
 289         # builds or binaries to extract a name from.
 290         if apppagename == pagename:
 291             noclobber = True
 292         if not noclobber:
 293             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 294
 295     for tcat, genp in [(wikicat, generated_pages),
 296                        (wikiredircat, generated_redirects)]:
 297         catpages = site.Pages['Category:' + tcat]
 298         existingpages = []
 299         for page in catpages:
 300             existingpages.append(page.name)
 301             if page.name in genp:
 302                 pagetxt = page.edit()
 303                 if pagetxt != genp[page.name]:
 304                     logging.debug("Updating modified page " + page.name)
 305                     page.save(genp[page.name], summary='Auto-updated')
 306                 else:
 307                     logging.debug("Page " + page.name + " is unchanged")
 308             else:
 309                 logging.warn("Deleting page " + page.name)
 310                 page.delete('No longer published')
 311         for pagename, text in genp.items():
 312             logging.debug("Checking " + pagename)
 313             if pagename not in existingpages:
 314                 logging.debug("Creating page " + pagename)
 315                 try:
 316                     newpage = site.Pages[pagename]
 317                     newpage.save(text, summary='Auto-created')
 318                 except Exception as e:
 319                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 320
 321     # Purge server cache to ensure counts are up to date
 322     site.pages['Repository Maintenance'].purge()
 323
 324
 325 def delete_disabled_builds(apps, apkcache, repodirs):
 326     """Delete disabled build outputs.
 327
 328     :param apps: list of all applications, as per metadata.read_metadata
 329     :param apkcache: current apk cache information
 330     :param repodirs: the repo directories to process
 331     """
 332     for appid, app in apps.items():
 333         for build in app['builds']:
 334             if not build.disable:
 335                 continue
 336             apkfilename = common.get_release_filename(app, build)
 337             iconfilename = "%s.%s.png" % (
 338                 appid,
 339                 build.versionCode)
 340             for repodir in repodirs:
 341                 files = [
 342                     os.path.join(repodir, apkfilename),
 343                     os.path.join(repodir, apkfilename + '.asc'),
 344                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 345                 ]
 346                 for density in all_screen_densities:
 347                     repo_dir = get_icon_dir(repodir, density)
 348                     files.append(os.path.join(repo_dir, iconfilename))
 349
 350                 for f in files:
 351                     if os.path.exists(f):
 352                         logging.info("Deleting disabled build output " + f)
 353                         os.remove(f)
 354             if apkfilename in apkcache:
 355                 del apkcache[apkfilename]
 356
 357
 358 def resize_icon(iconpath, density):
 359
 360     if not os.path.isfile(iconpath):
 361         return
 362
 363     fp = None
 364     try:
 365         fp = open(iconpath, 'rb')
 366         im = Image.open(fp)
 367         size = dpi_to_px(density)
 368
 369         if any(length > size for length in im.size):
 370             oldsize = im.size
 371             im.thumbnail((size, size), Image.ANTIALIAS)
 372             logging.debug("%s was too large at %s - new size is %s" % (
 373                 iconpath, oldsize, im.size))
 374             im.save(iconpath, "PNG")
 375
 376     except Exception as e:
 377         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 378
 379     finally:
 380         if fp:
 381             fp.close()
 382
 383
 384 def resize_all_icons(repodirs):
 385     """Resize all icons that exceed the max size
 386
 387     :param repodirs: the repo directories to process
 388     """
 389     for repodir in repodirs:
 390         for density in screen_densities:
 391             icon_dir = get_icon_dir(repodir, density)
 392             icon_glob = os.path.join(icon_dir, '*.png')
 393             for iconpath in glob.glob(icon_glob):
 394                 resize_icon(iconpath, density)
 395
 396
 397 def getsig(apkpath):
 398     """ Get the signing certificate of an apk. To get the same md5 has that
 399     Android gets, we encode the .RSA certificate in a specific format and pass
 400     it hex-encoded to the md5 digest algorithm.
 401
 402     :param apkpath: path to the apk
 403     :returns: A string containing the md5 of the signature of the apk or None
 404               if an error occurred.
 405     """
 406
 407     with zipfile.ZipFile(apkpath, 'r') as apk:
 408         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 409
 410         if len(certs) < 1:
 411             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 412             return None
 413         if len(certs) > 1:
 414             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 415             return None
 416
 417         cert = apk.read(certs[0])
 418
 419     cert_encoded = common.get_certificate(cert)
 420
 421     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 422
 423
 424 def get_cache_file():
 425     return os.path.join('tmp', 'apkcache')
 426
 427
 428 def get_cache():
 429     """Get the cached dict of the APK index
 430
 431     Gather information about all the apk files in the repo directory,
 432     using cached data if possible. Some of the index operations take a
 433     long time, like calculating the SHA-256 and verifying the APK
 434     signature.
 435
 436     The cache is invalidated if the metadata version is different, or
 437     the 'allow_disabled_algorithms' config/option is different.  In
 438     those cases, there is no easy way to know what has changed from
 439     the cache, so just rerun the whole thing.
 440
 441     :return: apkcache
 442
 443     """
 444     apkcachefile = get_cache_file()
 445     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 446     if not options.clean and os.path.exists(apkcachefile):
 447         with open(apkcachefile, 'rb') as cf:
 448             apkcache = pickle.load(cf, encoding='utf-8')
 449         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 450            or apkcache.get('allow_disabled_algorithms') != ada:
 451             apkcache = {}
 452     else:
 453         apkcache = {}
 454
 455     apkcache["METADATA_VERSION"] = METADATA_VERSION
 456     apkcache['allow_disabled_algorithms'] = ada
 457
 458     return apkcache
 459
 460
 461 def write_cache(apkcache):
 462     apkcachefile = get_cache_file()
 463     cache_path = os.path.dirname(apkcachefile)
 464     if not os.path.exists(cache_path):
 465         os.makedirs(cache_path)
 466     with open(apkcachefile, 'wb') as cf:
 467         pickle.dump(apkcache, cf)
 468
 469
 470 def get_icon_bytes(apkzip, iconsrc):
 471     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 472     try:
 473         return apkzip.read(iconsrc)
 474     except KeyError:
 475         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 476
 477
 478 def sha256sum(filename):
 479     '''Calculate the sha256 of the given file'''
 480     sha = hashlib.sha256()
 481     with open(filename, 'rb') as f:
 482         while True:
 483             t = f.read(16384)
 484             if len(t) == 0:
 485                 break
 486             sha.update(t)
 487     return sha.hexdigest()
 488
 489
 490 def has_known_vulnerability(filename):
 491     """checks for known vulnerabilities in the APK
 492
 493     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 494     version.  Google also enforces this:
 495     https://support.google.com/faqs/answer/6376725?hl=en
 496
 497     Checks whether there are more than one classes.dex or AndroidManifest.xml
 498     files, which is invalid and an essential part of the "Master Key" attack.
 499     http://www.saurik.com/id/17
 500
 501     Janus is similar to Master Key but is perhaps easier to scan for.
 502     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 503     """
 504
 505     found_vuln = False
 506
 507     # statically load this pattern
 508     if not hasattr(has_known_vulnerability, "pattern"):
 509         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 510
 511     with open(filename.encode(), 'rb') as fp:
 512         first4 = fp.read(4)
 513     if first4 != b'\x50\x4b\x03\x04':
 514         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 515                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 516                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 517
 518     files_in_apk = set()
 519     with zipfile.ZipFile(filename) as zf:
 520         for name in zf.namelist():
 521             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 522                 lib = zf.open(name)
 523                 while True:
 524                     chunk = lib.read(4096)
 525                     if chunk == b'':
 526                         break
 527                     m = has_known_vulnerability.pattern.search(chunk)
 528                     if m:
 529                         version = m.group(1).decode('ascii')
 530                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 531                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 532                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 533                             logging.debug(_('"{path}" contains recent {name} ({version})')
 534                                           .format(path=filename, name=name, version=version))
 535                         else:
 536                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 537                                             .format(path=filename, name=name, version=version))
 538                             found_vuln = True
 539                         break
 540             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 541                 if name in files_in_apk:
 542                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 543                                     .format(apkfilename=filename, name=name))
 544                     found_vuln = True
 545                 files_in_apk.add(name)
 546     return found_vuln
 547
 548
 549 def insert_obbs(repodir, apps, apks):
 550     """Scans the .obb files in a given repo directory and adds them to the
 551     relevant APK instances.  OBB files have versionCodes like APK
 552     files, and they are loosely associated.  If there is an OBB file
 553     present, then any APK with the same or higher versionCode will use
 554     that OBB file.  There are two OBB types: main and patch, each APK
 555     can only have only have one of each.
 556
 557     https://developer.android.com/google/play/expansion-files.html
 558
 559     :param repodir: repo directory to scan
 560     :param apps: list of current, valid apps
 561     :param apks: current information on all APKs
 562
 563     """
 564
 565     def obbWarnDelete(f, msg):
 566         logging.warning(msg + ' ' + f)
 567         if options.delete_unknown:
 568             logging.error(_("Deleting unknown file: {path}").format(path=f))
 569             os.remove(f)
 570
 571     obbs = []
 572     java_Integer_MIN_VALUE = -pow(2, 31)
 573     currentPackageNames = apps.keys()
 574     for f in glob.glob(os.path.join(repodir, '*.obb')):
 575         obbfile = os.path.basename(f)
 576         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 577         chunks = obbfile.split('.')
 578         if chunks[0] != 'main' and chunks[0] != 'patch':
 579             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 580             continue
 581         if not re.match(r'^-?[0-9]+$', chunks[1]):
 582             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 583                           .format(name=chunks[0]))
 584             continue
 585         versionCode = int(chunks[1])
 586         packagename = ".".join(chunks[2:-1])
 587
 588         highestVersionCode = java_Integer_MIN_VALUE
 589         if packagename not in currentPackageNames:
 590             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 591             continue
 592         for apk in apks:
 593             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 594                 highestVersionCode = apk['versionCode']
 595         if versionCode > highestVersionCode:
 596             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 597                           .format(integer=str(versionCode)))
 598             continue
 599         obbsha256 = sha256sum(f)
 600         obbs.append((packagename, versionCode, obbfile, obbsha256))
 601
 602     for apk in apks:
 603         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 604             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 605                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 606                     apk['obbMainFile'] = obbfile
 607                     apk['obbMainFileSha256'] = obbsha256
 608                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 609                     apk['obbPatchFile'] = obbfile
 610                     apk['obbPatchFileSha256'] = obbsha256
 611             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 612                 break
 613
 614
 615 def translate_per_build_anti_features(apps, apks):
 616     """Grab the anti-features list from the build metadata
 617
 618     For most Anti-Features, they are really most applicable per-APK,
 619     not for an app.  An app can fix a vulnerability, add/remove
 620     tracking, etc.  This reads the 'antifeatures' list from the Build
 621     entries in the fdroiddata metadata file, then transforms it into
 622     the 'antiFeatures' list of unique items for the index.
 623
 624     The field key is all lower case in the metadata file to match the
 625     rest of the Build fields.  It is 'antiFeatures' camel case in the
 626     implementation, index, and fdroidclient since it is translated
 627     from the build 'antifeatures' field, not directly included.
 628
 629     """
 630
 631     antiFeatures = dict()
 632     for packageName, app in apps.items():
 633         d = dict()
 634         for build in app['builds']:
 635             afl = build.get('antifeatures')
 636             if afl:
 637                 d[int(build.versionCode)] = afl
 638         if len(d) > 0:
 639             antiFeatures[packageName] = d
 640
 641     for apk in apks:
 642         d = antiFeatures.get(apk['packageName'])
 643         if d:
 644             afl = d.get(apk['versionCode'])
 645             if afl:
 646                 apk['antiFeatures'].update(afl)
 647
 648
 649 def _get_localized_dict(app, locale):
 650     '''get the dict to add localized store metadata to'''
 651     if 'localized' not in app:
 652         app['localized'] = collections.OrderedDict()
 653     if locale not in app['localized']:
 654         app['localized'][locale] = collections.OrderedDict()
 655     return app['localized'][locale]
 656
 657
 658 def _set_localized_text_entry(app, locale, key, f):
 659     limit = config['char_limits'][key]
 660     localized = _get_localized_dict(app, locale)
 661     with open(f) as fp:
 662         text = fp.read()[:limit]
 663         if len(text) > 0:
 664             localized[key] = text
 665
 666
 667 def _set_author_entry(app, key, f):
 668     limit = config['char_limits']['author']
 669     with open(f) as fp:
 670         text = fp.read()[:limit]
 671         if len(text) > 0:
 672             app[key] = text
 673
 674
 675 def copy_triple_t_store_metadata(apps):
 676     """Include store metadata from the app's source repo
 677
 678     The Triple-T Gradle Play Publisher is a plugin that has a standard
 679     file layout for all of the metadata and graphics that the Google
 680     Play Store accepts.  Since F-Droid has the git repo, it can just
 681     pluck those files directly.  This method reads any text files into
 682     the app dict, then copies any graphics into the fdroid repo
 683     directory structure.
 684
 685     This needs to be run before insert_localized_app_metadata() so that
 686     the graphics files that are copied into the fdroid repo get
 687     properly indexed.
 688
 689     https://github.com/Triple-T/gradle-play-publisher#upload-images
 690     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 691
 692     """
 693
 694     if not os.path.isdir('build'):
 695         return  # nothing to do
 696
 697     for packageName, app in apps.items():
 698         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 699             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 700             for root, dirs, files in os.walk(d):
 701                 segments = root.split('/')
 702                 locale = segments[-2]
 703                 for f in files:
 704                     if f == 'fulldescription':
 705                         _set_localized_text_entry(app, locale, 'description',
 706                                                   os.path.join(root, f))
 707                         continue
 708                     elif f == 'shortdescription':
 709                         _set_localized_text_entry(app, locale, 'summary',
 710                                                   os.path.join(root, f))
 711                         continue
 712                     elif f == 'title':
 713                         _set_localized_text_entry(app, locale, 'name',
 714                                                   os.path.join(root, f))
 715                         continue
 716                     elif f == 'video':
 717                         _set_localized_text_entry(app, locale, 'video',
 718                                                   os.path.join(root, f))
 719                         continue
 720                     elif f == 'whatsnew':
 721                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 722                                                   os.path.join(root, f))
 723                         continue
 724                     elif f == 'contactEmail':
 725                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 726                         continue
 727                     elif f == 'contactPhone':
 728                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 729                         continue
 730                     elif f == 'contactWebsite':
 731                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 732                         continue
 733
 734                     base, extension = common.get_extension(f)
 735                     dirname = os.path.basename(root)
 736                     if extension in ALLOWED_EXTENSIONS \
 737                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 738                         if segments[-2] == 'listing':
 739                             locale = segments[-3]
 740                         else:
 741                             locale = segments[-2]
 742                         destdir = os.path.join('repo', packageName, locale, dirname)
 743                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 744                         sourcefile = os.path.join(root, f)
 745                         destfile = os.path.join(destdir, os.path.basename(f))
 746                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 747                         shutil.copy(sourcefile, destfile)
 748
 749
 750 def insert_localized_app_metadata(apps):
 751     """scans standard locations for graphics and localized text
 752
 753     Scans for localized description files, store graphics, and
 754     screenshot PNG files in statically defined screenshots directory
 755     and adds them to the app metadata.  The screenshots and graphic
 756     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 757     and must be in the following layout:
 758     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 759
 760     repo/packageName/locale/featureGraphic.png
 761     repo/packageName/locale/phoneScreenshots/1.png
 762     repo/packageName/locale/phoneScreenshots/2.png
 763
 764     The changelog files must be text files named with the versionCode
 765     ending with ".txt" and must be in the following layout:
 766     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 767
 768     repo/packageName/locale/changelogs/12345.txt
 769
 770     This will scan the each app's source repo then the metadata/ dir
 771     for these standard locations of changelog files.  If it finds
 772     them, they will be added to the dict of all packages, with the
 773     versions in the metadata/ folder taking precendence over the what
 774     is in the app's source repo.
 775
 776     Where "packageName" is the app's packageName and "locale" is the locale
 777     of the graphics, e.g. what language they are in, using the IETF RFC5646
 778     format (en-US, fr-CA, es-MX, etc).
 779
 780     This will also scan the app's git for a fastlane folder, and the
 781     metadata/ folder and the apps' source repos for standard locations
 782     of graphic and screenshot files.  If it finds them, it will copy
 783     them into the repo.  The fastlane files follow this pattern:
 784     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 785
 786     """
 787
 788     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 789     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 790     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 791     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 792
 793     for srcd in sorted(sourcedirs):
 794         if not os.path.isdir(srcd):
 795             continue
 796         for root, dirs, files in os.walk(srcd):
 797             segments = root.split('/')
 798             packageName = segments[1]
 799             if packageName not in apps:
 800                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 801                 continue
 802             locale = segments[-1]
 803             destdir = os.path.join('repo', packageName, locale)
 804
 805             # flavours specified in build receipt
 806             build_flavours = ""
 807             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 808                     and 'gradle' in apps[packageName].builds[-1]:
 809                 build_flavours = apps[packageName].builds[-1].gradle
 810
 811             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 812                 logging.debug("ignoring due to wrong flavour")
 813                 continue
 814
 815             for f in files:
 816                 if f in ('description.txt', 'full_description.txt'):
 817                     _set_localized_text_entry(apps[packageName], locale, 'description',
 818                                               os.path.join(root, f))
 819                     continue
 820                 elif f in ('summary.txt', 'short_description.txt'):
 821                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 822                                               os.path.join(root, f))
 823                     continue
 824                 elif f in ('name.txt', 'title.txt'):
 825                     _set_localized_text_entry(apps[packageName], locale, 'name',
 826                                               os.path.join(root, f))
 827                     continue
 828                 elif f == 'video.txt':
 829                     _set_localized_text_entry(apps[packageName], locale, 'video',
 830                                               os.path.join(root, f))
 831                     continue
 832                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 833                     locale = segments[-2]
 834                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 835                                               os.path.join(root, f))
 836                     continue
 837
 838                 base, extension = common.get_extension(f)
 839                 if locale == 'images':
 840                     locale = segments[-2]
 841                     destdir = os.path.join('repo', packageName, locale)
 842                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 843                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 844                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 845                     shutil.copy(os.path.join(root, f), destdir)
 846             for d in dirs:
 847                 if d in SCREENSHOT_DIRS:
 848                     if locale == 'images':
 849                         locale = segments[-2]
 850                         destdir = os.path.join('repo', packageName, locale)
 851                     for f in glob.glob(os.path.join(root, d, '*.*')):
 852                         _ignored, extension = common.get_extension(f)
 853                         if extension in ALLOWED_EXTENSIONS:
 854                             screenshotdestdir = os.path.join(destdir, d)
 855                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 856                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 857                             shutil.copy(f, screenshotdestdir)
 858
 859     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 860     for d in repofiles:
 861         if not os.path.isdir(d):
 862             continue
 863         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 864             if not os.path.isfile(f):
 865                 continue
 866             segments = f.split('/')
 867             packageName = segments[1]
 868             locale = segments[2]
 869             screenshotdir = segments[3]
 870             filename = os.path.basename(f)
 871             base, extension = common.get_extension(filename)
 872
 873             if packageName not in apps:
 874                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 875                                 .format(path=filename, name=packageName))
 876                 continue
 877             graphics = _get_localized_dict(apps[packageName], locale)
 878
 879             if extension not in ALLOWED_EXTENSIONS:
 880                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 881             elif base in GRAPHIC_NAMES:
 882                 # there can only be zero or one of these per locale
 883                 graphics[base] = filename
 884             elif screenshotdir in SCREENSHOT_DIRS:
 885                 # there can any number of these per locale
 886                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 887                 if screenshotdir not in graphics:
 888                     graphics[screenshotdir] = []
 889                 graphics[screenshotdir].append(filename)
 890             else:
 891                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 892
 893
 894 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 895     """Scan a repo for all files with an extension except APK/OBB
 896
 897     :param apkcache: current cached info about all repo files
 898     :param repodir: repo directory to scan
 899     :param knownapks: list of all known files, as per metadata.read_metadata
 900     :param use_date_from_file: use date from file (instead of current date)
 901                                for newly added files
 902     """
 903
 904     cachechanged = False
 905     repo_files = []
 906     repodir = repodir.encode('utf-8')
 907     for name in os.listdir(repodir):
 908         file_extension = common.get_file_extension(name)
 909         if file_extension == 'apk' or file_extension == 'obb':
 910             continue
 911         filename = os.path.join(repodir, name)
 912         name_utf8 = name.decode('utf-8')
 913         if filename.endswith(b'_src.tar.gz'):
 914             logging.debug(_('skipping source tarball: {path}')
 915                           .format(path=filename.decode('utf-8')))
 916             continue
 917         if not common.is_repo_file(filename):
 918             continue
 919         stat = os.stat(filename)
 920         if stat.st_size == 0:
 921             raise FDroidException(_('{path} is zero size!')
 922                                   .format(path=filename))
 923
 924         shasum = sha256sum(filename)
 925         usecache = False
 926         if name in apkcache:
 927             repo_file = apkcache[name]
 928             # added time is cached as tuple but used here as datetime instance
 929             if 'added' in repo_file:
 930                 a = repo_file['added']
 931                 if isinstance(a, datetime):
 932                     repo_file['added'] = a
 933                 else:
 934                     repo_file['added'] = datetime(*a[:6])
 935             if repo_file.get('hash') == shasum:
 936                 logging.debug(_("Reading {apkfilename} from cache")
 937                               .format(apkfilename=name_utf8))
 938                 usecache = True
 939             else:
 940                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 941                               .format(apkfilename=name_utf8))
 942
 943         if not usecache:
 944             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 945             repo_file = collections.OrderedDict()
 946             repo_file['name'] = os.path.splitext(name_utf8)[0]
 947             # TODO rename apkname globally to something more generic
 948             repo_file['apkName'] = name_utf8
 949             repo_file['hash'] = shasum
 950             repo_file['hashType'] = 'sha256'
 951             repo_file['versionCode'] = 0
 952             repo_file['versionName'] = shasum
 953             # the static ID is the SHA256 unless it is set in the metadata
 954             repo_file['packageName'] = shasum
 955
 956             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 957             if m:
 958                 repo_file['packageName'] = m.group(1)
 959                 repo_file['versionCode'] = int(m.group(2))
 960             srcfilename = name + b'_src.tar.gz'
 961             if os.path.exists(os.path.join(repodir, srcfilename)):
 962                 repo_file['srcname'] = srcfilename.decode('utf-8')
 963             repo_file['size'] = stat.st_size
 964
 965             apkcache[name] = repo_file
 966             cachechanged = True
 967
 968         if use_date_from_file:
 969             timestamp = stat.st_ctime
 970             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 971         else:
 972             default_date_param = None
 973
 974         # Record in knownapks, getting the added date at the same time..
 975         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 976                                     default_date=default_date_param)
 977         if added:
 978             repo_file['added'] = added
 979
 980         repo_files.append(repo_file)
 981
 982     return repo_files, cachechanged
 983
 984
 985 def scan_apk(apk_file):
 986     """
 987     Scans an APK file and returns dictionary with metadata of the APK.
 988
 989     Attention: This does *not* verify that the APK signature is correct.
 990
 991     :param apk_file: The (ideally absolute) path to the APK file
 992     :raises BuildException
 993     :return A dict containing APK metadata
 994     """
 995     apk = {
 996         'hash': sha256sum(apk_file),
 997         'hashType': 'sha256',
 998         'uses-permission': [],
 999         'uses-permission-sdk-23': [],
1000         'features': [],
1001         'icons_src': {},
1002         'icons': {},
1003         'antiFeatures': set(),
1004     }
1005
1006     if SdkToolsPopen(['aapt', 'version'], output=False):
1007         scan_apk_aapt(apk, apk_file)
1008     else:
1009         scan_apk_androguard(apk, apk_file)
1010
1011     # Get the signature, or rather the signing key fingerprints
1012     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1013     apk['sig'] = getsig(apk_file)
1014     if not apk['sig']:
1015         raise BuildException("Failed to get apk signature")
1016     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1017                                                                apk_file))
1018     if not apk.get('signer'):
1019         raise BuildException("Failed to get apk signing key fingerprint")
1020
1021     # Get size of the APK
1022     apk['size'] = os.path.getsize(apk_file)
1023
1024     if 'minSdkVersion' not in apk:
1025         logging.warning("No SDK version information found in {0}".format(apk_file))
1026         apk['minSdkVersion'] = 1
1027
1028     # Check for known vulnerabilities
1029     if has_known_vulnerability(apk_file):
1030         apk['antiFeatures'].add('KnownVuln')
1031
1032     return apk
1033
1034
1035 def scan_apk_aapt(apk, apkfile):
1036     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1037     if p.returncode != 0:
1038         if options.delete_unknown:
1039             if os.path.exists(apkfile):
1040                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1041                 os.remove(apkfile)
1042             else:
1043                 logging.error("Could not find {0} to remove it".format(apkfile))
1044         else:
1045             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1046         raise BuildException(_("Invalid APK"))
1047     for line in p.output.splitlines():
1048         if line.startswith("package:"):
1049             try:
1050                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1051                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1052                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1053             except Exception as e:
1054                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1055         elif line.startswith("application:"):
1056             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1057             # Keep path to non-dpi icon in case we need it
1058             match = re.match(APK_ICON_PAT_NODPI, line)
1059             if match:
1060                 apk['icons_src']['-1'] = match.group(1)
1061         elif line.startswith("launchable-activity:"):
1062             # Only use launchable-activity as fallback to application
1063             if not apk['name']:
1064                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1065             if '-1' not in apk['icons_src']:
1066                 match = re.match(APK_ICON_PAT_NODPI, line)
1067                 if match:
1068                     apk['icons_src']['-1'] = match.group(1)
1069         elif line.startswith("application-icon-"):
1070             match = re.match(APK_ICON_PAT, line)
1071             if match:
1072                 density = match.group(1)
1073                 path = match.group(2)
1074                 apk['icons_src'][density] = path
1075         elif line.startswith("sdkVersion:"):
1076             m = re.match(APK_SDK_VERSION_PAT, line)
1077             if m is None:
1078                 logging.error(line.replace('sdkVersion:', '')
1079                               + ' is not a valid minSdkVersion!')
1080             else:
1081                 apk['minSdkVersion'] = m.group(1)
1082                 # if target not set, default to min
1083                 if 'targetSdkVersion' not in apk:
1084                     apk['targetSdkVersion'] = m.group(1)
1085         elif line.startswith("targetSdkVersion:"):
1086             m = re.match(APK_SDK_VERSION_PAT, line)
1087             if m is None:
1088                 logging.error(line.replace('targetSdkVersion:', '')
1089                               + ' is not a valid targetSdkVersion!')
1090             else:
1091                 apk['targetSdkVersion'] = m.group(1)
1092         elif line.startswith("maxSdkVersion:"):
1093             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1094         elif line.startswith("native-code:"):
1095             apk['nativecode'] = []
1096             for arch in line[13:].split(' '):
1097                 apk['nativecode'].append(arch[1:-1])
1098         elif line.startswith('uses-permission:'):
1099             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1100             if perm_match['maxSdkVersion']:
1101                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1102             permission = UsesPermission(
1103                 perm_match['name'],
1104                 perm_match['maxSdkVersion']
1105             )
1106
1107             apk['uses-permission'].append(permission)
1108         elif line.startswith('uses-permission-sdk-23:'):
1109             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1110             if perm_match['maxSdkVersion']:
1111                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1112             permission_sdk_23 = UsesPermissionSdk23(
1113                 perm_match['name'],
1114                 perm_match['maxSdkVersion']
1115             )
1116
1117             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1118
1119         elif line.startswith('uses-feature:'):
1120             feature = re.match(APK_FEATURE_PAT, line).group(1)
1121             # Filter out this, it's only added with the latest SDK tools and
1122             # causes problems for lots of apps.
1123             if feature != "android.hardware.screen.portrait" \
1124                     and feature != "android.hardware.screen.landscape":
1125                 if feature.startswith("android.feature."):
1126                     feature = feature[16:]
1127                 apk['features'].add(feature)
1128
1129
1130 def scan_apk_androguard(apk, apkfile):
1131     try:
1132         from androguard.core.bytecodes.apk import APK
1133         apkobject = APK(apkfile)
1134         if apkobject.is_valid_APK():
1135             arsc = apkobject.get_android_resources()
1136         else:
1137             if options.delete_unknown:
1138                 if os.path.exists(apkfile):
1139                     logging.error(_("Failed to get apk information, deleting {path}")
1140                                   .format(path=apkfile))
1141                     os.remove(apkfile)
1142                 else:
1143                     logging.error(_("Could not find {path} to remove it")
1144                                   .format(path=apkfile))
1145             else:
1146                 logging.error(_("Failed to get apk information, skipping {path}")
1147                               .format(path=apkfile))
1148             raise BuildException(_("Invalid APK"))
1149     except ImportError:
1150         raise FDroidException("androguard library is not installed and aapt not present")
1151     except FileNotFoundError:
1152         logging.error(_("Could not open apk file for analysis"))
1153         raise BuildException(_("Invalid APK"))
1154
1155     apk['packageName'] = apkobject.get_package()
1156     apk['versionCode'] = int(apkobject.get_androidversion_code())
1157     apk['versionName'] = apkobject.get_androidversion_name()
1158     if apk['versionName'][0] == "@":
1159         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1160         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1161         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1162     apk['name'] = apkobject.get_app_name()
1163
1164     if apkobject.get_max_sdk_version() is not None:
1165         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1166     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1167     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1168
1169     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1170     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1171
1172     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1173
1174     for file in apkobject.get_files():
1175         d_re = density_re.match(file)
1176         if d_re:
1177             folder = d_re.group(1).split('-')
1178             if len(folder) > 1:
1179                 resolution = folder[1]
1180             else:
1181                 resolution = 'mdpi'
1182             density = screen_resolutions[resolution]
1183             apk['icons_src'][density] = d_re.group(0)
1184
1185     if apk['icons_src'].get('-1') is None:
1186         apk['icons_src']['-1'] = apk['icons_src']['160']
1187
1188     arch_re = re.compile("^lib/(.*)/.*$")
1189     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1190     if len(arch) >= 1:
1191         apk['nativecode'] = []
1192         apk['nativecode'].extend(sorted(list(arch)))
1193
1194     xml = apkobject.get_android_manifest_xml()
1195
1196     for item in xml.getElementsByTagName('uses-permission'):
1197         name = str(item.getAttribute("android:name"))
1198         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1199         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1200         permission = UsesPermission(
1201             name,
1202             maxSdkVersion
1203         )
1204         apk['uses-permission'].append(permission)
1205
1206     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1207         name = str(item.getAttribute("android:name"))
1208         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1209         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1210         permission_sdk_23 = UsesPermissionSdk23(
1211             name,
1212             maxSdkVersion
1213         )
1214         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1215
1216     for item in xml.getElementsByTagName('uses-feature'):
1217         feature = str(item.getAttribute("android:name"))
1218         if feature != "android.hardware.screen.portrait" \
1219                 and feature != "android.hardware.screen.landscape":
1220             if feature.startswith("android.feature."):
1221                 feature = feature[16:]
1222         apk['features'].append(feature)
1223
1224
1225 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1226                 allow_disabled_algorithms=False, archive_bad_sig=False):
1227     """Processes the apk with the given filename in the given repo directory.
1228
1229     This also extracts the icons.
1230
1231     :param apkcache: current apk cache information
1232     :param apkfilename: the filename of the apk to scan
1233     :param repodir: repo directory to scan
1234     :param knownapks: known apks info
1235     :param use_date_from_apk: use date from APK (instead of current date)
1236                               for newly added APKs
1237     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1238                                       disabled algorithms in the signature (e.g. MD5)
1239     :param archive_bad_sig: move APKs with a bad signature to the archive
1240     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1241      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1242     """
1243
1244     apk = {}
1245     apkfile = os.path.join(repodir, apkfilename)
1246
1247     cachechanged = False
1248     usecache = False
1249     if apkfilename in apkcache:
1250         apk = apkcache[apkfilename]
1251         if apk.get('hash') == sha256sum(apkfile):
1252             logging.debug(_("Reading {apkfilename} from cache")
1253                           .format(apkfilename=apkfilename))
1254             usecache = True
1255         else:
1256             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1257                           .format(apkfilename=apkfilename))
1258
1259     if not usecache:
1260         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1261
1262         try:
1263             apk = scan_apk(apkfile)
1264         except BuildException:
1265             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1266                             .format(apkfilename=apkfilename))
1267             return True, None, False
1268
1269         # Check for debuggable apks...
1270         if common.isApkAndDebuggable(apkfile):
1271             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1272
1273         if options.rename_apks:
1274             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1275             std_short_name = os.path.join(repodir, n)
1276             if apkfile != std_short_name:
1277                 if os.path.exists(std_short_name):
1278                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1279                     if apkfile != std_long_name:
1280                         if os.path.exists(std_long_name):
1281                             dupdir = os.path.join('duplicates', repodir)
1282                             if not os.path.isdir(dupdir):
1283                                 os.makedirs(dupdir, exist_ok=True)
1284                             dupfile = os.path.join('duplicates', std_long_name)
1285                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1286                             os.rename(apkfile, dupfile)
1287                             return True, None, False
1288                         else:
1289                             os.rename(apkfile, std_long_name)
1290                     apkfile = std_long_name
1291                 else:
1292                     os.rename(apkfile, std_short_name)
1293                     apkfile = std_short_name
1294                 apkfilename = apkfile[len(repodir) + 1:]
1295
1296         apk['apkName'] = apkfilename
1297         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1298         if os.path.exists(os.path.join(repodir, srcfilename)):
1299             apk['srcname'] = srcfilename
1300
1301         # verify the jar signature is correct, allow deprecated
1302         # algorithms only if the APK is in the archive.
1303         skipapk = False
1304         if not common.verify_apk_signature(apkfile):
1305             if repodir == 'archive' or allow_disabled_algorithms:
1306                 if common.verify_old_apk_signature(apkfile):
1307                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1308                 else:
1309                     skipapk = True
1310             else:
1311                 skipapk = True
1312
1313         if skipapk:
1314             if archive_bad_sig:
1315                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1316                                 .format(apkfilename=apkfilename))
1317                 move_apk_between_sections(repodir, 'archive', apk)
1318             else:
1319                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1320                                 .format(apkfilename=apkfilename))
1321             return True, None, False
1322
1323         apkzip = zipfile.ZipFile(apkfile, 'r')
1324
1325         manifest = apkzip.getinfo('AndroidManifest.xml')
1326         if manifest.date_time[1] == 0:  # month can't be zero
1327             logging.debug(_('AndroidManifest.xml has no date'))
1328         else:
1329             common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1330
1331         # extract icons from APK zip file
1332         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1333         try:
1334             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1335         finally:
1336             apkzip.close()  # ensure that APK zip file gets closed
1337
1338         # resize existing icons for densities missing in the APK
1339         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1340
1341         if use_date_from_apk and manifest.date_time[1] != 0:
1342             default_date_param = datetime(*manifest.date_time)
1343         else:
1344             default_date_param = None
1345
1346         # Record in known apks, getting the added date at the same time..
1347         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1348                                     default_date=default_date_param)
1349         if added:
1350             apk['added'] = added
1351
1352         apkcache[apkfilename] = apk
1353         cachechanged = True
1354
1355     return False, apk, cachechanged
1356
1357
1358 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1359     """Processes the apks in the given repo directory.
1360
1361     This also extracts the icons.
1362
1363     :param apkcache: current apk cache information
1364     :param repodir: repo directory to scan
1365     :param knownapks: known apks info
1366     :param use_date_from_apk: use date from APK (instead of current date)
1367                               for newly added APKs
1368     :returns: (apks, cachechanged) where apks is a list of apk information,
1369               and cachechanged is True if the apkcache got changed.
1370     """
1371
1372     cachechanged = False
1373
1374     for icon_dir in get_all_icon_dirs(repodir):
1375         if os.path.exists(icon_dir):
1376             if options.clean:
1377                 shutil.rmtree(icon_dir)
1378                 os.makedirs(icon_dir)
1379         else:
1380             os.makedirs(icon_dir)
1381
1382     apks = []
1383     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1384         apkfilename = apkfile[len(repodir) + 1:]
1385         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1386         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1387                                              use_date_from_apk, ada, True)
1388         if skip:
1389             continue
1390         apks.append(apk)
1391         cachechanged = cachechanged or cachethis
1392
1393     return apks, cachechanged
1394
1395
1396 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1397     """
1398     Extracts icons from the given APK zip in various densities,
1399     saves them into given repo directory
1400     and stores their names in the APK metadata dictionary.
1401
1402     :param icon_filename: A string representing the icon's file name
1403     :param apk: A populated dictionary containing APK metadata.
1404                 Needs to have 'icons_src' key
1405     :param apkzip: An opened zipfile.ZipFile of the APK file
1406     :param repo_dir: The directory of the APK's repository
1407     :return: A list of icon densities that are missing
1408     """
1409     empty_densities = []
1410     for density in screen_densities:
1411         if density not in apk['icons_src']:
1412             empty_densities.append(density)
1413             continue
1414         icon_src = apk['icons_src'][density]
1415         icon_dir = get_icon_dir(repo_dir, density)
1416         icon_dest = os.path.join(icon_dir, icon_filename)
1417
1418         # Extract the icon files per density
1419         if icon_src.endswith('.xml'):
1420             png = os.path.basename(icon_src)[:-4] + '.png'
1421             for f in apkzip.namelist():
1422                 if f.endswith(png):
1423                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1424                     if m and screen_resolutions[m.group(2)] == density:
1425                         icon_src = f
1426             if icon_src.endswith('.xml'):
1427                 empty_densities.append(density)
1428                 continue
1429         try:
1430             with open(icon_dest, 'wb') as f:
1431                 f.write(get_icon_bytes(apkzip, icon_src))
1432             apk['icons'][density] = icon_filename
1433         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1434             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1435             del apk['icons_src'][density]
1436             empty_densities.append(density)
1437
1438     if '-1' in apk['icons_src']:
1439         icon_src = apk['icons_src']['-1']
1440         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1441         with open(icon_path, 'wb') as f:
1442             f.write(get_icon_bytes(apkzip, icon_src))
1443         try:
1444             im = Image.open(icon_path)
1445             dpi = px_to_dpi(im.size[0])
1446             for density in screen_densities:
1447                 if density in apk['icons']:
1448                     break
1449                 if density == screen_densities[-1] or dpi >= int(density):
1450                     apk['icons'][density] = icon_filename
1451                     shutil.move(icon_path,
1452                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1453                     empty_densities.remove(density)
1454                     break
1455         except Exception as e:
1456             logging.warning(_("Failed reading {path}: {error}")
1457                             .format(path=icon_path, error=e))
1458
1459     if apk['icons']:
1460         apk['icon'] = icon_filename
1461
1462     return empty_densities
1463
1464
1465 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1466     """
1467     Resize existing icons for densities missing in the APK to ensure all densities are available
1468
1469     :param empty_densities: A list of icon densities that are missing
1470     :param icon_filename: A string representing the icon's file name
1471     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1472     :param repo_dir: The directory of the APK's repository
1473     """
1474     # First try resizing down to not lose quality
1475     last_density = None
1476     for density in screen_densities:
1477         if density not in empty_densities:
1478             last_density = density
1479             continue
1480         if last_density is None:
1481             continue
1482         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1483
1484         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1485         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1486         fp = None
1487         try:
1488             fp = open(last_icon_path, 'rb')
1489             im = Image.open(fp)
1490
1491             size = dpi_to_px(density)
1492
1493             im.thumbnail((size, size), Image.ANTIALIAS)
1494             im.save(icon_path, "PNG")
1495             empty_densities.remove(density)
1496         except Exception as e:
1497             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1498         finally:
1499             if fp:
1500                 fp.close()
1501
1502     # Then just copy from the highest resolution available
1503     last_density = None
1504     for density in reversed(screen_densities):
1505         if density not in empty_densities:
1506             last_density = density
1507             continue
1508
1509         if last_density is None:
1510             continue
1511
1512         shutil.copyfile(
1513             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1514             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1515         )
1516         empty_densities.remove(density)
1517
1518     for density in screen_densities:
1519         icon_dir = get_icon_dir(repo_dir, density)
1520         icon_dest = os.path.join(icon_dir, icon_filename)
1521         resize_icon(icon_dest, density)
1522
1523     # Copy from icons-mdpi to icons since mdpi is the baseline density
1524     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1525     if os.path.isfile(baseline):
1526         apk['icons']['0'] = icon_filename
1527         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1528
1529
1530 def apply_info_from_latest_apk(apps, apks):
1531     """
1532     Some information from the apks needs to be applied up to the application level.
1533     When doing this, we use the info from the most recent version's apk.
1534     We deal with figuring out when the app was added and last updated at the same time.
1535     """
1536     for appid, app in apps.items():
1537         bestver = UNSET_VERSION_CODE
1538         for apk in apks:
1539             if apk['packageName'] == appid:
1540                 if apk['versionCode'] > bestver:
1541                     bestver = apk['versionCode']
1542                     bestapk = apk
1543
1544                 if 'added' in apk:
1545                     if not app.added or apk['added'] < app.added:
1546                         app.added = apk['added']
1547                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1548                         app.lastUpdated = apk['added']
1549
1550         if not app.added:
1551             logging.debug("Don't know when " + appid + " was added")
1552         if not app.lastUpdated:
1553             logging.debug("Don't know when " + appid + " was last updated")
1554
1555         if bestver == UNSET_VERSION_CODE:
1556
1557             if app.Name is None:
1558                 app.Name = app.AutoName or appid
1559             app.icon = None
1560             logging.debug("Application " + appid + " has no packages")
1561         else:
1562             if app.Name is None:
1563                 app.Name = bestapk['name']
1564             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1565             if app.CurrentVersionCode is None:
1566                 app.CurrentVersionCode = str(bestver)
1567
1568
1569 def make_categories_txt(repodir, categories):
1570     '''Write a category list in the repo to allow quick access'''
1571     catdata = ''
1572     for cat in sorted(categories):
1573         catdata += cat + '\n'
1574     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1575         f.write(catdata)
1576
1577
1578 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1579
1580     def filter_apk_list_sorted(apk_list):
1581         res = []
1582         for apk in apk_list:
1583             if apk['packageName'] == appid:
1584                 res.append(apk)
1585
1586         # Sort the apk list by version code. First is highest/newest.
1587         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1588
1589     for appid, app in apps.items():
1590
1591         if app.ArchivePolicy:
1592             keepversions = int(app.ArchivePolicy[:-9])
1593         else:
1594             keepversions = defaultkeepversions
1595
1596         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1597                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1598
1599         current_app_apks = filter_apk_list_sorted(apks)
1600         if len(current_app_apks) > keepversions:
1601             # Move back the ones we don't want.
1602             for apk in current_app_apks[keepversions:]:
1603                 move_apk_between_sections(repodir, archivedir, apk)
1604                 archapks.append(apk)
1605                 apks.remove(apk)
1606
1607         current_app_archapks = filter_apk_list_sorted(archapks)
1608         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1609             kept = 0
1610             # Move forward the ones we want again, except DisableAlgorithm
1611             for apk in current_app_archapks:
1612                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1613                     move_apk_between_sections(archivedir, repodir, apk)
1614                     archapks.remove(apk)
1615                     apks.append(apk)
1616                     kept += 1
1617                 if kept == keepversions:
1618                     break
1619
1620
1621 def move_apk_between_sections(from_dir, to_dir, apk):
1622     """move an APK from repo to archive or vice versa"""
1623
1624     def _move_file(from_dir, to_dir, filename, ignore_missing):
1625         from_path = os.path.join(from_dir, filename)
1626         if ignore_missing and not os.path.exists(from_path):
1627             return
1628         to_path = os.path.join(to_dir, filename)
1629         if not os.path.exists(to_dir):
1630             os.mkdir(to_dir)
1631         shutil.move(from_path, to_path)
1632
1633     if from_dir == to_dir:
1634         return
1635
1636     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1637     _move_file(from_dir, to_dir, apk['apkName'], False)
1638     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1639     for density in all_screen_densities:
1640         from_icon_dir = get_icon_dir(from_dir, density)
1641         to_icon_dir = get_icon_dir(to_dir, density)
1642         if density not in apk.get('icons', []):
1643             continue
1644         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1645     if 'srcname' in apk:
1646         _move_file(from_dir, to_dir, apk['srcname'], False)
1647
1648
1649 def add_apks_to_per_app_repos(repodir, apks):
1650     apks_per_app = dict()
1651     for apk in apks:
1652         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1653         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1654         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1655         apks_per_app[apk['packageName']] = apk
1656
1657         if not os.path.exists(apk['per_app_icons']):
1658             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1659             os.makedirs(apk['per_app_icons'])
1660
1661         apkpath = os.path.join(repodir, apk['apkName'])
1662         shutil.copy(apkpath, apk['per_app_repo'])
1663         apksigpath = apkpath + '.sig'
1664         if os.path.exists(apksigpath):
1665             shutil.copy(apksigpath, apk['per_app_repo'])
1666         apkascpath = apkpath + '.asc'
1667         if os.path.exists(apkascpath):
1668             shutil.copy(apkascpath, apk['per_app_repo'])
1669
1670
1671 def create_metadata_from_template(apk):
1672     '''create a new metadata file using internal or external template
1673
1674     Generate warnings for apk's with no metadata (or create skeleton
1675     metadata files, if requested on the command line).  Though the
1676     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1677     since those impose things on the metadata file made from the
1678     template: field sort order, empty field value, formatting, etc.
1679     '''
1680
1681     import yaml
1682     if os.path.exists('template.yml'):
1683         with open('template.yml') as f:
1684             metatxt = f.read()
1685         if 'name' in apk and apk['name'] != '':
1686             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1687                              r'\1 ' + apk['name'],
1688                              metatxt,
1689                              flags=re.IGNORECASE | re.MULTILINE)
1690         else:
1691             logging.warning(_('{appid} does not have a name! Using package name instead.')
1692                             .format(appid=apk['packageName']))
1693             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1694                              r'\1 ' + apk['packageName'],
1695                              metatxt,
1696                              flags=re.IGNORECASE | re.MULTILINE)
1697         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1698             f.write(metatxt)
1699     else:
1700         app = dict()
1701         app['Categories'] = [os.path.basename(os.getcwd())]
1702         # include some blanks as part of the template
1703         app['AuthorName'] = ''
1704         app['Summary'] = ''
1705         app['WebSite'] = ''
1706         app['IssueTracker'] = ''
1707         app['SourceCode'] = ''
1708         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1709         if 'name' in apk and apk['name'] != '':
1710             app['Name'] = apk['name']
1711         else:
1712             logging.warning(_('{appid} does not have a name! Using package name instead.')
1713                             .format(appid=apk['packageName']))
1714             app['Name'] = apk['packageName']
1715         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1716             yaml.dump(app, f, default_flow_style=False)
1717     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1718
1719
1720 config = None
1721 options = None
1722
1723
1724 def main():
1725
1726     global config, options
1727
1728     # Parse command line...
1729     parser = ArgumentParser()
1730     common.setup_global_opts(parser)
1731     parser.add_argument("--create-key", action="store_true", default=False,
1732                         help=_("Add a repo signing key to an unsigned repo"))
1733     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1734                         help=_("Add skeleton metadata files for APKs that are missing them"))
1735     parser.add_argument("--delete-unknown", action="store_true", default=False,
1736                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1737     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1738                         help=_("Report on build data status"))
1739     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1740                         help=_("Interactively ask about things that need updating."))
1741     parser.add_argument("-I", "--icons", action="store_true", default=False,
1742                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1743     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1744                         help=_("Specify editor to use in interactive mode. Default " +
1745                                "is {path}").format(path='/etc/alternatives/editor'))
1746     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1747                         help=_("Update the wiki"))
1748     parser.add_argument("--pretty", action="store_true", default=False,
1749                         help=_("Produce human-readable XML/JSON for index files"))
1750     parser.add_argument("--clean", action="store_true", default=False,
1751                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1752     parser.add_argument("--nosign", action="store_true", default=False,
1753                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1754     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1755                         help=_("Use date from APK instead of current time for newly added APKs"))
1756     parser.add_argument("--rename-apks", action="store_true", default=False,
1757                         help=_("Rename APK files that do not match package.name_123.apk"))
1758     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1759                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1760     metadata.add_metadata_arguments(parser)
1761     options = parser.parse_args()
1762     metadata.warnings_action = options.W
1763
1764     config = common.read_config(options)
1765
1766     if not ('jarsigner' in config and 'keytool' in config):
1767         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1768
1769     repodirs = ['repo']
1770     if config['archive_older'] != 0:
1771         repodirs.append('archive')
1772         if not os.path.exists('archive'):
1773             os.mkdir('archive')
1774
1775     if options.icons:
1776         resize_all_icons(repodirs)
1777         sys.exit(0)
1778
1779     if options.rename_apks:
1780         options.clean = True
1781
1782     # check that icons exist now, rather than fail at the end of `fdroid update`
1783     for k in ['repo_icon', 'archive_icon']:
1784         if k in config:
1785             if not os.path.exists(config[k]):
1786                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1787                                  .format(name=k, path=config[k]))
1788                 sys.exit(1)
1789
1790     # if the user asks to create a keystore, do it now, reusing whatever it can
1791     if options.create_key:
1792         if os.path.exists(config['keystore']):
1793             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1794             logging.critical("\t'" + config['keystore'] + "'")
1795             sys.exit(1)
1796
1797         if 'repo_keyalias' not in config:
1798             config['repo_keyalias'] = socket.getfqdn()
1799             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1800         if 'keydname' not in config:
1801             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1802             common.write_to_config(config, 'keydname', config['keydname'])
1803         if 'keystore' not in config:
1804             config['keystore'] = common.default_config['keystore']
1805             common.write_to_config(config, 'keystore', config['keystore'])
1806
1807         password = common.genpassword()
1808         if 'keystorepass' not in config:
1809             config['keystorepass'] = password
1810             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1811         if 'keypass' not in config:
1812             config['keypass'] = password
1813             common.write_to_config(config, 'keypass', config['keypass'])
1814         common.genkeystore(config)
1815
1816     # Get all apps...
1817     apps = metadata.read_metadata()
1818
1819     # Generate a list of categories...
1820     categories = set()
1821     for app in apps.values():
1822         categories.update(app.Categories)
1823
1824     # Read known apks data (will be updated and written back when we've finished)
1825     knownapks = common.KnownApks()
1826
1827     # Get APK cache
1828     apkcache = get_cache()
1829
1830     # Delete builds for disabled apps
1831     delete_disabled_builds(apps, apkcache, repodirs)
1832
1833     # Scan all apks in the main repo
1834     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1835
1836     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1837                                            options.use_date_from_apk)
1838     cachechanged = cachechanged or fcachechanged
1839     apks += files
1840     for apk in apks:
1841         if apk['packageName'] not in apps:
1842             if options.create_metadata:
1843                 create_metadata_from_template(apk)
1844                 apps = metadata.read_metadata()
1845             else:
1846                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1847                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1848                 if options.delete_unknown:
1849                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1850                                  .format(apkfilename=apk['apkName']))
1851                     rmf = os.path.join(repodirs[0], apk['apkName'])
1852                     if not os.path.exists(rmf):
1853                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1854                     else:
1855                         os.remove(rmf)
1856                 else:
1857                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1858
1859     copy_triple_t_store_metadata(apps)
1860     insert_obbs(repodirs[0], apps, apks)
1861     insert_localized_app_metadata(apps)
1862     translate_per_build_anti_features(apps, apks)
1863
1864     # Scan the archive repo for apks as well
1865     if len(repodirs) > 1:
1866         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1867         if cc:
1868             cachechanged = True
1869     else:
1870         archapks = []
1871
1872     # Apply information from latest apks to the application and update dates
1873     apply_info_from_latest_apk(apps, apks + archapks)
1874
1875     # Sort the app list by name, then the web site doesn't have to by default.
1876     # (we had to wait until we'd scanned the apks to do this, because mostly the
1877     # name comes from there!)
1878     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1879
1880     # APKs are placed into multiple repos based on the app package, providing
1881     # per-app subscription feeds for nightly builds and things like it
1882     if config['per_app_repos']:
1883         add_apks_to_per_app_repos(repodirs[0], apks)
1884         for appid, app in apps.items():
1885             repodir = os.path.join(appid, 'fdroid', 'repo')
1886             appdict = dict()
1887             appdict[appid] = app
1888             if os.path.isdir(repodir):
1889                 index.make(appdict, [appid], apks, repodir, False)
1890             else:
1891                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1892         return
1893
1894     if len(repodirs) > 1:
1895         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1896
1897     # Make the index for the main repo...
1898     index.make(apps, sortedids, apks, repodirs[0], False)
1899     make_categories_txt(repodirs[0], categories)
1900
1901     # If there's an archive repo,  make the index for it. We already scanned it
1902     # earlier on.
1903     if len(repodirs) > 1:
1904         index.make(apps, sortedids, archapks, repodirs[1], True)
1905
1906     git_remote = config.get('binary_transparency_remote')
1907     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1908         from . import btlog
1909         btlog.make_binary_transparency_log(repodirs)
1910
1911     if config['update_stats']:
1912         # Update known apks info...
1913         knownapks.writeifchanged()
1914
1915         # Generate latest apps data for widget
1916         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1917             data = ''
1918             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1919                 for line in f:
1920                     appid = line.rstrip()
1921                     data += appid + "\t"
1922                     app = apps[appid]
1923                     data += app.Name + "\t"
1924                     if app.icon is not None:
1925                         data += app.icon + "\t"
1926                     data += app.License + "\n"
1927             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1928                 f.write(data)
1929
1930     if cachechanged:
1931         write_cache(apkcache)
1932
1933     # Update the wiki...
1934     if options.wiki:
1935         update_wiki(apps, sortedids, apks + archapks)
1936
1937     logging.info(_("Finished"))
1938
1939
1940 if __name__ == "__main__":
1941     main()