fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in sorted(app.AntiFeatures):
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.LiberapayID,
 153             app.Bitcoin,
 154             app.Litecoin,
 155             app.License,
 156             requiresroot,
 157             app.AuthorName,
 158             app.AuthorEmail)
 159
 160         if app.Provides:
 161             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 162
 163         wikidata += app.Summary
 164         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 165
 166         wikidata += "=Description=\n"
 167         wikidata += metadata.description_wiki(app.Description) + "\n"
 168
 169         wikidata += "=Maintainer Notes=\n"
 170         if app.MaintainerNotes:
 171             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 172         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 173
 174         # Get a list of all packages for this application...
 175         apklist = []
 176         gotcurrentver = False
 177         cantupdate = False
 178         buildfails = False
 179         for apk in apks:
 180             if apk['packageName'] == appid:
 181                 if str(apk['versionCode']) == app.CurrentVersionCode:
 182                     gotcurrentver = True
 183                 apklist.append(apk)
 184         # Include ones we can't build, as a special case...
 185         for build in app.builds:
 186             if build.disable:
 187                 if build.versionCode == app.CurrentVersionCode:
 188                     cantupdate = True
 189                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 190                 apklist.append({'versionCode': int(build.versionCode),
 191                                 'versionName': build.versionName,
 192                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 193                                 })
 194             else:
 195                 builtit = False
 196                 for apk in apklist:
 197                     if apk['versionCode'] == int(build.versionCode):
 198                         builtit = True
 199                         break
 200                 if not builtit:
 201                     buildfails = True
 202                     apklist.append({'versionCode': int(build.versionCode),
 203                                     'versionName': build.versionName,
 204                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 205                                     })
 206         if app.CurrentVersionCode == '0':
 207             cantupdate = True
 208         # Sort with most recent first...
 209         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 210
 211         wikidata += "=Versions=\n"
 212         if len(apklist) == 0:
 213             wikidata += "We currently have no versions of this app available."
 214         elif not gotcurrentver:
 215             wikidata += "We don't have the current version of this app."
 216         else:
 217             wikidata += "We have the current version of this app."
 218         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 219         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 220         if len(app.NoSourceSince) > 0:
 221             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 222         if len(app.CurrentVersion) > 0:
 223             wikidata += "The current (recommended) version is " + app.CurrentVersion
 224             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 225         validapks = 0
 226         for apk in apklist:
 227             wikidata += "==" + apk['versionName'] + "==\n"
 228
 229             if 'buildproblem' in apk:
 230                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 231             else:
 232                 validapks += 1
 233                 wikidata += "This version is built and signed by "
 234                 if 'srcname' in apk:
 235                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 236                 else:
 237                     wikidata += "the original developer.\n\n"
 238             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 239
 240         wikidata += '\n[[Category:' + wikicat + ']]\n'
 241         if len(app.NoSourceSince) > 0:
 242             wikidata += '\n[[Category:Apps missing source code]]\n'
 243         if validapks == 0 and not app.Disabled:
 244             wikidata += '\n[[Category:Apps with no packages]]\n'
 245         if cantupdate and not app.Disabled:
 246             wikidata += "\n[[Category:Apps we cannot update]]\n"
 247         if buildfails and not app.Disabled:
 248             wikidata += "\n[[Category:Apps with failing builds]]\n"
 249         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 250             wikidata += '\n[[Category:Apps to Update]]\n'
 251         if app.Disabled:
 252             wikidata += '\n[[Category:Apps that are disabled]]\n'
 253         if app.UpdateCheckMode == 'None' and not app.Disabled:
 254             wikidata += '\n[[Category:Apps with no update check]]\n'
 255         for appcat in app.Categories:
 256             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 257
 258         # We can't have underscores in the page name, even if they're in
 259         # the package ID, because MediaWiki messes with them...
 260         pagename = appid.replace('_', ' ')
 261
 262         # Drop a trailing newline, because mediawiki is going to drop it anyway
 263         # and it we don't we'll think the page has changed when it hasn't...
 264         if wikidata.endswith('\n'):
 265             wikidata = wikidata[:-1]
 266
 267         generated_pages[pagename] = wikidata
 268
 269         # Make a redirect from the name to the ID too, unless there's
 270         # already an existing page with the name and it isn't a redirect.
 271         noclobber = False
 272         apppagename = app.Name.replace('_', ' ')
 273         apppagename = apppagename.replace('{', '')
 274         apppagename = apppagename.replace('}', ' ')
 275         apppagename = apppagename.replace(':', ' ')
 276         apppagename = apppagename.replace('[', ' ')
 277         apppagename = apppagename.replace(']', ' ')
 278         # Drop double spaces caused mostly by replacing ':' above
 279         apppagename = apppagename.replace('  ', ' ')
 280         for expagename in site.allpages(prefix=apppagename,
 281                                         filterredir='nonredirects',
 282                                         generator=False):
 283             if expagename == apppagename:
 284                 noclobber = True
 285         # Another reason not to make the redirect page is if the app name
 286         # is the same as it's ID, because that will overwrite the real page
 287         # with an redirect to itself! (Although it seems like an odd
 288         # scenario this happens a lot, e.g. where there is metadata but no
 289         # builds or binaries to extract a name from.
 290         if apppagename == pagename:
 291             noclobber = True
 292         if not noclobber:
 293             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 294
 295     for tcat, genp in [(wikicat, generated_pages),
 296                        (wikiredircat, generated_redirects)]:
 297         catpages = site.Pages['Category:' + tcat]
 298         existingpages = []
 299         for page in catpages:
 300             existingpages.append(page.name)
 301             if page.name in genp:
 302                 pagetxt = page.edit()
 303                 if pagetxt != genp[page.name]:
 304                     logging.debug("Updating modified page " + page.name)
 305                     page.save(genp[page.name], summary='Auto-updated')
 306                 else:
 307                     logging.debug("Page " + page.name + " is unchanged")
 308             else:
 309                 logging.warn("Deleting page " + page.name)
 310                 page.delete('No longer published')
 311         for pagename, text in genp.items():
 312             logging.debug("Checking " + pagename)
 313             if pagename not in existingpages:
 314                 logging.debug("Creating page " + pagename)
 315                 try:
 316                     newpage = site.Pages[pagename]
 317                     newpage.save(text, summary='Auto-created')
 318                 except Exception as e:
 319                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 320
 321     # Purge server cache to ensure counts are up to date
 322     site.pages['Repository Maintenance'].purge()
 323
 324
 325 def delete_disabled_builds(apps, apkcache, repodirs):
 326     """Delete disabled build outputs.
 327
 328     :param apps: list of all applications, as per metadata.read_metadata
 329     :param apkcache: current apk cache information
 330     :param repodirs: the repo directories to process
 331     """
 332     for appid, app in apps.items():
 333         for build in app['builds']:
 334             if not build.disable:
 335                 continue
 336             apkfilename = common.get_release_filename(app, build)
 337             iconfilename = "%s.%s.png" % (
 338                 appid,
 339                 build.versionCode)
 340             for repodir in repodirs:
 341                 files = [
 342                     os.path.join(repodir, apkfilename),
 343                     os.path.join(repodir, apkfilename + '.asc'),
 344                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 345                 ]
 346                 for density in all_screen_densities:
 347                     repo_dir = get_icon_dir(repodir, density)
 348                     files.append(os.path.join(repo_dir, iconfilename))
 349
 350                 for f in files:
 351                     if os.path.exists(f):
 352                         logging.info("Deleting disabled build output " + f)
 353                         os.remove(f)
 354             if apkfilename in apkcache:
 355                 del apkcache[apkfilename]
 356
 357
 358 def resize_icon(iconpath, density):
 359
 360     if not os.path.isfile(iconpath):
 361         return
 362
 363     fp = None
 364     try:
 365         fp = open(iconpath, 'rb')
 366         im = Image.open(fp)
 367         size = dpi_to_px(density)
 368
 369         if any(length > size for length in im.size):
 370             oldsize = im.size
 371             im.thumbnail((size, size), Image.ANTIALIAS)
 372             logging.debug("%s was too large at %s - new size is %s" % (
 373                 iconpath, oldsize, im.size))
 374             im.save(iconpath, "PNG")
 375
 376     except Exception as e:
 377         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 378
 379     finally:
 380         if fp:
 381             fp.close()
 382
 383
 384 def resize_all_icons(repodirs):
 385     """Resize all icons that exceed the max size
 386
 387     :param repodirs: the repo directories to process
 388     """
 389     for repodir in repodirs:
 390         for density in screen_densities:
 391             icon_dir = get_icon_dir(repodir, density)
 392             icon_glob = os.path.join(icon_dir, '*.png')
 393             for iconpath in glob.glob(icon_glob):
 394                 resize_icon(iconpath, density)
 395
 396
 397 def getsig(apkpath):
 398     """ Get the signing certificate of an apk. To get the same md5 has that
 399     Android gets, we encode the .RSA certificate in a specific format and pass
 400     it hex-encoded to the md5 digest algorithm.
 401
 402     :param apkpath: path to the apk
 403     :returns: A string containing the md5 of the signature of the apk or None
 404               if an error occurred.
 405     """
 406
 407     with zipfile.ZipFile(apkpath, 'r') as apk:
 408         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 409
 410         if len(certs) < 1:
 411             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 412             return None
 413         if len(certs) > 1:
 414             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 415             return None
 416
 417         cert = apk.read(certs[0])
 418
 419     cert_encoded = common.get_certificate(cert)
 420
 421     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 422
 423
 424 def get_cache_file():
 425     return os.path.join('tmp', 'apkcache')
 426
 427
 428 def get_cache():
 429     """Get the cached dict of the APK index
 430
 431     Gather information about all the apk files in the repo directory,
 432     using cached data if possible. Some of the index operations take a
 433     long time, like calculating the SHA-256 and verifying the APK
 434     signature.
 435
 436     The cache is invalidated if the metadata version is different, or
 437     the 'allow_disabled_algorithms' config/option is different.  In
 438     those cases, there is no easy way to know what has changed from
 439     the cache, so just rerun the whole thing.
 440
 441     :return: apkcache
 442
 443     """
 444     apkcachefile = get_cache_file()
 445     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 446     if not options.clean and os.path.exists(apkcachefile):
 447         with open(apkcachefile, 'rb') as cf:
 448             apkcache = pickle.load(cf, encoding='utf-8')
 449         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 450            or apkcache.get('allow_disabled_algorithms') != ada:
 451             apkcache = {}
 452     else:
 453         apkcache = {}
 454
 455     apkcache["METADATA_VERSION"] = METADATA_VERSION
 456     apkcache['allow_disabled_algorithms'] = ada
 457
 458     return apkcache
 459
 460
 461 def write_cache(apkcache):
 462     apkcachefile = get_cache_file()
 463     cache_path = os.path.dirname(apkcachefile)
 464     if not os.path.exists(cache_path):
 465         os.makedirs(cache_path)
 466     with open(apkcachefile, 'wb') as cf:
 467         pickle.dump(apkcache, cf)
 468
 469
 470 def get_icon_bytes(apkzip, iconsrc):
 471     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 472     try:
 473         return apkzip.read(iconsrc)
 474     except KeyError:
 475         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 476
 477
 478 def sha256sum(filename):
 479     '''Calculate the sha256 of the given file'''
 480     sha = hashlib.sha256()
 481     with open(filename, 'rb') as f:
 482         while True:
 483             t = f.read(16384)
 484             if len(t) == 0:
 485                 break
 486             sha.update(t)
 487     return sha.hexdigest()
 488
 489
 490 def has_known_vulnerability(filename):
 491     """checks for known vulnerabilities in the APK
 492
 493     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 494     version.  Google also enforces this:
 495     https://support.google.com/faqs/answer/6376725?hl=en
 496
 497     Checks whether there are more than one classes.dex or AndroidManifest.xml
 498     files, which is invalid and an essential part of the "Master Key" attack.
 499     http://www.saurik.com/id/17
 500
 501     Janus is similar to Master Key but is perhaps easier to scan for.
 502     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 503     """
 504
 505     found_vuln = False
 506
 507     # statically load this pattern
 508     if not hasattr(has_known_vulnerability, "pattern"):
 509         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 510
 511     with open(filename.encode(), 'rb') as fp:
 512         first4 = fp.read(4)
 513     if first4 != b'\x50\x4b\x03\x04':
 514         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 515                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 516                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 517
 518     files_in_apk = set()
 519     with zipfile.ZipFile(filename) as zf:
 520         for name in zf.namelist():
 521             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 522                 lib = zf.open(name)
 523                 while True:
 524                     chunk = lib.read(4096)
 525                     if chunk == b'':
 526                         break
 527                     m = has_known_vulnerability.pattern.search(chunk)
 528                     if m:
 529                         version = m.group(1).decode('ascii')
 530                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 531                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 532                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 533                             logging.debug(_('"{path}" contains recent {name} ({version})')
 534                                           .format(path=filename, name=name, version=version))
 535                         else:
 536                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 537                                             .format(path=filename, name=name, version=version))
 538                             found_vuln = True
 539                         break
 540             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 541                 if name in files_in_apk:
 542                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 543                                     .format(apkfilename=filename, name=name))
 544                     found_vuln = True
 545                 files_in_apk.add(name)
 546     return found_vuln
 547
 548
 549 def insert_obbs(repodir, apps, apks):
 550     """Scans the .obb files in a given repo directory and adds them to the
 551     relevant APK instances.  OBB files have versionCodes like APK
 552     files, and they are loosely associated.  If there is an OBB file
 553     present, then any APK with the same or higher versionCode will use
 554     that OBB file.  There are two OBB types: main and patch, each APK
 555     can only have only have one of each.
 556
 557     https://developer.android.com/google/play/expansion-files.html
 558
 559     :param repodir: repo directory to scan
 560     :param apps: list of current, valid apps
 561     :param apks: current information on all APKs
 562
 563     """
 564
 565     def obbWarnDelete(f, msg):
 566         logging.warning(msg + ' ' + f)
 567         if options.delete_unknown:
 568             logging.error(_("Deleting unknown file: {path}").format(path=f))
 569             os.remove(f)
 570
 571     obbs = []
 572     java_Integer_MIN_VALUE = -pow(2, 31)
 573     currentPackageNames = apps.keys()
 574     for f in glob.glob(os.path.join(repodir, '*.obb')):
 575         obbfile = os.path.basename(f)
 576         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 577         chunks = obbfile.split('.')
 578         if chunks[0] != 'main' and chunks[0] != 'patch':
 579             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 580             continue
 581         if not re.match(r'^-?[0-9]+$', chunks[1]):
 582             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 583                           .format(name=chunks[0]))
 584             continue
 585         versionCode = int(chunks[1])
 586         packagename = ".".join(chunks[2:-1])
 587
 588         highestVersionCode = java_Integer_MIN_VALUE
 589         if packagename not in currentPackageNames:
 590             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 591             continue
 592         for apk in apks:
 593             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 594                 highestVersionCode = apk['versionCode']
 595         if versionCode > highestVersionCode:
 596             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 597                           .format(integer=str(versionCode)))
 598             continue
 599         obbsha256 = sha256sum(f)
 600         obbs.append((packagename, versionCode, obbfile, obbsha256))
 601
 602     for apk in apks:
 603         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 604             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 605                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 606                     apk['obbMainFile'] = obbfile
 607                     apk['obbMainFileSha256'] = obbsha256
 608                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 609                     apk['obbPatchFile'] = obbfile
 610                     apk['obbPatchFileSha256'] = obbsha256
 611             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 612                 break
 613
 614
 615 def translate_per_build_anti_features(apps, apks):
 616     """Grab the anti-features list from the build metadata
 617
 618     For most Anti-Features, they are really most applicable per-APK,
 619     not for an app.  An app can fix a vulnerability, add/remove
 620     tracking, etc.  This reads the 'antifeatures' list from the Build
 621     entries in the fdroiddata metadata file, then transforms it into
 622     the 'antiFeatures' list of unique items for the index.
 623
 624     The field key is all lower case in the metadata file to match the
 625     rest of the Build fields.  It is 'antiFeatures' camel case in the
 626     implementation, index, and fdroidclient since it is translated
 627     from the build 'antifeatures' field, not directly included.
 628
 629     """
 630
 631     antiFeatures = dict()
 632     for packageName, app in apps.items():
 633         d = dict()
 634         for build in app['builds']:
 635             afl = build.get('antifeatures')
 636             if afl:
 637                 d[int(build.versionCode)] = afl
 638         if len(d) > 0:
 639             antiFeatures[packageName] = d
 640
 641     for apk in apks:
 642         d = antiFeatures.get(apk['packageName'])
 643         if d:
 644             afl = d.get(apk['versionCode'])
 645             if afl:
 646                 apk['antiFeatures'].update(afl)
 647
 648
 649 def _get_localized_dict(app, locale):
 650     '''get the dict to add localized store metadata to'''
 651     if 'localized' not in app:
 652         app['localized'] = collections.OrderedDict()
 653     if locale not in app['localized']:
 654         app['localized'][locale] = collections.OrderedDict()
 655     return app['localized'][locale]
 656
 657
 658 def _set_localized_text_entry(app, locale, key, f):
 659     limit = config['char_limits'][key]
 660     localized = _get_localized_dict(app, locale)
 661     with open(f) as fp:
 662         text = fp.read()[:limit]
 663         if len(text) > 0:
 664             localized[key] = text
 665
 666
 667 def _set_author_entry(app, key, f):
 668     limit = config['char_limits']['author']
 669     with open(f) as fp:
 670         text = fp.read()[:limit]
 671         if len(text) > 0:
 672             app[key] = text
 673
 674
 675 def _strip_and_copy_image(inpath, outpath):
 676     """Remove any metadata from image and copy it to new path
 677
 678     Sadly, image metadata like EXIF can be used to exploit devices.
 679     It is not used at all in the F-Droid ecosystem, so its much safer
 680     just to remove it entirely.  PNG does not have the same kind of
 681     issues.
 682
 683     """
 684
 685     if common.has_extension(inpath, 'png'):
 686         shutil.copy(inpath, outpath)
 687     else:
 688         with open(inpath) as fp:
 689             in_image = Image.open(fp)
 690             data = list(in_image.getdata())
 691             out_image = Image.new(in_image.mode, in_image.size)
 692         out_image.putdata(data)
 693         out_image.save(outpath, "JPEG", optimize=True)
 694
 695
 696 def copy_triple_t_store_metadata(apps):
 697     """Include store metadata from the app's source repo
 698
 699     The Triple-T Gradle Play Publisher is a plugin that has a standard
 700     file layout for all of the metadata and graphics that the Google
 701     Play Store accepts.  Since F-Droid has the git repo, it can just
 702     pluck those files directly.  This method reads any text files into
 703     the app dict, then copies any graphics into the fdroid repo
 704     directory structure.
 705
 706     This needs to be run before insert_localized_app_metadata() so that
 707     the graphics files that are copied into the fdroid repo get
 708     properly indexed.
 709
 710     https://github.com/Triple-T/gradle-play-publisher#upload-images
 711     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 712
 713     """
 714
 715     if not os.path.isdir('build'):
 716         return  # nothing to do
 717
 718     for packageName, app in apps.items():
 719         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 720             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 721             for root, dirs, files in os.walk(d):
 722                 segments = root.split('/')
 723                 locale = segments[-2]
 724                 for f in files:
 725                     if f == 'fulldescription':
 726                         _set_localized_text_entry(app, locale, 'description',
 727                                                   os.path.join(root, f))
 728                         continue
 729                     elif f == 'shortdescription':
 730                         _set_localized_text_entry(app, locale, 'summary',
 731                                                   os.path.join(root, f))
 732                         continue
 733                     elif f == 'title':
 734                         _set_localized_text_entry(app, locale, 'name',
 735                                                   os.path.join(root, f))
 736                         continue
 737                     elif f == 'video':
 738                         _set_localized_text_entry(app, locale, 'video',
 739                                                   os.path.join(root, f))
 740                         continue
 741                     elif f == 'whatsnew':
 742                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 743                                                   os.path.join(root, f))
 744                         continue
 745                     elif f == 'contactEmail':
 746                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 747                         continue
 748                     elif f == 'contactPhone':
 749                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 750                         continue
 751                     elif f == 'contactWebsite':
 752                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 753                         continue
 754
 755                     base, extension = common.get_extension(f)
 756                     dirname = os.path.basename(root)
 757                     if extension in ALLOWED_EXTENSIONS \
 758                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 759                         if segments[-2] == 'listing':
 760                             locale = segments[-3]
 761                         else:
 762                             locale = segments[-2]
 763                         destdir = os.path.join('repo', packageName, locale, dirname)
 764                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 765                         sourcefile = os.path.join(root, f)
 766                         destfile = os.path.join(destdir, os.path.basename(f))
 767                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 768                         _strip_and_copy_image(sourcefile, destfile)
 769
 770
 771 def insert_localized_app_metadata(apps):
 772     """scans standard locations for graphics and localized text
 773
 774     Scans for localized description files, store graphics, and
 775     screenshot PNG files in statically defined screenshots directory
 776     and adds them to the app metadata.  The screenshots and graphic
 777     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 778     and must be in the following layout:
 779     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 780
 781     repo/packageName/locale/featureGraphic.png
 782     repo/packageName/locale/phoneScreenshots/1.png
 783     repo/packageName/locale/phoneScreenshots/2.png
 784
 785     The changelog files must be text files named with the versionCode
 786     ending with ".txt" and must be in the following layout:
 787     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 788
 789     repo/packageName/locale/changelogs/12345.txt
 790
 791     This will scan the each app's source repo then the metadata/ dir
 792     for these standard locations of changelog files.  If it finds
 793     them, they will be added to the dict of all packages, with the
 794     versions in the metadata/ folder taking precendence over the what
 795     is in the app's source repo.
 796
 797     Where "packageName" is the app's packageName and "locale" is the locale
 798     of the graphics, e.g. what language they are in, using the IETF RFC5646
 799     format (en-US, fr-CA, es-MX, etc).
 800
 801     This will also scan the app's git for a fastlane folder, and the
 802     metadata/ folder and the apps' source repos for standard locations
 803     of graphic and screenshot files.  If it finds them, it will copy
 804     them into the repo.  The fastlane files follow this pattern:
 805     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 806
 807     """
 808
 809     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 810     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 811     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 812     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 813
 814     for srcd in sorted(sourcedirs):
 815         if not os.path.isdir(srcd):
 816             continue
 817         for root, dirs, files in os.walk(srcd):
 818             segments = root.split('/')
 819             packageName = segments[1]
 820             if packageName not in apps:
 821                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 822                 continue
 823             locale = segments[-1]
 824             destdir = os.path.join('repo', packageName, locale)
 825
 826             # flavours specified in build receipt
 827             build_flavours = ""
 828             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 829                     and 'gradle' in apps[packageName].builds[-1]:
 830                 build_flavours = apps[packageName].builds[-1].gradle
 831
 832             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 833                 logging.debug("ignoring due to wrong flavour")
 834                 continue
 835
 836             for f in files:
 837                 if f in ('description.txt', 'full_description.txt'):
 838                     _set_localized_text_entry(apps[packageName], locale, 'description',
 839                                               os.path.join(root, f))
 840                     continue
 841                 elif f in ('summary.txt', 'short_description.txt'):
 842                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 843                                               os.path.join(root, f))
 844                     continue
 845                 elif f in ('name.txt', 'title.txt'):
 846                     _set_localized_text_entry(apps[packageName], locale, 'name',
 847                                               os.path.join(root, f))
 848                     continue
 849                 elif f == 'video.txt':
 850                     _set_localized_text_entry(apps[packageName], locale, 'video',
 851                                               os.path.join(root, f))
 852                     continue
 853                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 854                     locale = segments[-2]
 855                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 856                                               os.path.join(root, f))
 857                     continue
 858
 859                 base, extension = common.get_extension(f)
 860                 if locale == 'images':
 861                     locale = segments[-2]
 862                     destdir = os.path.join('repo', packageName, locale)
 863                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 864                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 865                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 866                     _strip_and_copy_image(os.path.join(root, f), destdir)
 867             for d in dirs:
 868                 if d in SCREENSHOT_DIRS:
 869                     if locale == 'images':
 870                         locale = segments[-2]
 871                         destdir = os.path.join('repo', packageName, locale)
 872                     for f in glob.glob(os.path.join(root, d, '*.*')):
 873                         _ignored, extension = common.get_extension(f)
 874                         if extension in ALLOWED_EXTENSIONS:
 875                             screenshotdestdir = os.path.join(destdir, d)
 876                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 877                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 878                             _strip_and_copy_image(f, screenshotdestdir)
 879
 880     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 881     for d in repofiles:
 882         if not os.path.isdir(d):
 883             continue
 884         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 885             if not os.path.isfile(f):
 886                 continue
 887             segments = f.split('/')
 888             packageName = segments[1]
 889             locale = segments[2]
 890             screenshotdir = segments[3]
 891             filename = os.path.basename(f)
 892             base, extension = common.get_extension(filename)
 893
 894             if packageName not in apps:
 895                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 896                                 .format(path=filename, name=packageName))
 897                 continue
 898             graphics = _get_localized_dict(apps[packageName], locale)
 899
 900             if extension not in ALLOWED_EXTENSIONS:
 901                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 902             elif base in GRAPHIC_NAMES:
 903                 # there can only be zero or one of these per locale
 904                 graphics[base] = filename
 905             elif screenshotdir in SCREENSHOT_DIRS:
 906                 # there can any number of these per locale
 907                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 908                 if screenshotdir not in graphics:
 909                     graphics[screenshotdir] = []
 910                 graphics[screenshotdir].append(filename)
 911             else:
 912                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 913
 914
 915 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 916     """Scan a repo for all files with an extension except APK/OBB
 917
 918     :param apkcache: current cached info about all repo files
 919     :param repodir: repo directory to scan
 920     :param knownapks: list of all known files, as per metadata.read_metadata
 921     :param use_date_from_file: use date from file (instead of current date)
 922                                for newly added files
 923     """
 924
 925     cachechanged = False
 926     repo_files = []
 927     repodir = repodir.encode('utf-8')
 928     for name in os.listdir(repodir):
 929         file_extension = common.get_file_extension(name)
 930         if file_extension == 'apk' or file_extension == 'obb':
 931             continue
 932         filename = os.path.join(repodir, name)
 933         name_utf8 = name.decode('utf-8')
 934         if filename.endswith(b'_src.tar.gz'):
 935             logging.debug(_('skipping source tarball: {path}')
 936                           .format(path=filename.decode('utf-8')))
 937             continue
 938         if not common.is_repo_file(filename):
 939             continue
 940         stat = os.stat(filename)
 941         if stat.st_size == 0:
 942             raise FDroidException(_('{path} is zero size!')
 943                                   .format(path=filename))
 944
 945         shasum = sha256sum(filename)
 946         usecache = False
 947         if name in apkcache:
 948             repo_file = apkcache[name]
 949             # added time is cached as tuple but used here as datetime instance
 950             if 'added' in repo_file:
 951                 a = repo_file['added']
 952                 if isinstance(a, datetime):
 953                     repo_file['added'] = a
 954                 else:
 955                     repo_file['added'] = datetime(*a[:6])
 956             if repo_file.get('hash') == shasum:
 957                 logging.debug(_("Reading {apkfilename} from cache")
 958                               .format(apkfilename=name_utf8))
 959                 usecache = True
 960             else:
 961                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 962                               .format(apkfilename=name_utf8))
 963
 964         if not usecache:
 965             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 966             repo_file = collections.OrderedDict()
 967             repo_file['name'] = os.path.splitext(name_utf8)[0]
 968             # TODO rename apkname globally to something more generic
 969             repo_file['apkName'] = name_utf8
 970             repo_file['hash'] = shasum
 971             repo_file['hashType'] = 'sha256'
 972             repo_file['versionCode'] = 0
 973             repo_file['versionName'] = shasum
 974             # the static ID is the SHA256 unless it is set in the metadata
 975             repo_file['packageName'] = shasum
 976
 977             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 978             if m:
 979                 repo_file['packageName'] = m.group(1)
 980                 repo_file['versionCode'] = int(m.group(2))
 981             srcfilename = name + b'_src.tar.gz'
 982             if os.path.exists(os.path.join(repodir, srcfilename)):
 983                 repo_file['srcname'] = srcfilename.decode('utf-8')
 984             repo_file['size'] = stat.st_size
 985
 986             apkcache[name] = repo_file
 987             cachechanged = True
 988
 989         if use_date_from_file:
 990             timestamp = stat.st_ctime
 991             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 992         else:
 993             default_date_param = None
 994
 995         # Record in knownapks, getting the added date at the same time..
 996         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 997                                     default_date=default_date_param)
 998         if added:
 999             repo_file['added'] = added
1000
1001         repo_files.append(repo_file)
1002
1003     return repo_files, cachechanged
1004
1005
1006 def scan_apk(apk_file):
1007     """
1008     Scans an APK file and returns dictionary with metadata of the APK.
1009
1010     Attention: This does *not* verify that the APK signature is correct.
1011
1012     :param apk_file: The (ideally absolute) path to the APK file
1013     :raises BuildException
1014     :return A dict containing APK metadata
1015     """
1016     apk = {
1017         'hash': sha256sum(apk_file),
1018         'hashType': 'sha256',
1019         'uses-permission': [],
1020         'uses-permission-sdk-23': [],
1021         'features': [],
1022         'icons_src': {},
1023         'icons': {},
1024         'antiFeatures': set(),
1025     }
1026
1027     if SdkToolsPopen(['aapt', 'version'], output=False):
1028         scan_apk_aapt(apk, apk_file)
1029     else:
1030         scan_apk_androguard(apk, apk_file)
1031
1032     # Get the signature, or rather the signing key fingerprints
1033     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1034     apk['sig'] = getsig(apk_file)
1035     if not apk['sig']:
1036         raise BuildException("Failed to get apk signature")
1037     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1038                                                                apk_file))
1039     if not apk.get('signer'):
1040         raise BuildException("Failed to get apk signing key fingerprint")
1041
1042     # Get size of the APK
1043     apk['size'] = os.path.getsize(apk_file)
1044
1045     if 'minSdkVersion' not in apk:
1046         logging.warning("No SDK version information found in {0}".format(apk_file))
1047         apk['minSdkVersion'] = 1
1048
1049     # Check for known vulnerabilities
1050     if has_known_vulnerability(apk_file):
1051         apk['antiFeatures'].add('KnownVuln')
1052
1053     return apk
1054
1055
1056 def scan_apk_aapt(apk, apkfile):
1057     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1058     if p.returncode != 0:
1059         if options.delete_unknown:
1060             if os.path.exists(apkfile):
1061                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1062                 os.remove(apkfile)
1063             else:
1064                 logging.error("Could not find {0} to remove it".format(apkfile))
1065         else:
1066             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1067         raise BuildException(_("Invalid APK"))
1068     for line in p.output.splitlines():
1069         if line.startswith("package:"):
1070             try:
1071                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1072                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1073                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1074             except Exception as e:
1075                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1076         elif line.startswith("application:"):
1077             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1078             # Keep path to non-dpi icon in case we need it
1079             match = re.match(APK_ICON_PAT_NODPI, line)
1080             if match:
1081                 apk['icons_src']['-1'] = match.group(1)
1082         elif line.startswith("launchable-activity:"):
1083             # Only use launchable-activity as fallback to application
1084             if not apk['name']:
1085                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1086             if '-1' not in apk['icons_src']:
1087                 match = re.match(APK_ICON_PAT_NODPI, line)
1088                 if match:
1089                     apk['icons_src']['-1'] = match.group(1)
1090         elif line.startswith("application-icon-"):
1091             match = re.match(APK_ICON_PAT, line)
1092             if match:
1093                 density = match.group(1)
1094                 path = match.group(2)
1095                 apk['icons_src'][density] = path
1096         elif line.startswith("sdkVersion:"):
1097             m = re.match(APK_SDK_VERSION_PAT, line)
1098             if m is None:
1099                 logging.error(line.replace('sdkVersion:', '')
1100                               + ' is not a valid minSdkVersion!')
1101             else:
1102                 apk['minSdkVersion'] = m.group(1)
1103                 # if target not set, default to min
1104                 if 'targetSdkVersion' not in apk:
1105                     apk['targetSdkVersion'] = m.group(1)
1106         elif line.startswith("targetSdkVersion:"):
1107             m = re.match(APK_SDK_VERSION_PAT, line)
1108             if m is None:
1109                 logging.error(line.replace('targetSdkVersion:', '')
1110                               + ' is not a valid targetSdkVersion!')
1111             else:
1112                 apk['targetSdkVersion'] = m.group(1)
1113         elif line.startswith("maxSdkVersion:"):
1114             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1115         elif line.startswith("native-code:"):
1116             apk['nativecode'] = []
1117             for arch in line[13:].split(' '):
1118                 apk['nativecode'].append(arch[1:-1])
1119         elif line.startswith('uses-permission:'):
1120             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1121             if perm_match['maxSdkVersion']:
1122                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1123             permission = UsesPermission(
1124                 perm_match['name'],
1125                 perm_match['maxSdkVersion']
1126             )
1127
1128             apk['uses-permission'].append(permission)
1129         elif line.startswith('uses-permission-sdk-23:'):
1130             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1131             if perm_match['maxSdkVersion']:
1132                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1133             permission_sdk_23 = UsesPermissionSdk23(
1134                 perm_match['name'],
1135                 perm_match['maxSdkVersion']
1136             )
1137
1138             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1139
1140         elif line.startswith('uses-feature:'):
1141             feature = re.match(APK_FEATURE_PAT, line).group(1)
1142             # Filter out this, it's only added with the latest SDK tools and
1143             # causes problems for lots of apps.
1144             if feature != "android.hardware.screen.portrait" \
1145                     and feature != "android.hardware.screen.landscape":
1146                 if feature.startswith("android.feature."):
1147                     feature = feature[16:]
1148                 apk['features'].add(feature)
1149
1150
1151 def scan_apk_androguard(apk, apkfile):
1152     try:
1153         from androguard.core.bytecodes.apk import APK
1154         apkobject = APK(apkfile)
1155         if apkobject.is_valid_APK():
1156             arsc = apkobject.get_android_resources()
1157         else:
1158             if options.delete_unknown:
1159                 if os.path.exists(apkfile):
1160                     logging.error(_("Failed to get apk information, deleting {path}")
1161                                   .format(path=apkfile))
1162                     os.remove(apkfile)
1163                 else:
1164                     logging.error(_("Could not find {path} to remove it")
1165                                   .format(path=apkfile))
1166             else:
1167                 logging.error(_("Failed to get apk information, skipping {path}")
1168                               .format(path=apkfile))
1169             raise BuildException(_("Invalid APK"))
1170     except ImportError:
1171         raise FDroidException("androguard library is not installed and aapt not present")
1172     except FileNotFoundError:
1173         logging.error(_("Could not open apk file for analysis"))
1174         raise BuildException(_("Invalid APK"))
1175
1176     apk['packageName'] = apkobject.get_package()
1177     apk['versionCode'] = int(apkobject.get_androidversion_code())
1178     apk['versionName'] = apkobject.get_androidversion_name()
1179     if apk['versionName'][0] == "@":
1180         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1181         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1182         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1183     apk['name'] = apkobject.get_app_name()
1184
1185     if apkobject.get_max_sdk_version() is not None:
1186         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1187     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1188     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1189
1190     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1191     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1192
1193     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1194
1195     for file in apkobject.get_files():
1196         d_re = density_re.match(file)
1197         if d_re:
1198             folder = d_re.group(1).split('-')
1199             if len(folder) > 1:
1200                 resolution = folder[1]
1201             else:
1202                 resolution = 'mdpi'
1203             density = screen_resolutions[resolution]
1204             apk['icons_src'][density] = d_re.group(0)
1205
1206     if apk['icons_src'].get('-1') is None:
1207         apk['icons_src']['-1'] = apk['icons_src']['160']
1208
1209     arch_re = re.compile("^lib/(.*)/.*$")
1210     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1211     if len(arch) >= 1:
1212         apk['nativecode'] = []
1213         apk['nativecode'].extend(sorted(list(arch)))
1214
1215     xml = apkobject.get_android_manifest_xml()
1216
1217     for item in xml.getElementsByTagName('uses-permission'):
1218         name = str(item.getAttribute("android:name"))
1219         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1220         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1221         permission = UsesPermission(
1222             name,
1223             maxSdkVersion
1224         )
1225         apk['uses-permission'].append(permission)
1226
1227     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1228         name = str(item.getAttribute("android:name"))
1229         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1230         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1231         permission_sdk_23 = UsesPermissionSdk23(
1232             name,
1233             maxSdkVersion
1234         )
1235         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1236
1237     for item in xml.getElementsByTagName('uses-feature'):
1238         feature = str(item.getAttribute("android:name"))
1239         if feature != "android.hardware.screen.portrait" \
1240                 and feature != "android.hardware.screen.landscape":
1241             if feature.startswith("android.feature."):
1242                 feature = feature[16:]
1243         apk['features'].append(feature)
1244
1245
1246 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1247                 allow_disabled_algorithms=False, archive_bad_sig=False):
1248     """Processes the apk with the given filename in the given repo directory.
1249
1250     This also extracts the icons.
1251
1252     :param apkcache: current apk cache information
1253     :param apkfilename: the filename of the apk to scan
1254     :param repodir: repo directory to scan
1255     :param knownapks: known apks info
1256     :param use_date_from_apk: use date from APK (instead of current date)
1257                               for newly added APKs
1258     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1259                                       disabled algorithms in the signature (e.g. MD5)
1260     :param archive_bad_sig: move APKs with a bad signature to the archive
1261     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1262      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1263     """
1264
1265     apk = {}
1266     apkfile = os.path.join(repodir, apkfilename)
1267
1268     cachechanged = False
1269     usecache = False
1270     if apkfilename in apkcache:
1271         apk = apkcache[apkfilename]
1272         if apk.get('hash') == sha256sum(apkfile):
1273             logging.debug(_("Reading {apkfilename} from cache")
1274                           .format(apkfilename=apkfilename))
1275             usecache = True
1276         else:
1277             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1278                           .format(apkfilename=apkfilename))
1279
1280     if not usecache:
1281         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1282
1283         try:
1284             apk = scan_apk(apkfile)
1285         except BuildException:
1286             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1287                             .format(apkfilename=apkfilename))
1288             return True, None, False
1289
1290         # Check for debuggable apks...
1291         if common.isApkAndDebuggable(apkfile):
1292             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1293
1294         if options.rename_apks:
1295             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1296             std_short_name = os.path.join(repodir, n)
1297             if apkfile != std_short_name:
1298                 if os.path.exists(std_short_name):
1299                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1300                     if apkfile != std_long_name:
1301                         if os.path.exists(std_long_name):
1302                             dupdir = os.path.join('duplicates', repodir)
1303                             if not os.path.isdir(dupdir):
1304                                 os.makedirs(dupdir, exist_ok=True)
1305                             dupfile = os.path.join('duplicates', std_long_name)
1306                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1307                             os.rename(apkfile, dupfile)
1308                             return True, None, False
1309                         else:
1310                             os.rename(apkfile, std_long_name)
1311                     apkfile = std_long_name
1312                 else:
1313                     os.rename(apkfile, std_short_name)
1314                     apkfile = std_short_name
1315                 apkfilename = apkfile[len(repodir) + 1:]
1316
1317         apk['apkName'] = apkfilename
1318         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1319         if os.path.exists(os.path.join(repodir, srcfilename)):
1320             apk['srcname'] = srcfilename
1321
1322         # verify the jar signature is correct, allow deprecated
1323         # algorithms only if the APK is in the archive.
1324         skipapk = False
1325         if not common.verify_apk_signature(apkfile):
1326             if repodir == 'archive' or allow_disabled_algorithms:
1327                 if common.verify_old_apk_signature(apkfile):
1328                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1329                 else:
1330                     skipapk = True
1331             else:
1332                 skipapk = True
1333
1334         if skipapk:
1335             if archive_bad_sig:
1336                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1337                                 .format(apkfilename=apkfilename))
1338                 move_apk_between_sections(repodir, 'archive', apk)
1339             else:
1340                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1341                                 .format(apkfilename=apkfilename))
1342             return True, None, False
1343
1344         apkzip = zipfile.ZipFile(apkfile, 'r')
1345
1346         manifest = apkzip.getinfo('AndroidManifest.xml')
1347         if manifest.date_time[1] == 0:  # month can't be zero
1348             logging.debug(_('AndroidManifest.xml has no date'))
1349         else:
1350             common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1351
1352         # extract icons from APK zip file
1353         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1354         try:
1355             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1356         finally:
1357             apkzip.close()  # ensure that APK zip file gets closed
1358
1359         # resize existing icons for densities missing in the APK
1360         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1361
1362         if use_date_from_apk and manifest.date_time[1] != 0:
1363             default_date_param = datetime(*manifest.date_time)
1364         else:
1365             default_date_param = None
1366
1367         # Record in known apks, getting the added date at the same time..
1368         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1369                                     default_date=default_date_param)
1370         if added:
1371             apk['added'] = added
1372
1373         apkcache[apkfilename] = apk
1374         cachechanged = True
1375
1376     return False, apk, cachechanged
1377
1378
1379 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1380     """Processes the apks in the given repo directory.
1381
1382     This also extracts the icons.
1383
1384     :param apkcache: current apk cache information
1385     :param repodir: repo directory to scan
1386     :param knownapks: known apks info
1387     :param use_date_from_apk: use date from APK (instead of current date)
1388                               for newly added APKs
1389     :returns: (apks, cachechanged) where apks is a list of apk information,
1390               and cachechanged is True if the apkcache got changed.
1391     """
1392
1393     cachechanged = False
1394
1395     for icon_dir in get_all_icon_dirs(repodir):
1396         if os.path.exists(icon_dir):
1397             if options.clean:
1398                 shutil.rmtree(icon_dir)
1399                 os.makedirs(icon_dir)
1400         else:
1401             os.makedirs(icon_dir)
1402
1403     apks = []
1404     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1405         apkfilename = apkfile[len(repodir) + 1:]
1406         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1407         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1408                                              use_date_from_apk, ada, True)
1409         if skip:
1410             continue
1411         apks.append(apk)
1412         cachechanged = cachechanged or cachethis
1413
1414     return apks, cachechanged
1415
1416
1417 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1418     """
1419     Extracts icons from the given APK zip in various densities,
1420     saves them into given repo directory
1421     and stores their names in the APK metadata dictionary.
1422
1423     :param icon_filename: A string representing the icon's file name
1424     :param apk: A populated dictionary containing APK metadata.
1425                 Needs to have 'icons_src' key
1426     :param apkzip: An opened zipfile.ZipFile of the APK file
1427     :param repo_dir: The directory of the APK's repository
1428     :return: A list of icon densities that are missing
1429     """
1430     empty_densities = []
1431     for density in screen_densities:
1432         if density not in apk['icons_src']:
1433             empty_densities.append(density)
1434             continue
1435         icon_src = apk['icons_src'][density]
1436         icon_dir = get_icon_dir(repo_dir, density)
1437         icon_dest = os.path.join(icon_dir, icon_filename)
1438
1439         # Extract the icon files per density
1440         if icon_src.endswith('.xml'):
1441             png = os.path.basename(icon_src)[:-4] + '.png'
1442             for f in apkzip.namelist():
1443                 if f.endswith(png):
1444                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1445                     if m and screen_resolutions[m.group(2)] == density:
1446                         icon_src = f
1447             if icon_src.endswith('.xml'):
1448                 empty_densities.append(density)
1449                 continue
1450         try:
1451             with open(icon_dest, 'wb') as f:
1452                 f.write(get_icon_bytes(apkzip, icon_src))
1453             apk['icons'][density] = icon_filename
1454         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1455             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1456             del apk['icons_src'][density]
1457             empty_densities.append(density)
1458
1459     if '-1' in apk['icons_src']:
1460         icon_src = apk['icons_src']['-1']
1461         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1462         with open(icon_path, 'wb') as f:
1463             f.write(get_icon_bytes(apkzip, icon_src))
1464         try:
1465             im = Image.open(icon_path)
1466             dpi = px_to_dpi(im.size[0])
1467             for density in screen_densities:
1468                 if density in apk['icons']:
1469                     break
1470                 if density == screen_densities[-1] or dpi >= int(density):
1471                     apk['icons'][density] = icon_filename
1472                     shutil.move(icon_path,
1473                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1474                     empty_densities.remove(density)
1475                     break
1476         except Exception as e:
1477             logging.warning(_("Failed reading {path}: {error}")
1478                             .format(path=icon_path, error=e))
1479
1480     if apk['icons']:
1481         apk['icon'] = icon_filename
1482
1483     return empty_densities
1484
1485
1486 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1487     """
1488     Resize existing icons for densities missing in the APK to ensure all densities are available
1489
1490     :param empty_densities: A list of icon densities that are missing
1491     :param icon_filename: A string representing the icon's file name
1492     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1493     :param repo_dir: The directory of the APK's repository
1494     """
1495     # First try resizing down to not lose quality
1496     last_density = None
1497     for density in screen_densities:
1498         if density not in empty_densities:
1499             last_density = density
1500             continue
1501         if last_density is None:
1502             continue
1503         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1504
1505         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1506         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1507         fp = None
1508         try:
1509             fp = open(last_icon_path, 'rb')
1510             im = Image.open(fp)
1511
1512             size = dpi_to_px(density)
1513
1514             im.thumbnail((size, size), Image.ANTIALIAS)
1515             im.save(icon_path, "PNG")
1516             empty_densities.remove(density)
1517         except Exception as e:
1518             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1519         finally:
1520             if fp:
1521                 fp.close()
1522
1523     # Then just copy from the highest resolution available
1524     last_density = None
1525     for density in reversed(screen_densities):
1526         if density not in empty_densities:
1527             last_density = density
1528             continue
1529
1530         if last_density is None:
1531             continue
1532
1533         shutil.copyfile(
1534             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1535             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1536         )
1537         empty_densities.remove(density)
1538
1539     for density in screen_densities:
1540         icon_dir = get_icon_dir(repo_dir, density)
1541         icon_dest = os.path.join(icon_dir, icon_filename)
1542         resize_icon(icon_dest, density)
1543
1544     # Copy from icons-mdpi to icons since mdpi is the baseline density
1545     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1546     if os.path.isfile(baseline):
1547         apk['icons']['0'] = icon_filename
1548         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1549
1550
1551 def apply_info_from_latest_apk(apps, apks):
1552     """
1553     Some information from the apks needs to be applied up to the application level.
1554     When doing this, we use the info from the most recent version's apk.
1555     We deal with figuring out when the app was added and last updated at the same time.
1556     """
1557     for appid, app in apps.items():
1558         bestver = UNSET_VERSION_CODE
1559         for apk in apks:
1560             if apk['packageName'] == appid:
1561                 if apk['versionCode'] > bestver:
1562                     bestver = apk['versionCode']
1563                     bestapk = apk
1564
1565                 if 'added' in apk:
1566                     if not app.added or apk['added'] < app.added:
1567                         app.added = apk['added']
1568                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1569                         app.lastUpdated = apk['added']
1570
1571         if not app.added:
1572             logging.debug("Don't know when " + appid + " was added")
1573         if not app.lastUpdated:
1574             logging.debug("Don't know when " + appid + " was last updated")
1575
1576         if bestver == UNSET_VERSION_CODE:
1577
1578             if app.Name is None:
1579                 app.Name = app.AutoName or appid
1580             app.icon = None
1581             logging.debug("Application " + appid + " has no packages")
1582         else:
1583             if app.Name is None:
1584                 app.Name = bestapk['name']
1585             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1586             if app.CurrentVersionCode is None:
1587                 app.CurrentVersionCode = str(bestver)
1588
1589
1590 def make_categories_txt(repodir, categories):
1591     '''Write a category list in the repo to allow quick access'''
1592     catdata = ''
1593     for cat in sorted(categories):
1594         catdata += cat + '\n'
1595     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1596         f.write(catdata)
1597
1598
1599 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1600
1601     def filter_apk_list_sorted(apk_list):
1602         res = []
1603         for apk in apk_list:
1604             if apk['packageName'] == appid:
1605                 res.append(apk)
1606
1607         # Sort the apk list by version code. First is highest/newest.
1608         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1609
1610     for appid, app in apps.items():
1611
1612         if app.ArchivePolicy:
1613             keepversions = int(app.ArchivePolicy[:-9])
1614         else:
1615             keepversions = defaultkeepversions
1616
1617         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1618                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1619
1620         current_app_apks = filter_apk_list_sorted(apks)
1621         if len(current_app_apks) > keepversions:
1622             # Move back the ones we don't want.
1623             for apk in current_app_apks[keepversions:]:
1624                 move_apk_between_sections(repodir, archivedir, apk)
1625                 archapks.append(apk)
1626                 apks.remove(apk)
1627
1628         current_app_archapks = filter_apk_list_sorted(archapks)
1629         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1630             kept = 0
1631             # Move forward the ones we want again, except DisableAlgorithm
1632             for apk in current_app_archapks:
1633                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1634                     move_apk_between_sections(archivedir, repodir, apk)
1635                     archapks.remove(apk)
1636                     apks.append(apk)
1637                     kept += 1
1638                 if kept == keepversions:
1639                     break
1640
1641
1642 def move_apk_between_sections(from_dir, to_dir, apk):
1643     """move an APK from repo to archive or vice versa"""
1644
1645     def _move_file(from_dir, to_dir, filename, ignore_missing):
1646         from_path = os.path.join(from_dir, filename)
1647         if ignore_missing and not os.path.exists(from_path):
1648             return
1649         to_path = os.path.join(to_dir, filename)
1650         if not os.path.exists(to_dir):
1651             os.mkdir(to_dir)
1652         shutil.move(from_path, to_path)
1653
1654     if from_dir == to_dir:
1655         return
1656
1657     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1658     _move_file(from_dir, to_dir, apk['apkName'], False)
1659     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1660     for density in all_screen_densities:
1661         from_icon_dir = get_icon_dir(from_dir, density)
1662         to_icon_dir = get_icon_dir(to_dir, density)
1663         if density not in apk.get('icons', []):
1664             continue
1665         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1666     if 'srcname' in apk:
1667         _move_file(from_dir, to_dir, apk['srcname'], False)
1668
1669
1670 def add_apks_to_per_app_repos(repodir, apks):
1671     apks_per_app = dict()
1672     for apk in apks:
1673         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1674         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1675         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1676         apks_per_app[apk['packageName']] = apk
1677
1678         if not os.path.exists(apk['per_app_icons']):
1679             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1680             os.makedirs(apk['per_app_icons'])
1681
1682         apkpath = os.path.join(repodir, apk['apkName'])
1683         shutil.copy(apkpath, apk['per_app_repo'])
1684         apksigpath = apkpath + '.sig'
1685         if os.path.exists(apksigpath):
1686             shutil.copy(apksigpath, apk['per_app_repo'])
1687         apkascpath = apkpath + '.asc'
1688         if os.path.exists(apkascpath):
1689             shutil.copy(apkascpath, apk['per_app_repo'])
1690
1691
1692 def create_metadata_from_template(apk):
1693     '''create a new metadata file using internal or external template
1694
1695     Generate warnings for apk's with no metadata (or create skeleton
1696     metadata files, if requested on the command line).  Though the
1697     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1698     since those impose things on the metadata file made from the
1699     template: field sort order, empty field value, formatting, etc.
1700     '''
1701
1702     import yaml
1703     if os.path.exists('template.yml'):
1704         with open('template.yml') as f:
1705             metatxt = f.read()
1706         if 'name' in apk and apk['name'] != '':
1707             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1708                              r'\1 ' + apk['name'],
1709                              metatxt,
1710                              flags=re.IGNORECASE | re.MULTILINE)
1711         else:
1712             logging.warning(_('{appid} does not have a name! Using package name instead.')
1713                             .format(appid=apk['packageName']))
1714             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1715                              r'\1 ' + apk['packageName'],
1716                              metatxt,
1717                              flags=re.IGNORECASE | re.MULTILINE)
1718         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1719             f.write(metatxt)
1720     else:
1721         app = dict()
1722         app['Categories'] = [os.path.basename(os.getcwd())]
1723         # include some blanks as part of the template
1724         app['AuthorName'] = ''
1725         app['Summary'] = ''
1726         app['WebSite'] = ''
1727         app['IssueTracker'] = ''
1728         app['SourceCode'] = ''
1729         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1730         if 'name' in apk and apk['name'] != '':
1731             app['Name'] = apk['name']
1732         else:
1733             logging.warning(_('{appid} does not have a name! Using package name instead.')
1734                             .format(appid=apk['packageName']))
1735             app['Name'] = apk['packageName']
1736         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1737             yaml.dump(app, f, default_flow_style=False)
1738     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1739
1740
1741 config = None
1742 options = None
1743
1744
1745 def main():
1746
1747     global config, options
1748
1749     # Parse command line...
1750     parser = ArgumentParser()
1751     common.setup_global_opts(parser)
1752     parser.add_argument("--create-key", action="store_true", default=False,
1753                         help=_("Add a repo signing key to an unsigned repo"))
1754     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1755                         help=_("Add skeleton metadata files for APKs that are missing them"))
1756     parser.add_argument("--delete-unknown", action="store_true", default=False,
1757                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1758     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1759                         help=_("Report on build data status"))
1760     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1761                         help=_("Interactively ask about things that need updating."))
1762     parser.add_argument("-I", "--icons", action="store_true", default=False,
1763                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1764     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1765                         help=_("Specify editor to use in interactive mode. Default " +
1766                                "is {path}").format(path='/etc/alternatives/editor'))
1767     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1768                         help=_("Update the wiki"))
1769     parser.add_argument("--pretty", action="store_true", default=False,
1770                         help=_("Produce human-readable XML/JSON for index files"))
1771     parser.add_argument("--clean", action="store_true", default=False,
1772                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1773     parser.add_argument("--nosign", action="store_true", default=False,
1774                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1775     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1776                         help=_("Use date from APK instead of current time for newly added APKs"))
1777     parser.add_argument("--rename-apks", action="store_true", default=False,
1778                         help=_("Rename APK files that do not match package.name_123.apk"))
1779     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1780                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1781     metadata.add_metadata_arguments(parser)
1782     options = parser.parse_args()
1783     metadata.warnings_action = options.W
1784
1785     config = common.read_config(options)
1786
1787     if not ('jarsigner' in config and 'keytool' in config):
1788         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1789
1790     repodirs = ['repo']
1791     if config['archive_older'] != 0:
1792         repodirs.append('archive')
1793         if not os.path.exists('archive'):
1794             os.mkdir('archive')
1795
1796     if options.icons:
1797         resize_all_icons(repodirs)
1798         sys.exit(0)
1799
1800     if options.rename_apks:
1801         options.clean = True
1802
1803     # check that icons exist now, rather than fail at the end of `fdroid update`
1804     for k in ['repo_icon', 'archive_icon']:
1805         if k in config:
1806             if not os.path.exists(config[k]):
1807                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1808                                  .format(name=k, path=config[k]))
1809                 sys.exit(1)
1810
1811     # if the user asks to create a keystore, do it now, reusing whatever it can
1812     if options.create_key:
1813         if os.path.exists(config['keystore']):
1814             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1815             logging.critical("\t'" + config['keystore'] + "'")
1816             sys.exit(1)
1817
1818         if 'repo_keyalias' not in config:
1819             config['repo_keyalias'] = socket.getfqdn()
1820             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1821         if 'keydname' not in config:
1822             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1823             common.write_to_config(config, 'keydname', config['keydname'])
1824         if 'keystore' not in config:
1825             config['keystore'] = common.default_config['keystore']
1826             common.write_to_config(config, 'keystore', config['keystore'])
1827
1828         password = common.genpassword()
1829         if 'keystorepass' not in config:
1830             config['keystorepass'] = password
1831             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1832         if 'keypass' not in config:
1833             config['keypass'] = password
1834             common.write_to_config(config, 'keypass', config['keypass'])
1835         common.genkeystore(config)
1836
1837     # Get all apps...
1838     apps = metadata.read_metadata()
1839
1840     # Generate a list of categories...
1841     categories = set()
1842     for app in apps.values():
1843         categories.update(app.Categories)
1844
1845     # Read known apks data (will be updated and written back when we've finished)
1846     knownapks = common.KnownApks()
1847
1848     # Get APK cache
1849     apkcache = get_cache()
1850
1851     # Delete builds for disabled apps
1852     delete_disabled_builds(apps, apkcache, repodirs)
1853
1854     # Scan all apks in the main repo
1855     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1856
1857     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1858                                            options.use_date_from_apk)
1859     cachechanged = cachechanged or fcachechanged
1860     apks += files
1861     for apk in apks:
1862         if apk['packageName'] not in apps:
1863             if options.create_metadata:
1864                 create_metadata_from_template(apk)
1865                 apps = metadata.read_metadata()
1866             else:
1867                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1868                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1869                 if options.delete_unknown:
1870                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1871                                  .format(apkfilename=apk['apkName']))
1872                     rmf = os.path.join(repodirs[0], apk['apkName'])
1873                     if not os.path.exists(rmf):
1874                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1875                     else:
1876                         os.remove(rmf)
1877                 else:
1878                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1879
1880     copy_triple_t_store_metadata(apps)
1881     insert_obbs(repodirs[0], apps, apks)
1882     insert_localized_app_metadata(apps)
1883     translate_per_build_anti_features(apps, apks)
1884
1885     # Scan the archive repo for apks as well
1886     if len(repodirs) > 1:
1887         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1888         if cc:
1889             cachechanged = True
1890     else:
1891         archapks = []
1892
1893     # Apply information from latest apks to the application and update dates
1894     apply_info_from_latest_apk(apps, apks + archapks)
1895
1896     # Sort the app list by name, then the web site doesn't have to by default.
1897     # (we had to wait until we'd scanned the apks to do this, because mostly the
1898     # name comes from there!)
1899     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1900
1901     # APKs are placed into multiple repos based on the app package, providing
1902     # per-app subscription feeds for nightly builds and things like it
1903     if config['per_app_repos']:
1904         add_apks_to_per_app_repos(repodirs[0], apks)
1905         for appid, app in apps.items():
1906             repodir = os.path.join(appid, 'fdroid', 'repo')
1907             appdict = dict()
1908             appdict[appid] = app
1909             if os.path.isdir(repodir):
1910                 index.make(appdict, [appid], apks, repodir, False)
1911             else:
1912                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1913         return
1914
1915     if len(repodirs) > 1:
1916         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1917
1918     # Make the index for the main repo...
1919     index.make(apps, sortedids, apks, repodirs[0], False)
1920     make_categories_txt(repodirs[0], categories)
1921
1922     # If there's an archive repo,  make the index for it. We already scanned it
1923     # earlier on.
1924     if len(repodirs) > 1:
1925         index.make(apps, sortedids, archapks, repodirs[1], True)
1926
1927     git_remote = config.get('binary_transparency_remote')
1928     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1929         from . import btlog
1930         btlog.make_binary_transparency_log(repodirs)
1931
1932     if config['update_stats']:
1933         # Update known apks info...
1934         knownapks.writeifchanged()
1935
1936         # Generate latest apps data for widget
1937         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1938             data = ''
1939             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1940                 for line in f:
1941                     appid = line.rstrip()
1942                     data += appid + "\t"
1943                     app = apps[appid]
1944                     data += app.Name + "\t"
1945                     if app.icon is not None:
1946                         data += app.icon + "\t"
1947                     data += app.License + "\n"
1948             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1949                 f.write(data)
1950
1951     if cachechanged:
1952         write_cache(apkcache)
1953
1954     # Update the wiki...
1955     if options.wiki:
1956         update_wiki(apps, sortedids, apks + archapks)
1957
1958     logging.info(_("Finished"))
1959
1960
1961 if __name__ == "__main__":
1962     main()