fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import common
  41 from . import index
  42 from . import metadata
  43 from .common import SdkToolsPopen
  44 from .exception import BuildException, FDroidException
  45
  46 METADATA_VERSION = 18
  47
  48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  49 UNSET_VERSION_CODE = -0x100000000
  50
  51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85
  86 def dpi_to_px(density):
  87     return (int(density) * 48) / 160
  88
  89
  90 def px_to_dpi(px):
  91     return (int(px) * 160) / 48
  92
  93
  94 def get_icon_dir(repodir, density):
  95     if density == '0':
  96         return os.path.join(repodir, "icons")
  97     return os.path.join(repodir, "icons-%s" % density)
  98
  99
 100 def get_icon_dirs(repodir):
 101     for density in screen_densities:
 102         yield get_icon_dir(repodir, density)
 103
 104
 105 def get_all_icon_dirs(repodir):
 106     for density in all_screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def update_wiki(apps, sortedids, apks):
 111     """Update the wiki
 112
 113     :param apps: fully populated list of all applications
 114     :param apks: all apks, except...
 115     """
 116     logging.info("Updating wiki")
 117     wikicat = 'Apps'
 118     wikiredircat = 'App Redirects'
 119     import mwclient
 120     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 121                          path=config['wiki_path'])
 122     site.login(config['wiki_user'], config['wiki_password'])
 123     generated_pages = {}
 124     generated_redirects = {}
 125
 126     for appid in sortedids:
 127         app = metadata.App(apps[appid])
 128
 129         wikidata = ''
 130         if app.Disabled:
 131             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 132         if app.AntiFeatures:
 133             for af in app.AntiFeatures:
 134                 wikidata += '{{AntiFeature|' + af + '}}\n'
 135         if app.RequiresRoot:
 136             requiresroot = 'Yes'
 137         else:
 138             requiresroot = 'No'
 139         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 140             appid,
 141             app.Name,
 142             app.added.strftime('%Y-%m-%d') if app.added else '',
 143             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 144             app.SourceCode,
 145             app.IssueTracker,
 146             app.WebSite,
 147             app.Changelog,
 148             app.Donate,
 149             app.FlattrID,
 150             app.Bitcoin,
 151             app.Litecoin,
 152             app.License,
 153             requiresroot,
 154             app.AuthorName,
 155             app.AuthorEmail)
 156
 157         if app.Provides:
 158             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 159
 160         wikidata += app.Summary
 161         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 162
 163         wikidata += "=Description=\n"
 164         wikidata += metadata.description_wiki(app.Description) + "\n"
 165
 166         wikidata += "=Maintainer Notes=\n"
 167         if app.MaintainerNotes:
 168             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 169         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 170
 171         # Get a list of all packages for this application...
 172         apklist = []
 173         gotcurrentver = False
 174         cantupdate = False
 175         buildfails = False
 176         for apk in apks:
 177             if apk['packageName'] == appid:
 178                 if str(apk['versionCode']) == app.CurrentVersionCode:
 179                     gotcurrentver = True
 180                 apklist.append(apk)
 181         # Include ones we can't build, as a special case...
 182         for build in app.builds:
 183             if build.disable:
 184                 if build.versionCode == app.CurrentVersionCode:
 185                     cantupdate = True
 186                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 187                 apklist.append({'versionCode': int(build.versionCode),
 188                                 'versionName': build.versionName,
 189                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 190                                 })
 191             else:
 192                 builtit = False
 193                 for apk in apklist:
 194                     if apk['versionCode'] == int(build.versionCode):
 195                         builtit = True
 196                         break
 197                 if not builtit:
 198                     buildfails = True
 199                     apklist.append({'versionCode': int(build.versionCode),
 200                                     'versionName': build.versionName,
 201                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 202                                     })
 203         if app.CurrentVersionCode == '0':
 204             cantupdate = True
 205         # Sort with most recent first...
 206         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 207
 208         wikidata += "=Versions=\n"
 209         if len(apklist) == 0:
 210             wikidata += "We currently have no versions of this app available."
 211         elif not gotcurrentver:
 212             wikidata += "We don't have the current version of this app."
 213         else:
 214             wikidata += "We have the current version of this app."
 215         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 216         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 217         if len(app.NoSourceSince) > 0:
 218             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 219         if len(app.CurrentVersion) > 0:
 220             wikidata += "The current (recommended) version is " + app.CurrentVersion
 221             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 222         validapks = 0
 223         for apk in apklist:
 224             wikidata += "==" + apk['versionName'] + "==\n"
 225
 226             if 'buildproblem' in apk:
 227                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 228             else:
 229                 validapks += 1
 230                 wikidata += "This version is built and signed by "
 231                 if 'srcname' in apk:
 232                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 233                 else:
 234                     wikidata += "the original developer.\n\n"
 235             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 236
 237         wikidata += '\n[[Category:' + wikicat + ']]\n'
 238         if len(app.NoSourceSince) > 0:
 239             wikidata += '\n[[Category:Apps missing source code]]\n'
 240         if validapks == 0 and not app.Disabled:
 241             wikidata += '\n[[Category:Apps with no packages]]\n'
 242         if cantupdate and not app.Disabled:
 243             wikidata += "\n[[Category:Apps we cannot update]]\n"
 244         if buildfails and not app.Disabled:
 245             wikidata += "\n[[Category:Apps with failing builds]]\n"
 246         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 247             wikidata += '\n[[Category:Apps to Update]]\n'
 248         if app.Disabled:
 249             wikidata += '\n[[Category:Apps that are disabled]]\n'
 250         if app.UpdateCheckMode == 'None' and not app.Disabled:
 251             wikidata += '\n[[Category:Apps with no update check]]\n'
 252         for appcat in app.Categories:
 253             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 254
 255         # We can't have underscores in the page name, even if they're in
 256         # the package ID, because MediaWiki messes with them...
 257         pagename = appid.replace('_', ' ')
 258
 259         # Drop a trailing newline, because mediawiki is going to drop it anyway
 260         # and it we don't we'll think the page has changed when it hasn't...
 261         if wikidata.endswith('\n'):
 262             wikidata = wikidata[:-1]
 263
 264         generated_pages[pagename] = wikidata
 265
 266         # Make a redirect from the name to the ID too, unless there's
 267         # already an existing page with the name and it isn't a redirect.
 268         noclobber = False
 269         apppagename = app.Name.replace('_', ' ')
 270         apppagename = apppagename.replace('{', '')
 271         apppagename = apppagename.replace('}', ' ')
 272         apppagename = apppagename.replace(':', ' ')
 273         apppagename = apppagename.replace('[', ' ')
 274         apppagename = apppagename.replace(']', ' ')
 275         # Drop double spaces caused mostly by replacing ':' above
 276         apppagename = apppagename.replace('  ', ' ')
 277         for expagename in site.allpages(prefix=apppagename,
 278                                         filterredir='nonredirects',
 279                                         generator=False):
 280             if expagename == apppagename:
 281                 noclobber = True
 282         # Another reason not to make the redirect page is if the app name
 283         # is the same as it's ID, because that will overwrite the real page
 284         # with an redirect to itself! (Although it seems like an odd
 285         # scenario this happens a lot, e.g. where there is metadata but no
 286         # builds or binaries to extract a name from.
 287         if apppagename == pagename:
 288             noclobber = True
 289         if not noclobber:
 290             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 291
 292     for tcat, genp in [(wikicat, generated_pages),
 293                        (wikiredircat, generated_redirects)]:
 294         catpages = site.Pages['Category:' + tcat]
 295         existingpages = []
 296         for page in catpages:
 297             existingpages.append(page.name)
 298             if page.name in genp:
 299                 pagetxt = page.edit()
 300                 if pagetxt != genp[page.name]:
 301                     logging.debug("Updating modified page " + page.name)
 302                     page.save(genp[page.name], summary='Auto-updated')
 303                 else:
 304                     logging.debug("Page " + page.name + " is unchanged")
 305             else:
 306                 logging.warn("Deleting page " + page.name)
 307                 page.delete('No longer published')
 308         for pagename, text in genp.items():
 309             logging.debug("Checking " + pagename)
 310             if pagename not in existingpages:
 311                 logging.debug("Creating page " + pagename)
 312                 try:
 313                     newpage = site.Pages[pagename]
 314                     newpage.save(text, summary='Auto-created')
 315                 except Exception as e:
 316                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 317
 318     # Purge server cache to ensure counts are up to date
 319     site.pages['Repository Maintenance'].purge()
 320
 321
 322 def delete_disabled_builds(apps, apkcache, repodirs):
 323     """Delete disabled build outputs.
 324
 325     :param apps: list of all applications, as per metadata.read_metadata
 326     :param apkcache: current apk cache information
 327     :param repodirs: the repo directories to process
 328     """
 329     for appid, app in apps.items():
 330         for build in app['builds']:
 331             if not build.disable:
 332                 continue
 333             apkfilename = common.get_release_filename(app, build)
 334             iconfilename = "%s.%s.png" % (
 335                 appid,
 336                 build.versionCode)
 337             for repodir in repodirs:
 338                 files = [
 339                     os.path.join(repodir, apkfilename),
 340                     os.path.join(repodir, apkfilename + '.asc'),
 341                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 342                 ]
 343                 for density in all_screen_densities:
 344                     repo_dir = get_icon_dir(repodir, density)
 345                     files.append(os.path.join(repo_dir, iconfilename))
 346
 347                 for f in files:
 348                     if os.path.exists(f):
 349                         logging.info("Deleting disabled build output " + f)
 350                         os.remove(f)
 351             if apkfilename in apkcache:
 352                 del apkcache[apkfilename]
 353
 354
 355 def resize_icon(iconpath, density):
 356
 357     if not os.path.isfile(iconpath):
 358         return
 359
 360     fp = None
 361     try:
 362         fp = open(iconpath, 'rb')
 363         im = Image.open(fp)
 364         size = dpi_to_px(density)
 365
 366         if any(length > size for length in im.size):
 367             oldsize = im.size
 368             im.thumbnail((size, size), Image.ANTIALIAS)
 369             logging.debug("%s was too large at %s - new size is %s" % (
 370                 iconpath, oldsize, im.size))
 371             im.save(iconpath, "PNG")
 372
 373     except Exception as e:
 374         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 375
 376     finally:
 377         if fp:
 378             fp.close()
 379
 380
 381 def resize_all_icons(repodirs):
 382     """Resize all icons that exceed the max size
 383
 384     :param repodirs: the repo directories to process
 385     """
 386     for repodir in repodirs:
 387         for density in screen_densities:
 388             icon_dir = get_icon_dir(repodir, density)
 389             icon_glob = os.path.join(icon_dir, '*.png')
 390             for iconpath in glob.glob(icon_glob):
 391                 resize_icon(iconpath, density)
 392
 393
 394 def getsig(apkpath):
 395     """ Get the signing certificate of an apk. To get the same md5 has that
 396     Android gets, we encode the .RSA certificate in a specific format and pass
 397     it hex-encoded to the md5 digest algorithm.
 398
 399     :param apkpath: path to the apk
 400     :returns: A string containing the md5 of the signature of the apk or None
 401               if an error occurred.
 402     """
 403
 404     with zipfile.ZipFile(apkpath, 'r') as apk:
 405         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 406
 407         if len(certs) < 1:
 408             logging.error("Found no signing certificates on %s" % apkpath)
 409             return None
 410         if len(certs) > 1:
 411             logging.error("Found multiple signing certificates on %s" % apkpath)
 412             return None
 413
 414         cert = apk.read(certs[0])
 415
 416     cert_encoded = common.get_certificate(cert)
 417
 418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 419
 420
 421 def get_cache_file():
 422     return os.path.join('tmp', 'apkcache')
 423
 424
 425 def get_cache():
 426     """
 427     Gather information about all the apk files in the repo directory,
 428     using cached data if possible.
 429     :return: apkcache
 430     """
 431     apkcachefile = get_cache_file()
 432     if not options.clean and os.path.exists(apkcachefile):
 433         with open(apkcachefile, 'rb') as cf:
 434             apkcache = pickle.load(cf, encoding='utf-8')
 435         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
 436             apkcache = {}
 437     else:
 438         apkcache = {}
 439
 440     return apkcache
 441
 442
 443 def write_cache(apkcache):
 444     apkcachefile = get_cache_file()
 445     cache_path = os.path.dirname(apkcachefile)
 446     if not os.path.exists(cache_path):
 447         os.makedirs(cache_path)
 448     apkcache["METADATA_VERSION"] = METADATA_VERSION
 449     with open(apkcachefile, 'wb') as cf:
 450         pickle.dump(apkcache, cf)
 451
 452
 453 def get_icon_bytes(apkzip, iconsrc):
 454     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 455     try:
 456         return apkzip.read(iconsrc)
 457     except KeyError:
 458         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 459
 460
 461 def sha256sum(filename):
 462     '''Calculate the sha256 of the given file'''
 463     sha = hashlib.sha256()
 464     with open(filename, 'rb') as f:
 465         while True:
 466             t = f.read(16384)
 467             if len(t) == 0:
 468                 break
 469             sha.update(t)
 470     return sha.hexdigest()
 471
 472
 473 def has_known_vulnerability(filename):
 474     """checks for known vulnerabilities in the APK
 475
 476     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 477     version.  Google also enforces this:
 478     https://support.google.com/faqs/answer/6376725?hl=en
 479
 480     Checks whether there are more than one classes.dex or AndroidManifest.xml
 481     files, which is invalid and an essential part of the "Master Key" attack.
 482
 483     http://www.saurik.com/id/17
 484     """
 485
 486     # statically load this pattern
 487     if not hasattr(has_known_vulnerability, "pattern"):
 488         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 489
 490     files_in_apk = set()
 491     with zipfile.ZipFile(filename) as zf:
 492         for name in zf.namelist():
 493             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 494                 lib = zf.open(name)
 495                 while True:
 496                     chunk = lib.read(4096)
 497                     if chunk == b'':
 498                         break
 499                     m = has_known_vulnerability.pattern.search(chunk)
 500                     if m:
 501                         version = m.group(1).decode('ascii')
 502                         if version.startswith('1.0.1') and version[5] >= 'r' \
 503                            or version.startswith('1.0.2') and version[5] >= 'f':
 504                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 505                         else:
 506                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 507                             return True
 508                         break
 509             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 510                 if name in files_in_apk:
 511                     return True
 512                 files_in_apk.add(name)
 513
 514     return False
 515
 516
 517 def insert_obbs(repodir, apps, apks):
 518     """Scans the .obb files in a given repo directory and adds them to the
 519     relevant APK instances.  OBB files have versionCodes like APK
 520     files, and they are loosely associated.  If there is an OBB file
 521     present, then any APK with the same or higher versionCode will use
 522     that OBB file.  There are two OBB types: main and patch, each APK
 523     can only have only have one of each.
 524
 525     https://developer.android.com/google/play/expansion-files.html
 526
 527     :param repodir: repo directory to scan
 528     :param apps: list of current, valid apps
 529     :param apks: current information on all APKs
 530
 531     """
 532
 533     def obbWarnDelete(f, msg):
 534         logging.warning(msg + f)
 535         if options.delete_unknown:
 536             logging.error("Deleting unknown file: " + f)
 537             os.remove(f)
 538
 539     obbs = []
 540     java_Integer_MIN_VALUE = -pow(2, 31)
 541     currentPackageNames = apps.keys()
 542     for f in glob.glob(os.path.join(repodir, '*.obb')):
 543         obbfile = os.path.basename(f)
 544         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 545         chunks = obbfile.split('.')
 546         if chunks[0] != 'main' and chunks[0] != 'patch':
 547             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 548             continue
 549         if not re.match(r'^-?[0-9]+$', chunks[1]):
 550             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 551             continue
 552         versionCode = int(chunks[1])
 553         packagename = ".".join(chunks[2:-1])
 554
 555         highestVersionCode = java_Integer_MIN_VALUE
 556         if packagename not in currentPackageNames:
 557             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 558             continue
 559         for apk in apks:
 560             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 561                 highestVersionCode = apk['versionCode']
 562         if versionCode > highestVersionCode:
 563             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 564                           + ') than any APK: ')
 565             continue
 566         obbsha256 = sha256sum(f)
 567         obbs.append((packagename, versionCode, obbfile, obbsha256))
 568
 569     for apk in apks:
 570         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 571             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 572                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 573                     apk['obbMainFile'] = obbfile
 574                     apk['obbMainFileSha256'] = obbsha256
 575                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 576                     apk['obbPatchFile'] = obbfile
 577                     apk['obbPatchFileSha256'] = obbsha256
 578             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 579                 break
 580
 581
 582 def _get_localized_dict(app, locale):
 583     '''get the dict to add localized store metadata to'''
 584     if 'localized' not in app:
 585         app['localized'] = collections.OrderedDict()
 586     if locale not in app['localized']:
 587         app['localized'][locale] = collections.OrderedDict()
 588     return app['localized'][locale]
 589
 590
 591 def _set_localized_text_entry(app, locale, key, f):
 592     limit = config['char_limits'][key]
 593     localized = _get_localized_dict(app, locale)
 594     with open(f) as fp:
 595         text = fp.read()[:limit]
 596         if len(text) > 0:
 597             localized[key] = text
 598
 599
 600 def _set_author_entry(app, key, f):
 601     limit = config['char_limits']['author']
 602     with open(f) as fp:
 603         text = fp.read()[:limit]
 604         if len(text) > 0:
 605             app[key] = text
 606
 607
 608 def copy_triple_t_store_metadata(apps):
 609     """Include store metadata from the app's source repo
 610
 611     The Triple-T Gradle Play Publisher is a plugin that has a standard
 612     file layout for all of the metadata and graphics that the Google
 613     Play Store accepts.  Since F-Droid has the git repo, it can just
 614     pluck those files directly.  This method reads any text files into
 615     the app dict, then copies any graphics into the fdroid repo
 616     directory structure.
 617
 618     This needs to be run before insert_localized_app_metadata() so that
 619     the graphics files that are copied into the fdroid repo get
 620     properly indexed.
 621
 622     https://github.com/Triple-T/gradle-play-publisher#upload-images
 623     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 624
 625     """
 626
 627     if not os.path.isdir('build'):
 628         return  # nothing to do
 629
 630     for packageName, app in apps.items():
 631         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 632             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 633             for root, dirs, files in os.walk(d):
 634                 segments = root.split('/')
 635                 locale = segments[-2]
 636                 for f in files:
 637                     if f == 'fulldescription':
 638                         _set_localized_text_entry(app, locale, 'description',
 639                                                   os.path.join(root, f))
 640                         continue
 641                     elif f == 'shortdescription':
 642                         _set_localized_text_entry(app, locale, 'summary',
 643                                                   os.path.join(root, f))
 644                         continue
 645                     elif f == 'title':
 646                         _set_localized_text_entry(app, locale, 'name',
 647                                                   os.path.join(root, f))
 648                         continue
 649                     elif f == 'video':
 650                         _set_localized_text_entry(app, locale, 'video',
 651                                                   os.path.join(root, f))
 652                         continue
 653                     elif f == 'whatsnew':
 654                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 655                                                   os.path.join(root, f))
 656                         continue
 657                     elif f == 'contactEmail':
 658                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 659                         continue
 660                     elif f == 'contactPhone':
 661                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 662                         continue
 663                     elif f == 'contactWebsite':
 664                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 665                         continue
 666
 667                     base, extension = common.get_extension(f)
 668                     dirname = os.path.basename(root)
 669                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 670                         if segments[-2] == 'listing':
 671                             locale = segments[-3]
 672                         else:
 673                             locale = segments[-2]
 674                         destdir = os.path.join('repo', packageName, locale)
 675                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 676                         sourcefile = os.path.join(root, f)
 677                         destfile = os.path.join(destdir, dirname + '.' + extension)
 678                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 679                         shutil.copy(sourcefile, destfile)
 680
 681
 682 def insert_localized_app_metadata(apps):
 683     """scans standard locations for graphics and localized text
 684
 685     Scans for localized description files, store graphics, and
 686     screenshot PNG files in statically defined screenshots directory
 687     and adds them to the app metadata.  The screenshots and graphic
 688     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 689     and must be in the following layout:
 690     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 691
 692     repo/packageName/locale/featureGraphic.png
 693     repo/packageName/locale/phoneScreenshots/1.png
 694     repo/packageName/locale/phoneScreenshots/2.png
 695
 696     The changelog files must be text files named with the versionCode
 697     ending with ".txt" and must be in the following layout:
 698     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 699
 700     repo/packageName/locale/changelogs/12345.txt
 701
 702     This will scan the each app's source repo then the metadata/ dir
 703     for these standard locations of changelog files.  If it finds
 704     them, they will be added to the dict of all packages, with the
 705     versions in the metadata/ folder taking precendence over the what
 706     is in the app's source repo.
 707
 708     Where "packageName" is the app's packageName and "locale" is the locale
 709     of the graphics, e.g. what language they are in, using the IETF RFC5646
 710     format (en-US, fr-CA, es-MX, etc).
 711
 712     This will also scan the app's git for a fastlane folder, and the
 713     metadata/ folder and the apps' source repos for standard locations
 714     of graphic and screenshot files.  If it finds them, it will copy
 715     them into the repo.  The fastlane files follow this pattern:
 716     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 717
 718     """
 719
 720     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 721     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 722     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 723
 724     for srcd in sorted(sourcedirs):
 725         if not os.path.isdir(srcd):
 726             continue
 727         for root, dirs, files in os.walk(srcd):
 728             segments = root.split('/')
 729             packageName = segments[1]
 730             if packageName not in apps:
 731                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 732                 continue
 733             locale = segments[-1]
 734             destdir = os.path.join('repo', packageName, locale)
 735             for f in files:
 736                 if f in ('description.txt', 'full_description.txt'):
 737                     _set_localized_text_entry(apps[packageName], locale, 'description',
 738                                               os.path.join(root, f))
 739                     continue
 740                 elif f in ('summary.txt', 'short_description.txt'):
 741                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 742                                               os.path.join(root, f))
 743                     continue
 744                 elif f in ('name.txt', 'title.txt'):
 745                     _set_localized_text_entry(apps[packageName], locale, 'name',
 746                                               os.path.join(root, f))
 747                     continue
 748                 elif f == 'video.txt':
 749                     _set_localized_text_entry(apps[packageName], locale, 'video',
 750                                               os.path.join(root, f))
 751                     continue
 752                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 753                     locale = segments[-2]
 754                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 755                                               os.path.join(root, f))
 756                     continue
 757
 758                 base, extension = common.get_extension(f)
 759                 if locale == 'images':
 760                     locale = segments[-2]
 761                     destdir = os.path.join('repo', packageName, locale)
 762                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 763                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 764                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 765                     shutil.copy(os.path.join(root, f), destdir)
 766             for d in dirs:
 767                 if d in SCREENSHOT_DIRS:
 768                     for f in glob.glob(os.path.join(root, d, '*.*')):
 769                         _, extension = common.get_extension(f)
 770                         if extension in ALLOWED_EXTENSIONS:
 771                             screenshotdestdir = os.path.join(destdir, d)
 772                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 773                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 774                             shutil.copy(f, screenshotdestdir)
 775
 776     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 777     for d in repofiles:
 778         if not os.path.isdir(d):
 779             continue
 780         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 781             if not os.path.isfile(f):
 782                 continue
 783             segments = f.split('/')
 784             packageName = segments[1]
 785             locale = segments[2]
 786             screenshotdir = segments[3]
 787             filename = os.path.basename(f)
 788             base, extension = common.get_extension(filename)
 789
 790             if packageName not in apps:
 791                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 792                                 % (filename, packageName))
 793                 continue
 794             graphics = _get_localized_dict(apps[packageName], locale)
 795
 796             if extension not in ALLOWED_EXTENSIONS:
 797                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 798             elif base in GRAPHIC_NAMES:
 799                 # there can only be zero or one of these per locale
 800                 graphics[base] = filename
 801             elif screenshotdir in SCREENSHOT_DIRS:
 802                 # there can any number of these per locale
 803                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 804                 if screenshotdir not in graphics:
 805                     graphics[screenshotdir] = []
 806                 graphics[screenshotdir].append(filename)
 807             else:
 808                 logging.warning('Unsupported graphics file found: ' + f)
 809
 810
 811 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 812     """Scan a repo for all files with an extension except APK/OBB
 813
 814     :param apkcache: current cached info about all repo files
 815     :param repodir: repo directory to scan
 816     :param knownapks: list of all known files, as per metadata.read_metadata
 817     :param use_date_from_file: use date from file (instead of current date)
 818                                for newly added files
 819     """
 820
 821     cachechanged = False
 822     repo_files = []
 823     repodir = repodir.encode('utf-8')
 824     for name in os.listdir(repodir):
 825         file_extension = common.get_file_extension(name)
 826         if file_extension == 'apk' or file_extension == 'obb':
 827             continue
 828         filename = os.path.join(repodir, name)
 829         name_utf8 = name.decode('utf-8')
 830         if filename.endswith(b'_src.tar.gz'):
 831             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 832             continue
 833         if not common.is_repo_file(filename):
 834             continue
 835         stat = os.stat(filename)
 836         if stat.st_size == 0:
 837             raise FDroidException(filename + ' is zero size!')
 838
 839         shasum = sha256sum(filename)
 840         usecache = False
 841         if name in apkcache:
 842             repo_file = apkcache[name]
 843             # added time is cached as tuple but used here as datetime instance
 844             if 'added' in repo_file:
 845                 a = repo_file['added']
 846                 if isinstance(a, datetime):
 847                     repo_file['added'] = a
 848                 else:
 849                     repo_file['added'] = datetime(*a[:6])
 850             if repo_file.get('hash') == shasum:
 851                 logging.debug("Reading " + name_utf8 + " from cache")
 852                 usecache = True
 853             else:
 854                 logging.debug("Ignoring stale cache data for " + name)
 855
 856         if not usecache:
 857             logging.debug("Processing " + name_utf8)
 858             repo_file = collections.OrderedDict()
 859             repo_file['name'] = os.path.splitext(name_utf8)[0]
 860             # TODO rename apkname globally to something more generic
 861             repo_file['apkName'] = name_utf8
 862             repo_file['hash'] = shasum
 863             repo_file['hashType'] = 'sha256'
 864             repo_file['versionCode'] = 0
 865             repo_file['versionName'] = shasum
 866             # the static ID is the SHA256 unless it is set in the metadata
 867             repo_file['packageName'] = shasum
 868
 869             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 870             if m:
 871                 repo_file['packageName'] = m.group(1)
 872                 repo_file['versionCode'] = int(m.group(2))
 873             srcfilename = name + b'_src.tar.gz'
 874             if os.path.exists(os.path.join(repodir, srcfilename)):
 875                 repo_file['srcname'] = srcfilename.decode('utf-8')
 876             repo_file['size'] = stat.st_size
 877
 878             apkcache[name] = repo_file
 879             cachechanged = True
 880
 881         if use_date_from_file:
 882             timestamp = stat.st_ctime
 883             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 884         else:
 885             default_date_param = None
 886
 887         # Record in knownapks, getting the added date at the same time..
 888         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 889                                     default_date=default_date_param)
 890         if added:
 891             repo_file['added'] = added
 892
 893         repo_files.append(repo_file)
 894
 895     return repo_files, cachechanged
 896
 897
 898 def scan_apk_aapt(apk, apkfile):
 899     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 900     if p.returncode != 0:
 901         if options.delete_unknown:
 902             if os.path.exists(apkfile):
 903                 logging.error("Failed to get apk information, deleting " + apkfile)
 904                 os.remove(apkfile)
 905             else:
 906                 logging.error("Could not find {0} to remove it".format(apkfile))
 907         else:
 908             logging.error("Failed to get apk information, skipping " + apkfile)
 909         raise BuildException("Invalid APK")
 910     for line in p.output.splitlines():
 911         if line.startswith("package:"):
 912             try:
 913                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 914                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 915                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 916             except Exception as e:
 917                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
 918         elif line.startswith("application:"):
 919             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 920             # Keep path to non-dpi icon in case we need it
 921             match = re.match(APK_ICON_PAT_NODPI, line)
 922             if match:
 923                 apk['icons_src']['-1'] = match.group(1)
 924         elif line.startswith("launchable-activity:"):
 925             # Only use launchable-activity as fallback to application
 926             if not apk['name']:
 927                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 928             if '-1' not in apk['icons_src']:
 929                 match = re.match(APK_ICON_PAT_NODPI, line)
 930                 if match:
 931                     apk['icons_src']['-1'] = match.group(1)
 932         elif line.startswith("application-icon-"):
 933             match = re.match(APK_ICON_PAT, line)
 934             if match:
 935                 density = match.group(1)
 936                 path = match.group(2)
 937                 apk['icons_src'][density] = path
 938         elif line.startswith("sdkVersion:"):
 939             m = re.match(APK_SDK_VERSION_PAT, line)
 940             if m is None:
 941                 logging.error(line.replace('sdkVersion:', '')
 942                               + ' is not a valid minSdkVersion!')
 943             else:
 944                 apk['minSdkVersion'] = m.group(1)
 945                 # if target not set, default to min
 946                 if 'targetSdkVersion' not in apk:
 947                     apk['targetSdkVersion'] = m.group(1)
 948         elif line.startswith("targetSdkVersion:"):
 949             m = re.match(APK_SDK_VERSION_PAT, line)
 950             if m is None:
 951                 logging.error(line.replace('targetSdkVersion:', '')
 952                               + ' is not a valid targetSdkVersion!')
 953             else:
 954                 apk['targetSdkVersion'] = m.group(1)
 955         elif line.startswith("maxSdkVersion:"):
 956             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 957         elif line.startswith("native-code:"):
 958             apk['nativecode'] = []
 959             for arch in line[13:].split(' '):
 960                 apk['nativecode'].append(arch[1:-1])
 961         elif line.startswith('uses-permission:'):
 962             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 963             if perm_match['maxSdkVersion']:
 964                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 965             permission = UsesPermission(
 966                 perm_match['name'],
 967                 perm_match['maxSdkVersion']
 968             )
 969
 970             apk['uses-permission'].append(permission)
 971         elif line.startswith('uses-permission-sdk-23:'):
 972             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 973             if perm_match['maxSdkVersion']:
 974                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 975             permission_sdk_23 = UsesPermissionSdk23(
 976                 perm_match['name'],
 977                 perm_match['maxSdkVersion']
 978             )
 979
 980             apk['uses-permission-sdk-23'].append(permission_sdk_23)
 981
 982         elif line.startswith('uses-feature:'):
 983             feature = re.match(APK_FEATURE_PAT, line).group(1)
 984             # Filter out this, it's only added with the latest SDK tools and
 985             # causes problems for lots of apps.
 986             if feature != "android.hardware.screen.portrait" \
 987                     and feature != "android.hardware.screen.landscape":
 988                 if feature.startswith("android.feature."):
 989                     feature = feature[16:]
 990                 apk['features'].add(feature)
 991
 992
 993 def scan_apk_androguard(apk, apkfile):
 994     try:
 995         from androguard.core.bytecodes.apk import APK
 996         apkobject = APK(apkfile)
 997         if apkobject.is_valid_APK():
 998             arsc = apkobject.get_android_resources()
 999         else:
1000             if options.delete_unknown:
1001                 if os.path.exists(apkfile):
1002                     logging.error("Failed to get apk information, deleting " + apkfile)
1003                     os.remove(apkfile)
1004                 else:
1005                     logging.error("Could not find {0} to remove it".format(apkfile))
1006             else:
1007                 logging.error("Failed to get apk information, skipping " + apkfile)
1008             raise BuildException("Invaild APK")
1009     except ImportError:
1010         raise FDroidException("androguard library is not installed and aapt not present")
1011     except FileNotFoundError:
1012         logging.error("Could not open apk file for analysis")
1013         raise BuildException("Invalid APK")
1014
1015     apk['packageName'] = apkobject.get_package()
1016     apk['versionCode'] = int(apkobject.get_androidversion_code())
1017     apk['versionName'] = apkobject.get_androidversion_name()
1018     if apk['versionName'][0] == "@":
1019         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1020         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1021         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1022     apk['name'] = apkobject.get_app_name()
1023
1024     if apkobject.get_max_sdk_version() is not None:
1025         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1026     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1027     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1028
1029     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1030     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1031
1032     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1033
1034     for file in apkobject.get_files():
1035         d_re = density_re.match(file)
1036         if d_re:
1037             folder = d_re.group(1).split('-')
1038             if len(folder) > 1:
1039                 resolution = folder[1]
1040             else:
1041                 resolution = 'mdpi'
1042             density = screen_resolutions[resolution]
1043             apk['icons_src'][density] = d_re.group(0)
1044
1045     if apk['icons_src'].get('-1') is None:
1046         apk['icons_src']['-1'] = apk['icons_src']['160']
1047
1048     arch_re = re.compile("^lib/(.*)/.*$")
1049     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1050     if len(arch) >= 1:
1051         apk['nativecode'] = []
1052         apk['nativecode'].extend(sorted(list(arch)))
1053
1054     xml = apkobject.get_android_manifest_xml()
1055
1056     for item in xml.getElementsByTagName('uses-permission'):
1057         name = str(item.getAttribute("android:name"))
1058         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1059         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1060         permission = UsesPermission(
1061             name,
1062             maxSdkVersion
1063         )
1064         apk['uses-permission'].append(permission)
1065
1066     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1067         name = str(item.getAttribute("android:name"))
1068         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1069         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1070         permission_sdk_23 = UsesPermissionSdk23(
1071             name,
1072             maxSdkVersion
1073         )
1074         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1075
1076     for item in xml.getElementsByTagName('uses-feature'):
1077         feature = str(item.getAttribute("android:name"))
1078         if feature != "android.hardware.screen.portrait" \
1079                 and feature != "android.hardware.screen.landscape":
1080             if feature.startswith("android.feature."):
1081                 feature = feature[16:]
1082         apk['features'].append(feature)
1083
1084
1085 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
1086     """Scan the apk with the given filename in the given repo directory.
1087
1088     This also extracts the icons.
1089
1090     :param apkcache: current apk cache information
1091     :param apkfilename: the filename of the apk to scan
1092     :param repodir: repo directory to scan
1093     :param knownapks: known apks info
1094     :param use_date_from_apk: use date from APK (instead of current date)
1095                               for newly added APKs
1096     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1097      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1098     """
1099
1100     if ' ' in apkfilename:
1101         if options.rename_apks:
1102             newfilename = apkfilename.replace(' ', '_')
1103             os.rename(os.path.join(repodir, apkfilename),
1104                       os.path.join(repodir, newfilename))
1105             apkfilename = newfilename
1106         else:
1107             logging.critical("Spaces in filenames are not allowed.")
1108             return True, None, False
1109
1110     apkfile = os.path.join(repodir, apkfilename)
1111     shasum = sha256sum(apkfile)
1112
1113     cachechanged = False
1114     usecache = False
1115     if apkfilename in apkcache:
1116         apk = apkcache[apkfilename]
1117         if apk.get('hash') == shasum:
1118             logging.debug("Reading " + apkfilename + " from cache")
1119             usecache = True
1120         else:
1121             logging.debug("Ignoring stale cache data for " + apkfilename)
1122
1123     if not usecache:
1124         logging.debug("Processing " + apkfilename)
1125         apk = {}
1126         apk['hash'] = shasum
1127         apk['hashType'] = 'sha256'
1128         apk['uses-permission'] = []
1129         apk['uses-permission-sdk-23'] = []
1130         apk['features'] = []
1131         apk['icons_src'] = {}
1132         apk['icons'] = {}
1133         apk['antiFeatures'] = set()
1134
1135         try:
1136             if SdkToolsPopen(['aapt', 'version'], output=False):
1137                 scan_apk_aapt(apk, apkfile)
1138             else:
1139                 scan_apk_androguard(apk, apkfile)
1140         except BuildException:
1141             return True, None, False
1142
1143         if 'minSdkVersion' not in apk:
1144             logging.warn("No SDK version information found in {0}".format(apkfile))
1145             apk['minSdkVersion'] = 1
1146
1147         # Check for debuggable apks...
1148         if common.isApkAndDebuggable(apkfile):
1149             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1150
1151         # Get the signature (or md5 of, to be precise)...
1152         logging.debug('Getting signature of {0}'.format(apkfile))
1153         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1154         if not apk['sig']:
1155             logging.critical("Failed to get apk signature")
1156             return True, None, False
1157
1158         if options.rename_apks:
1159             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1160             std_short_name = os.path.join(repodir, n)
1161             if apkfile != std_short_name:
1162                 if os.path.exists(std_short_name):
1163                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1164                     if apkfile != std_long_name:
1165                         if os.path.exists(std_long_name):
1166                             dupdir = os.path.join('duplicates', repodir)
1167                             if not os.path.isdir(dupdir):
1168                                 os.makedirs(dupdir, exist_ok=True)
1169                             dupfile = os.path.join('duplicates', std_long_name)
1170                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1171                             os.rename(apkfile, dupfile)
1172                             return True, None, False
1173                         else:
1174                             os.rename(apkfile, std_long_name)
1175                     apkfile = std_long_name
1176                 else:
1177                     os.rename(apkfile, std_short_name)
1178                     apkfile = std_short_name
1179                 apkfilename = apkfile[len(repodir) + 1:]
1180
1181         apk['apkName'] = apkfilename
1182         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1183         if os.path.exists(os.path.join(repodir, srcfilename)):
1184             apk['srcname'] = srcfilename
1185         apk['size'] = os.path.getsize(apkfile)
1186
1187         # verify the jar signature is correct, allow deprecated
1188         # algorithms only if the APK is in the archive.
1189         if not common.verify_apk_signature(apkfile):
1190             if repodir == 'archive':
1191                 if common.verify_old_apk_signature(apkfile):
1192                     apk['antiFeatures'].add('KnownVuln')
1193                 else:
1194                     return True, None, False
1195             else:
1196                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1197                 move_apk_between_sections('repo', 'archive', apk)
1198                 return True, None, False
1199
1200         if has_known_vulnerability(apkfile):
1201             apk['antiFeatures'].add('KnownVuln')
1202
1203         apkzip = zipfile.ZipFile(apkfile, 'r')
1204
1205         # if an APK has files newer than the system time, suggest updating
1206         # the system clock.  This is useful for offline systems, used for
1207         # signing, which do not have another source of clock sync info. It
1208         # has to be more than 24 hours newer because ZIP/APK files do not
1209         # store timezone info
1210         manifest = apkzip.getinfo('AndroidManifest.xml')
1211         if manifest.date_time[1] == 0:  # month can't be zero
1212             logging.debug('AndroidManifest.xml has no date')
1213         else:
1214             dt_obj = datetime(*manifest.date_time)
1215             checkdt = dt_obj - timedelta(1)
1216             if datetime.today() < checkdt:
1217                 logging.warn('System clock is older than manifest in: '
1218                              + apkfilename
1219                              + '\nSet clock to that time using:\n'
1220                              + 'sudo date -s "' + str(dt_obj) + '"')
1221
1222         iconfilename = "%s.%s.png" % (
1223             apk['packageName'],
1224             apk['versionCode'])
1225
1226         # Extract the icon file...
1227         empty_densities = []
1228         for density in screen_densities:
1229             if density not in apk['icons_src']:
1230                 empty_densities.append(density)
1231                 continue
1232             iconsrc = apk['icons_src'][density]
1233             icon_dir = get_icon_dir(repodir, density)
1234             icondest = os.path.join(icon_dir, iconfilename)
1235
1236             try:
1237                 with open(icondest, 'wb') as f:
1238                     f.write(get_icon_bytes(apkzip, iconsrc))
1239                 apk['icons'][density] = iconfilename
1240             except (zipfile.BadZipFile, ValueError, KeyError) as e:
1241                 logging.warning("Error retrieving icon file: %s" % (icondest))
1242                 del apk['icons_src'][density]
1243                 empty_densities.append(density)
1244
1245         if '-1' in apk['icons_src']:
1246             iconsrc = apk['icons_src']['-1']
1247             iconpath = os.path.join(
1248                 get_icon_dir(repodir, '0'), iconfilename)
1249             with open(iconpath, 'wb') as f:
1250                 f.write(get_icon_bytes(apkzip, iconsrc))
1251             try:
1252                 im = Image.open(iconpath)
1253                 dpi = px_to_dpi(im.size[0])
1254                 for density in screen_densities:
1255                     if density in apk['icons']:
1256                         break
1257                     if density == screen_densities[-1] or dpi >= int(density):
1258                         apk['icons'][density] = iconfilename
1259                         shutil.move(iconpath,
1260                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
1261                         empty_densities.remove(density)
1262                         break
1263             except Exception as e:
1264                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1265
1266         if apk['icons']:
1267             apk['icon'] = iconfilename
1268
1269         apkzip.close()
1270
1271         # First try resizing down to not lose quality
1272         last_density = None
1273         for density in screen_densities:
1274             if density not in empty_densities:
1275                 last_density = density
1276                 continue
1277             if last_density is None:
1278                 continue
1279             logging.debug("Density %s not available, resizing down from %s"
1280                           % (density, last_density))
1281
1282             last_iconpath = os.path.join(
1283                 get_icon_dir(repodir, last_density), iconfilename)
1284             iconpath = os.path.join(
1285                 get_icon_dir(repodir, density), iconfilename)
1286             fp = None
1287             try:
1288                 fp = open(last_iconpath, 'rb')
1289                 im = Image.open(fp)
1290
1291                 size = dpi_to_px(density)
1292
1293                 im.thumbnail((size, size), Image.ANTIALIAS)
1294                 im.save(iconpath, "PNG")
1295                 empty_densities.remove(density)
1296             except Exception as e:
1297                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1298             finally:
1299                 if fp:
1300                     fp.close()
1301
1302         # Then just copy from the highest resolution available
1303         last_density = None
1304         for density in reversed(screen_densities):
1305             if density not in empty_densities:
1306                 last_density = density
1307                 continue
1308             if last_density is None:
1309                 continue
1310             logging.debug("Density %s not available, copying from lower density %s"
1311                           % (density, last_density))
1312
1313             shutil.copyfile(
1314                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1315                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1316
1317             empty_densities.remove(density)
1318
1319         for density in screen_densities:
1320             icon_dir = get_icon_dir(repodir, density)
1321             icondest = os.path.join(icon_dir, iconfilename)
1322             resize_icon(icondest, density)
1323
1324         # Copy from icons-mdpi to icons since mdpi is the baseline density
1325         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1326         if os.path.isfile(baseline):
1327             apk['icons']['0'] = iconfilename
1328             shutil.copyfile(baseline,
1329                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1330
1331         if use_date_from_apk and manifest.date_time[1] != 0:
1332             default_date_param = datetime(*manifest.date_time)
1333         else:
1334             default_date_param = None
1335
1336         # Record in known apks, getting the added date at the same time..
1337         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1338                                     default_date=default_date_param)
1339         if added:
1340             apk['added'] = added
1341
1342         apkcache[apkfilename] = apk
1343         cachechanged = True
1344
1345     return False, apk, cachechanged
1346
1347
1348 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1349     """Scan the apks in the given repo directory.
1350
1351     This also extracts the icons.
1352
1353     :param apkcache: current apk cache information
1354     :param repodir: repo directory to scan
1355     :param knownapks: known apks info
1356     :param use_date_from_apk: use date from APK (instead of current date)
1357                               for newly added APKs
1358     :returns: (apks, cachechanged) where apks is a list of apk information,
1359               and cachechanged is True if the apkcache got changed.
1360     """
1361
1362     cachechanged = False
1363
1364     for icon_dir in get_all_icon_dirs(repodir):
1365         if os.path.exists(icon_dir):
1366             if options.clean:
1367                 shutil.rmtree(icon_dir)
1368                 os.makedirs(icon_dir)
1369         else:
1370             os.makedirs(icon_dir)
1371
1372     apks = []
1373     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1374         apkfilename = apkfile[len(repodir) + 1:]
1375         (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1376         if skip:
1377             continue
1378         apks.append(apk)
1379
1380     return apks, cachechanged
1381
1382
1383 def apply_info_from_latest_apk(apps, apks):
1384     """
1385     Some information from the apks needs to be applied up to the application level.
1386     When doing this, we use the info from the most recent version's apk.
1387     We deal with figuring out when the app was added and last updated at the same time.
1388     """
1389     for appid, app in apps.items():
1390         bestver = UNSET_VERSION_CODE
1391         for apk in apks:
1392             if apk['packageName'] == appid:
1393                 if apk['versionCode'] > bestver:
1394                     bestver = apk['versionCode']
1395                     bestapk = apk
1396
1397                 if 'added' in apk:
1398                     if not app.added or apk['added'] < app.added:
1399                         app.added = apk['added']
1400                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1401                         app.lastUpdated = apk['added']
1402
1403         if not app.added:
1404             logging.debug("Don't know when " + appid + " was added")
1405         if not app.lastUpdated:
1406             logging.debug("Don't know when " + appid + " was last updated")
1407
1408         if bestver == UNSET_VERSION_CODE:
1409
1410             if app.Name is None:
1411                 app.Name = app.AutoName or appid
1412             app.icon = None
1413             logging.debug("Application " + appid + " has no packages")
1414         else:
1415             if app.Name is None:
1416                 app.Name = bestapk['name']
1417             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1418             if app.CurrentVersionCode is None:
1419                 app.CurrentVersionCode = str(bestver)
1420
1421
1422 def make_categories_txt(repodir, categories):
1423     '''Write a category list in the repo to allow quick access'''
1424     catdata = ''
1425     for cat in sorted(categories):
1426         catdata += cat + '\n'
1427     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1428         f.write(catdata)
1429
1430
1431 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1432
1433     def filter_apk_list_sorted(apk_list):
1434         res = []
1435         for apk in apk_list:
1436             if apk['packageName'] == appid:
1437                 res.append(apk)
1438
1439         # Sort the apk list by version code. First is highest/newest.
1440         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1441
1442     for appid, app in apps.items():
1443
1444         if app.ArchivePolicy:
1445             keepversions = int(app.ArchivePolicy[:-9])
1446         else:
1447             keepversions = defaultkeepversions
1448
1449         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1450                       .format(appid, len(apks), keepversions, len(archapks)))
1451
1452         current_app_apks = filter_apk_list_sorted(apks)
1453         if len(current_app_apks) > keepversions:
1454             # Move back the ones we don't want.
1455             for apk in current_app_apks[keepversions:]:
1456                 move_apk_between_sections(repodir, archivedir, apk)
1457                 archapks.append(apk)
1458                 apks.remove(apk)
1459
1460         current_app_archapks = filter_apk_list_sorted(archapks)
1461         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1462             required = keepversions - len(apks)
1463             # Move forward the ones we want again.
1464             for apk in current_app_archapks[:required]:
1465                 move_apk_between_sections(archivedir, repodir, apk)
1466                 archapks.remove(apk)
1467                 apks.append(apk)
1468
1469
1470 def move_apk_between_sections(from_dir, to_dir, apk):
1471     """move an APK from repo to archive or vice versa"""
1472
1473     def _move_file(from_dir, to_dir, filename, ignore_missing):
1474         from_path = os.path.join(from_dir, filename)
1475         if ignore_missing and not os.path.exists(from_path):
1476             return
1477         to_path = os.path.join(to_dir, filename)
1478         shutil.move(from_path, to_path)
1479
1480     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1481     _move_file(from_dir, to_dir, apk['apkName'], False)
1482     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1483     for density in all_screen_densities:
1484         from_icon_dir = get_icon_dir(from_dir, density)
1485         to_icon_dir = get_icon_dir(to_dir, density)
1486         if density not in apk['icons']:
1487             continue
1488         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1489     if 'srcname' in apk:
1490         _move_file(from_dir, to_dir, apk['srcname'], False)
1491
1492
1493 def add_apks_to_per_app_repos(repodir, apks):
1494     apks_per_app = dict()
1495     for apk in apks:
1496         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1497         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1498         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1499         apks_per_app[apk['packageName']] = apk
1500
1501         if not os.path.exists(apk['per_app_icons']):
1502             logging.info('Adding new repo for only ' + apk['packageName'])
1503             os.makedirs(apk['per_app_icons'])
1504
1505         apkpath = os.path.join(repodir, apk['apkName'])
1506         shutil.copy(apkpath, apk['per_app_repo'])
1507         apksigpath = apkpath + '.sig'
1508         if os.path.exists(apksigpath):
1509             shutil.copy(apksigpath, apk['per_app_repo'])
1510         apkascpath = apkpath + '.asc'
1511         if os.path.exists(apkascpath):
1512             shutil.copy(apkascpath, apk['per_app_repo'])
1513
1514
1515 config = None
1516 options = None
1517
1518
1519 def main():
1520
1521     global config, options
1522
1523     # Parse command line...
1524     parser = ArgumentParser()
1525     common.setup_global_opts(parser)
1526     parser.add_argument("--create-key", action="store_true", default=False,
1527                         help="Create a repo signing key in a keystore")
1528     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1529                         help="Create skeleton metadata files that are missing")
1530     parser.add_argument("--delete-unknown", action="store_true", default=False,
1531                         help="Delete APKs and/or OBBs without metadata from the repo")
1532     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1533                         help="Report on build data status")
1534     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1535                         help="Interactively ask about things that need updating.")
1536     parser.add_argument("-I", "--icons", action="store_true", default=False,
1537                         help="Resize all the icons exceeding the max pixel size and exit")
1538     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1539                         help="Specify editor to use in interactive mode. Default " +
1540                         "is /etc/alternatives/editor")
1541     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1542                         help="Update the wiki")
1543     parser.add_argument("--pretty", action="store_true", default=False,
1544                         help="Produce human-readable index.xml")
1545     parser.add_argument("--clean", action="store_true", default=False,
1546                         help="Clean update - don't uses caches, reprocess all apks")
1547     parser.add_argument("--nosign", action="store_true", default=False,
1548                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1549     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1550                         help="Use date from apk instead of current time for newly added apks")
1551     parser.add_argument("--rename-apks", action="store_true", default=False,
1552                         help="Rename APK files that do not match package.name_123.apk")
1553     metadata.add_metadata_arguments(parser)
1554     options = parser.parse_args()
1555     metadata.warnings_action = options.W
1556
1557     config = common.read_config(options)
1558
1559     if not ('jarsigner' in config and 'keytool' in config):
1560         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1561
1562     repodirs = ['repo']
1563     if config['archive_older'] != 0:
1564         repodirs.append('archive')
1565         if not os.path.exists('archive'):
1566             os.mkdir('archive')
1567
1568     if options.icons:
1569         resize_all_icons(repodirs)
1570         sys.exit(0)
1571
1572     if options.rename_apks:
1573         options.clean = True
1574
1575     # check that icons exist now, rather than fail at the end of `fdroid update`
1576     for k in ['repo_icon', 'archive_icon']:
1577         if k in config:
1578             if not os.path.exists(config[k]):
1579                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1580                 sys.exit(1)
1581
1582     # if the user asks to create a keystore, do it now, reusing whatever it can
1583     if options.create_key:
1584         if os.path.exists(config['keystore']):
1585             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1586             logging.critical("\t'" + config['keystore'] + "'")
1587             sys.exit(1)
1588
1589         if 'repo_keyalias' not in config:
1590             config['repo_keyalias'] = socket.getfqdn()
1591             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1592         if 'keydname' not in config:
1593             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1594             common.write_to_config(config, 'keydname', config['keydname'])
1595         if 'keystore' not in config:
1596             config['keystore'] = common.default_config['keystore']
1597             common.write_to_config(config, 'keystore', config['keystore'])
1598
1599         password = common.genpassword()
1600         if 'keystorepass' not in config:
1601             config['keystorepass'] = password
1602             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1603         if 'keypass' not in config:
1604             config['keypass'] = password
1605             common.write_to_config(config, 'keypass', config['keypass'])
1606         common.genkeystore(config)
1607
1608     # Get all apps...
1609     apps = metadata.read_metadata()
1610
1611     # Generate a list of categories...
1612     categories = set()
1613     for app in apps.values():
1614         categories.update(app.Categories)
1615
1616     # Read known apks data (will be updated and written back when we've finished)
1617     knownapks = common.KnownApks()
1618
1619     # Get APK cache
1620     apkcache = get_cache()
1621
1622     # Delete builds for disabled apps
1623     delete_disabled_builds(apps, apkcache, repodirs)
1624
1625     # Scan all apks in the main repo
1626     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1627
1628     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1629                                            options.use_date_from_apk)
1630     cachechanged = cachechanged or fcachechanged
1631     apks += files
1632     # Generate warnings for apk's with no metadata (or create skeleton
1633     # metadata files, if requested on the command line)
1634     newmetadata = False
1635     for apk in apks:
1636         if apk['packageName'] not in apps:
1637             if options.create_metadata:
1638                 if 'name' not in apk:
1639                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1640                     continue
1641                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1642                 f.write("License:Unknown\n")
1643                 f.write("Web Site:\n")
1644                 f.write("Source Code:\n")
1645                 f.write("Issue Tracker:\n")
1646                 f.write("Changelog:\n")
1647                 f.write("Summary:" + apk['name'] + "\n")
1648                 f.write("Description:\n")
1649                 f.write(apk['name'] + "\n")
1650                 f.write(".\n")
1651                 f.write("Name:" + apk['name'] + "\n")
1652                 f.close()
1653                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1654                 newmetadata = True
1655             else:
1656                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1657                 if options.delete_unknown:
1658                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1659                     rmf = os.path.join(repodirs[0], apk['apkName'])
1660                     if not os.path.exists(rmf):
1661                         logging.error("Could not find {0} to remove it".format(rmf))
1662                     else:
1663                         os.remove(rmf)
1664                 else:
1665                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1666
1667     # update the metadata with the newly created ones included
1668     if newmetadata:
1669         apps = metadata.read_metadata()
1670
1671     copy_triple_t_store_metadata(apps)
1672     insert_obbs(repodirs[0], apps, apks)
1673     insert_localized_app_metadata(apps)
1674
1675     # Scan the archive repo for apks as well
1676     if len(repodirs) > 1:
1677         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1678         if cc:
1679             cachechanged = True
1680     else:
1681         archapks = []
1682
1683     # Apply information from latest apks to the application and update dates
1684     apply_info_from_latest_apk(apps, apks + archapks)
1685
1686     # Sort the app list by name, then the web site doesn't have to by default.
1687     # (we had to wait until we'd scanned the apks to do this, because mostly the
1688     # name comes from there!)
1689     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1690
1691     # APKs are placed into multiple repos based on the app package, providing
1692     # per-app subscription feeds for nightly builds and things like it
1693     if config['per_app_repos']:
1694         add_apks_to_per_app_repos(repodirs[0], apks)
1695         for appid, app in apps.items():
1696             repodir = os.path.join(appid, 'fdroid', 'repo')
1697             appdict = dict()
1698             appdict[appid] = app
1699             if os.path.isdir(repodir):
1700                 index.make(appdict, [appid], apks, repodir, False)
1701             else:
1702                 logging.info('Skipping index generation for ' + appid)
1703         return
1704
1705     if len(repodirs) > 1:
1706         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1707
1708     # Make the index for the main repo...
1709     index.make(apps, sortedids, apks, repodirs[0], False)
1710     make_categories_txt(repodirs[0], categories)
1711
1712     # If there's an archive repo,  make the index for it. We already scanned it
1713     # earlier on.
1714     if len(repodirs) > 1:
1715         index.make(apps, sortedids, archapks, repodirs[1], True)
1716
1717     git_remote = config.get('binary_transparency_remote')
1718     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1719         from . import btlog
1720         btlog.make_binary_transparency_log(repodirs)
1721
1722     if config['update_stats']:
1723         # Update known apks info...
1724         knownapks.writeifchanged()
1725
1726         # Generate latest apps data for widget
1727         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1728             data = ''
1729             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1730                 for line in f:
1731                     appid = line.rstrip()
1732                     data += appid + "\t"
1733                     app = apps[appid]
1734                     data += app.Name + "\t"
1735                     if app.icon is not None:
1736                         data += app.icon + "\t"
1737                     data += app.License + "\n"
1738             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1739                 f.write(data)
1740
1741     if cachechanged:
1742         write_cache(apkcache)
1743
1744     # Update the wiki...
1745     if options.wiki:
1746         update_wiki(apps, sortedids, apks + archapks)
1747
1748     logging.info("Finished.")
1749
1750
1751 if __name__ == "__main__":
1752     main()