fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import btlog
  41 from . import common
  42 from . import index
  43 from . import metadata
  44 from .common import SdkToolsPopen
  45 from .exception import BuildException, FDroidException
  46
  47 METADATA_VERSION = 18
  48
  49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  50 UNSET_VERSION_CODE = -0x100000000
  51
  52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  59 APK_PERMISSION_PAT = \
  60     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  62
  63 screen_densities = ['640', '480', '320', '240', '160', '120']
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "undefined": '-1',
  72     "anydpi": '65534',
  73     "nodpi": '65535'
  74 }
  75
  76 all_screen_densities = ['0'] + screen_densities
  77
  78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  80
  81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  84                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  85
  86
  87 def dpi_to_px(density):
  88     return (int(density) * 48) / 160
  89
  90
  91 def px_to_dpi(px):
  92     return (int(px) * 160) / 48
  93
  94
  95 def get_icon_dir(repodir, density):
  96     if density == '0':
  97         return os.path.join(repodir, "icons")
  98     return os.path.join(repodir, "icons-%s" % density)
  99
 100
 101 def get_icon_dirs(repodir):
 102     for density in screen_densities:
 103         yield get_icon_dir(repodir, density)
 104
 105
 106 def get_all_icon_dirs(repodir):
 107     for density in all_screen_densities:
 108         yield get_icon_dir(repodir, density)
 109
 110
 111 def update_wiki(apps, sortedids, apks):
 112     """Update the wiki
 113
 114     :param apps: fully populated list of all applications
 115     :param apks: all apks, except...
 116     """
 117     logging.info("Updating wiki")
 118     wikicat = 'Apps'
 119     wikiredircat = 'App Redirects'
 120     import mwclient
 121     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 122                          path=config['wiki_path'])
 123     site.login(config['wiki_user'], config['wiki_password'])
 124     generated_pages = {}
 125     generated_redirects = {}
 126
 127     for appid in sortedids:
 128         app = metadata.App(apps[appid])
 129
 130         wikidata = ''
 131         if app.Disabled:
 132             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 133         if app.AntiFeatures:
 134             for af in app.AntiFeatures:
 135                 wikidata += '{{AntiFeature|' + af + '}}\n'
 136         if app.RequiresRoot:
 137             requiresroot = 'Yes'
 138         else:
 139             requiresroot = 'No'
 140         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 141             appid,
 142             app.Name,
 143             app.added.strftime('%Y-%m-%d') if app.added else '',
 144             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 145             app.SourceCode,
 146             app.IssueTracker,
 147             app.WebSite,
 148             app.Changelog,
 149             app.Donate,
 150             app.FlattrID,
 151             app.Bitcoin,
 152             app.Litecoin,
 153             app.License,
 154             requiresroot,
 155             app.AuthorName,
 156             app.AuthorEmail)
 157
 158         if app.Provides:
 159             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 160
 161         wikidata += app.Summary
 162         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 163
 164         wikidata += "=Description=\n"
 165         wikidata += metadata.description_wiki(app.Description) + "\n"
 166
 167         wikidata += "=Maintainer Notes=\n"
 168         if app.MaintainerNotes:
 169             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 170         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 171
 172         # Get a list of all packages for this application...
 173         apklist = []
 174         gotcurrentver = False
 175         cantupdate = False
 176         buildfails = False
 177         for apk in apks:
 178             if apk['packageName'] == appid:
 179                 if str(apk['versionCode']) == app.CurrentVersionCode:
 180                     gotcurrentver = True
 181                 apklist.append(apk)
 182         # Include ones we can't build, as a special case...
 183         for build in app.builds:
 184             if build.disable:
 185                 if build.versionCode == app.CurrentVersionCode:
 186                     cantupdate = True
 187                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 188                 apklist.append({'versionCode': int(build.versionCode),
 189                                 'versionName': build.versionName,
 190                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 191                                 })
 192             else:
 193                 builtit = False
 194                 for apk in apklist:
 195                     if apk['versionCode'] == int(build.versionCode):
 196                         builtit = True
 197                         break
 198                 if not builtit:
 199                     buildfails = True
 200                     apklist.append({'versionCode': int(build.versionCode),
 201                                     'versionName': build.versionName,
 202                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 203                                     })
 204         if app.CurrentVersionCode == '0':
 205             cantupdate = True
 206         # Sort with most recent first...
 207         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 208
 209         wikidata += "=Versions=\n"
 210         if len(apklist) == 0:
 211             wikidata += "We currently have no versions of this app available."
 212         elif not gotcurrentver:
 213             wikidata += "We don't have the current version of this app."
 214         else:
 215             wikidata += "We have the current version of this app."
 216         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 217         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 218         if len(app.NoSourceSince) > 0:
 219             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 220         if len(app.CurrentVersion) > 0:
 221             wikidata += "The current (recommended) version is " + app.CurrentVersion
 222             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 223         validapks = 0
 224         for apk in apklist:
 225             wikidata += "==" + apk['versionName'] + "==\n"
 226
 227             if 'buildproblem' in apk:
 228                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 229             else:
 230                 validapks += 1
 231                 wikidata += "This version is built and signed by "
 232                 if 'srcname' in apk:
 233                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 234                 else:
 235                     wikidata += "the original developer.\n\n"
 236             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 237
 238         wikidata += '\n[[Category:' + wikicat + ']]\n'
 239         if len(app.NoSourceSince) > 0:
 240             wikidata += '\n[[Category:Apps missing source code]]\n'
 241         if validapks == 0 and not app.Disabled:
 242             wikidata += '\n[[Category:Apps with no packages]]\n'
 243         if cantupdate and not app.Disabled:
 244             wikidata += "\n[[Category:Apps we cannot update]]\n"
 245         if buildfails and not app.Disabled:
 246             wikidata += "\n[[Category:Apps with failing builds]]\n"
 247         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 248             wikidata += '\n[[Category:Apps to Update]]\n'
 249         if app.Disabled:
 250             wikidata += '\n[[Category:Apps that are disabled]]\n'
 251         if app.UpdateCheckMode == 'None' and not app.Disabled:
 252             wikidata += '\n[[Category:Apps with no update check]]\n'
 253         for appcat in app.Categories:
 254             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 255
 256         # We can't have underscores in the page name, even if they're in
 257         # the package ID, because MediaWiki messes with them...
 258         pagename = appid.replace('_', ' ')
 259
 260         # Drop a trailing newline, because mediawiki is going to drop it anyway
 261         # and it we don't we'll think the page has changed when it hasn't...
 262         if wikidata.endswith('\n'):
 263             wikidata = wikidata[:-1]
 264
 265         generated_pages[pagename] = wikidata
 266
 267         # Make a redirect from the name to the ID too, unless there's
 268         # already an existing page with the name and it isn't a redirect.
 269         noclobber = False
 270         apppagename = app.Name.replace('_', ' ')
 271         apppagename = apppagename.replace('{', '')
 272         apppagename = apppagename.replace('}', ' ')
 273         apppagename = apppagename.replace(':', ' ')
 274         apppagename = apppagename.replace('[', ' ')
 275         apppagename = apppagename.replace(']', ' ')
 276         # Drop double spaces caused mostly by replacing ':' above
 277         apppagename = apppagename.replace('  ', ' ')
 278         for expagename in site.allpages(prefix=apppagename,
 279                                         filterredir='nonredirects',
 280                                         generator=False):
 281             if expagename == apppagename:
 282                 noclobber = True
 283         # Another reason not to make the redirect page is if the app name
 284         # is the same as it's ID, because that will overwrite the real page
 285         # with an redirect to itself! (Although it seems like an odd
 286         # scenario this happens a lot, e.g. where there is metadata but no
 287         # builds or binaries to extract a name from.
 288         if apppagename == pagename:
 289             noclobber = True
 290         if not noclobber:
 291             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 292
 293     for tcat, genp in [(wikicat, generated_pages),
 294                        (wikiredircat, generated_redirects)]:
 295         catpages = site.Pages['Category:' + tcat]
 296         existingpages = []
 297         for page in catpages:
 298             existingpages.append(page.name)
 299             if page.name in genp:
 300                 pagetxt = page.edit()
 301                 if pagetxt != genp[page.name]:
 302                     logging.debug("Updating modified page " + page.name)
 303                     page.save(genp[page.name], summary='Auto-updated')
 304                 else:
 305                     logging.debug("Page " + page.name + " is unchanged")
 306             else:
 307                 logging.warn("Deleting page " + page.name)
 308                 page.delete('No longer published')
 309         for pagename, text in genp.items():
 310             logging.debug("Checking " + pagename)
 311             if pagename not in existingpages:
 312                 logging.debug("Creating page " + pagename)
 313                 try:
 314                     newpage = site.Pages[pagename]
 315                     newpage.save(text, summary='Auto-created')
 316                 except Exception as e:
 317                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 318
 319     # Purge server cache to ensure counts are up to date
 320     site.pages['Repository Maintenance'].purge()
 321
 322
 323 def delete_disabled_builds(apps, apkcache, repodirs):
 324     """Delete disabled build outputs.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodirs: the repo directories to process
 329     """
 330     for appid, app in apps.items():
 331         for build in app['builds']:
 332             if not build.disable:
 333                 continue
 334             apkfilename = common.get_release_filename(app, build)
 335             iconfilename = "%s.%s.png" % (
 336                 appid,
 337                 build.versionCode)
 338             for repodir in repodirs:
 339                 files = [
 340                     os.path.join(repodir, apkfilename),
 341                     os.path.join(repodir, apkfilename + '.asc'),
 342                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 343                 ]
 344                 for density in all_screen_densities:
 345                     repo_dir = get_icon_dir(repodir, density)
 346                     files.append(os.path.join(repo_dir, iconfilename))
 347
 348                 for f in files:
 349                     if os.path.exists(f):
 350                         logging.info("Deleting disabled build output " + f)
 351                         os.remove(f)
 352             if apkfilename in apkcache:
 353                 del apkcache[apkfilename]
 354
 355
 356 def resize_icon(iconpath, density):
 357
 358     if not os.path.isfile(iconpath):
 359         return
 360
 361     fp = None
 362     try:
 363         fp = open(iconpath, 'rb')
 364         im = Image.open(fp)
 365         size = dpi_to_px(density)
 366
 367         if any(length > size for length in im.size):
 368             oldsize = im.size
 369             im.thumbnail((size, size), Image.ANTIALIAS)
 370             logging.debug("%s was too large at %s - new size is %s" % (
 371                 iconpath, oldsize, im.size))
 372             im.save(iconpath, "PNG")
 373
 374     except Exception as e:
 375         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 376
 377     finally:
 378         if fp:
 379             fp.close()
 380
 381
 382 def resize_all_icons(repodirs):
 383     """Resize all icons that exceed the max size
 384
 385     :param repodirs: the repo directories to process
 386     """
 387     for repodir in repodirs:
 388         for density in screen_densities:
 389             icon_dir = get_icon_dir(repodir, density)
 390             icon_glob = os.path.join(icon_dir, '*.png')
 391             for iconpath in glob.glob(icon_glob):
 392                 resize_icon(iconpath, density)
 393
 394
 395 def getsig(apkpath):
 396     """ Get the signing certificate of an apk. To get the same md5 has that
 397     Android gets, we encode the .RSA certificate in a specific format and pass
 398     it hex-encoded to the md5 digest algorithm.
 399
 400     :param apkpath: path to the apk
 401     :returns: A string containing the md5 of the signature of the apk or None
 402               if an error occurred.
 403     """
 404
 405     # verify the jar signature is correct
 406     if not common.verify_apk_signature(apkpath):
 407         return None
 408
 409     with zipfile.ZipFile(apkpath, 'r') as apk:
 410         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 411
 412         if len(certs) < 1:
 413             logging.error("Found no signing certificates on %s" % apkpath)
 414             return None
 415         if len(certs) > 1:
 416             logging.error("Found multiple signing certificates on %s" % apkpath)
 417             return None
 418
 419         cert = apk.read(certs[0])
 420
 421     cert_encoded = common.get_certificate(cert)
 422
 423     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 424
 425
 426 def get_cache_file():
 427     return os.path.join('tmp', 'apkcache')
 428
 429
 430 def get_cache():
 431     """
 432     Gather information about all the apk files in the repo directory,
 433     using cached data if possible.
 434     :return: apkcache
 435     """
 436     apkcachefile = get_cache_file()
 437     if not options.clean and os.path.exists(apkcachefile):
 438         with open(apkcachefile, 'rb') as cf:
 439             apkcache = pickle.load(cf, encoding='utf-8')
 440         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
 441             apkcache = {}
 442     else:
 443         apkcache = {}
 444
 445     return apkcache
 446
 447
 448 def write_cache(apkcache):
 449     apkcachefile = get_cache_file()
 450     cache_path = os.path.dirname(apkcachefile)
 451     if not os.path.exists(cache_path):
 452         os.makedirs(cache_path)
 453     apkcache["METADATA_VERSION"] = METADATA_VERSION
 454     with open(apkcachefile, 'wb') as cf:
 455         pickle.dump(apkcache, cf)
 456
 457
 458 def get_icon_bytes(apkzip, iconsrc):
 459     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 460     try:
 461         return apkzip.read(iconsrc)
 462     except KeyError:
 463         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 464
 465
 466 def sha256sum(filename):
 467     '''Calculate the sha256 of the given file'''
 468     sha = hashlib.sha256()
 469     with open(filename, 'rb') as f:
 470         while True:
 471             t = f.read(16384)
 472             if len(t) == 0:
 473                 break
 474             sha.update(t)
 475     return sha.hexdigest()
 476
 477
 478 def has_old_openssl(filename):
 479     '''checks for known vulnerable openssl versions in the APK'''
 480
 481     # statically load this pattern
 482     if not hasattr(has_old_openssl, "pattern"):
 483         has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 484
 485     with zipfile.ZipFile(filename) as zf:
 486         for name in zf.namelist():
 487             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 488                 lib = zf.open(name)
 489                 while True:
 490                     chunk = lib.read(4096)
 491                     if chunk == b'':
 492                         break
 493                     m = has_old_openssl.pattern.search(chunk)
 494                     if m:
 495                         version = m.group(1).decode('ascii')
 496                         if version.startswith('1.0.1') and version[5] >= 'r' \
 497                            or version.startswith('1.0.2') and version[5] >= 'f':
 498                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 499                         else:
 500                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 501                             return True
 502                         break
 503     return False
 504
 505
 506 def insert_obbs(repodir, apps, apks):
 507     """Scans the .obb files in a given repo directory and adds them to the
 508     relevant APK instances.  OBB files have versionCodes like APK
 509     files, and they are loosely associated.  If there is an OBB file
 510     present, then any APK with the same or higher versionCode will use
 511     that OBB file.  There are two OBB types: main and patch, each APK
 512     can only have only have one of each.
 513
 514     https://developer.android.com/google/play/expansion-files.html
 515
 516     :param repodir: repo directory to scan
 517     :param apps: list of current, valid apps
 518     :param apks: current information on all APKs
 519
 520     """
 521
 522     def obbWarnDelete(f, msg):
 523         logging.warning(msg + f)
 524         if options.delete_unknown:
 525             logging.error("Deleting unknown file: " + f)
 526             os.remove(f)
 527
 528     obbs = []
 529     java_Integer_MIN_VALUE = -pow(2, 31)
 530     currentPackageNames = apps.keys()
 531     for f in glob.glob(os.path.join(repodir, '*.obb')):
 532         obbfile = os.path.basename(f)
 533         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 534         chunks = obbfile.split('.')
 535         if chunks[0] != 'main' and chunks[0] != 'patch':
 536             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 537             continue
 538         if not re.match(r'^-?[0-9]+$', chunks[1]):
 539             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 540             continue
 541         versionCode = int(chunks[1])
 542         packagename = ".".join(chunks[2:-1])
 543
 544         highestVersionCode = java_Integer_MIN_VALUE
 545         if packagename not in currentPackageNames:
 546             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 547             continue
 548         for apk in apks:
 549             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 550                 highestVersionCode = apk['versionCode']
 551         if versionCode > highestVersionCode:
 552             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 553                           + ') than any APK: ')
 554             continue
 555         obbsha256 = sha256sum(f)
 556         obbs.append((packagename, versionCode, obbfile, obbsha256))
 557
 558     for apk in apks:
 559         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 560             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 561                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 562                     apk['obbMainFile'] = obbfile
 563                     apk['obbMainFileSha256'] = obbsha256
 564                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 565                     apk['obbPatchFile'] = obbfile
 566                     apk['obbPatchFileSha256'] = obbsha256
 567             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 568                 break
 569
 570
 571 def _get_localized_dict(app, locale):
 572     '''get the dict to add localized store metadata to'''
 573     if 'localized' not in app:
 574         app['localized'] = collections.OrderedDict()
 575     if locale not in app['localized']:
 576         app['localized'][locale] = collections.OrderedDict()
 577     return app['localized'][locale]
 578
 579
 580 def _set_localized_text_entry(app, locale, key, f):
 581     limit = config['char_limits'][key]
 582     localized = _get_localized_dict(app, locale)
 583     with open(f) as fp:
 584         text = fp.read()[:limit]
 585         if len(text) > 0:
 586             localized[key] = text
 587
 588
 589 def _set_author_entry(app, key, f):
 590     limit = config['char_limits']['author']
 591     with open(f) as fp:
 592         text = fp.read()[:limit]
 593         if len(text) > 0:
 594             app[key] = text
 595
 596
 597 def copy_triple_t_store_metadata(apps):
 598     """Include store metadata from the app's source repo
 599
 600     The Triple-T Gradle Play Publisher is a plugin that has a standard
 601     file layout for all of the metadata and graphics that the Google
 602     Play Store accepts.  Since F-Droid has the git repo, it can just
 603     pluck those files directly.  This method reads any text files into
 604     the app dict, then copies any graphics into the fdroid repo
 605     directory structure.
 606
 607     This needs to be run before insert_localized_app_metadata() so that
 608     the graphics files that are copied into the fdroid repo get
 609     properly indexed.
 610
 611     https://github.com/Triple-T/gradle-play-publisher#upload-images
 612     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 613
 614     """
 615
 616     if not os.path.isdir('build'):
 617         return  # nothing to do
 618
 619     for packageName, app in apps.items():
 620         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 621             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 622             for root, dirs, files in os.walk(d):
 623                 segments = root.split('/')
 624                 locale = segments[-2]
 625                 for f in files:
 626                     if f == 'fulldescription':
 627                         _set_localized_text_entry(app, locale, 'description',
 628                                                   os.path.join(root, f))
 629                         continue
 630                     elif f == 'shortdescription':
 631                         _set_localized_text_entry(app, locale, 'summary',
 632                                                   os.path.join(root, f))
 633                         continue
 634                     elif f == 'title':
 635                         _set_localized_text_entry(app, locale, 'name',
 636                                                   os.path.join(root, f))
 637                         continue
 638                     elif f == 'video':
 639                         _set_localized_text_entry(app, locale, 'video',
 640                                                   os.path.join(root, f))
 641                         continue
 642                     elif f == 'whatsnew':
 643                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 644                                                   os.path.join(root, f))
 645                         continue
 646                     elif f == 'contactEmail':
 647                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 648                         continue
 649                     elif f == 'contactPhone':
 650                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 651                         continue
 652                     elif f == 'contactWebsite':
 653                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 654                         continue
 655
 656                     base, extension = common.get_extension(f)
 657                     dirname = os.path.basename(root)
 658                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 659                         if segments[-2] == 'listing':
 660                             locale = segments[-3]
 661                         else:
 662                             locale = segments[-2]
 663                         destdir = os.path.join('repo', packageName, locale)
 664                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 665                         sourcefile = os.path.join(root, f)
 666                         destfile = os.path.join(destdir, dirname + '.' + extension)
 667                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 668                         shutil.copy(sourcefile, destfile)
 669
 670
 671 def insert_localized_app_metadata(apps):
 672     """scans standard locations for graphics and localized text
 673
 674     Scans for localized description files, store graphics, and
 675     screenshot PNG files in statically defined screenshots directory
 676     and adds them to the app metadata.  The screenshots and graphic
 677     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 678     and must be in the following layout:
 679     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 680
 681     repo/packageName/locale/featureGraphic.png
 682     repo/packageName/locale/phoneScreenshots/1.png
 683     repo/packageName/locale/phoneScreenshots/2.png
 684
 685     The changelog files must be text files named with the versionCode
 686     ending with ".txt" and must be in the following layout:
 687     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 688
 689     repo/packageName/locale/changelogs/12345.txt
 690
 691     This will scan the each app's source repo then the metadata/ dir
 692     for these standard locations of changelog files.  If it finds
 693     them, they will be added to the dict of all packages, with the
 694     versions in the metadata/ folder taking precendence over the what
 695     is in the app's source repo.
 696
 697     Where "packageName" is the app's packageName and "locale" is the locale
 698     of the graphics, e.g. what language they are in, using the IETF RFC5646
 699     format (en-US, fr-CA, es-MX, etc).
 700
 701     This will also scan the app's git for a fastlane folder, and the
 702     metadata/ folder and the apps' source repos for standard locations
 703     of graphic and screenshot files.  If it finds them, it will copy
 704     them into the repo.  The fastlane files follow this pattern:
 705     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 706
 707     """
 708
 709     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 710     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 711
 712     for d in sorted(sourcedirs):
 713         if not os.path.isdir(d):
 714             continue
 715         for root, dirs, files in os.walk(d):
 716             segments = root.split('/')
 717             packageName = segments[1]
 718             if packageName not in apps:
 719                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 720                 continue
 721             locale = segments[-1]
 722             for f in files:
 723                 if f == 'full_description.txt':
 724                     _set_localized_text_entry(apps[packageName], locale, 'description',
 725                                               os.path.join(root, f))
 726                     continue
 727                 elif f == 'short_description.txt':
 728                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 729                                               os.path.join(root, f))
 730                     continue
 731                 elif f == 'title.txt':
 732                     _set_localized_text_entry(apps[packageName], locale, 'name',
 733                                               os.path.join(root, f))
 734                     continue
 735                 elif f == 'video.txt':
 736                     _set_localized_text_entry(apps[packageName], locale, 'video',
 737                                               os.path.join(root, f))
 738                     continue
 739                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 740                     locale = segments[-2]
 741                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 742                                               os.path.join(root, f))
 743                     continue
 744
 745                 base, extension = common.get_extension(f)
 746                 if locale == 'images':
 747                     locale = segments[-2]
 748                 destdir = os.path.join('repo', packageName, locale)
 749                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 750                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 751                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 752                     shutil.copy(os.path.join(root, f), destdir)
 753             for d in dirs:
 754                 if d in SCREENSHOT_DIRS:
 755                     for f in glob.glob(os.path.join(root, d, '*.*')):
 756                         _, extension = common.get_extension(f)
 757                         if extension in ALLOWED_EXTENSIONS:
 758                             screenshotdestdir = os.path.join(destdir, d)
 759                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 760                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 761                             shutil.copy(f, screenshotdestdir)
 762
 763     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 764     for d in repofiles:
 765         if not os.path.isdir(d):
 766             continue
 767         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 768             if not os.path.isfile(f):
 769                 continue
 770             segments = f.split('/')
 771             packageName = segments[1]
 772             locale = segments[2]
 773             screenshotdir = segments[3]
 774             filename = os.path.basename(f)
 775             base, extension = common.get_extension(filename)
 776
 777             if packageName not in apps:
 778                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 779                                 % (filename, packageName))
 780                 continue
 781             graphics = _get_localized_dict(apps[packageName], locale)
 782
 783             if extension not in ALLOWED_EXTENSIONS:
 784                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 785             elif base in GRAPHIC_NAMES:
 786                 # there can only be zero or one of these per locale
 787                 graphics[base] = filename
 788             elif screenshotdir in SCREENSHOT_DIRS:
 789                 # there can any number of these per locale
 790                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 791                 if screenshotdir not in graphics:
 792                     graphics[screenshotdir] = []
 793                 graphics[screenshotdir].append(filename)
 794             else:
 795                 logging.warning('Unsupported graphics file found: ' + f)
 796
 797
 798 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 799     """Scan a repo for all files with an extension except APK/OBB
 800
 801     :param apkcache: current cached info about all repo files
 802     :param repodir: repo directory to scan
 803     :param knownapks: list of all known files, as per metadata.read_metadata
 804     :param use_date_from_file: use date from file (instead of current date)
 805                                for newly added files
 806     """
 807
 808     cachechanged = False
 809     repo_files = []
 810     repodir = repodir.encode('utf-8')
 811     for name in os.listdir(repodir):
 812         file_extension = common.get_file_extension(name)
 813         if file_extension == 'apk' or file_extension == 'obb':
 814             continue
 815         filename = os.path.join(repodir, name)
 816         name_utf8 = name.decode('utf-8')
 817         if filename.endswith(b'_src.tar.gz'):
 818             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 819             continue
 820         if not common.is_repo_file(filename):
 821             continue
 822         stat = os.stat(filename)
 823         if stat.st_size == 0:
 824             raise FDroidException(filename + ' is zero size!')
 825
 826         shasum = sha256sum(filename)
 827         usecache = False
 828         if name in apkcache:
 829             repo_file = apkcache[name]
 830             # added time is cached as tuple but used here as datetime instance
 831             if 'added' in repo_file:
 832                 a = repo_file['added']
 833                 if isinstance(a, datetime):
 834                     repo_file['added'] = a
 835                 else:
 836                     repo_file['added'] = datetime(*a[:6])
 837             if repo_file.get('hash') == shasum:
 838                 logging.debug("Reading " + name_utf8 + " from cache")
 839                 usecache = True
 840             else:
 841                 logging.debug("Ignoring stale cache data for " + name)
 842
 843         if not usecache:
 844             logging.debug("Processing " + name_utf8)
 845             repo_file = collections.OrderedDict()
 846             # TODO rename apkname globally to something more generic
 847             repo_file['name'] = name_utf8
 848             repo_file['apkName'] = name_utf8
 849             repo_file['hash'] = shasum
 850             repo_file['hashType'] = 'sha256'
 851             repo_file['versionCode'] = 0
 852             repo_file['versionName'] = shasum
 853             # the static ID is the SHA256 unless it is set in the metadata
 854             repo_file['packageName'] = shasum
 855
 856             n = name_utf8.rsplit('_', maxsplit=1)
 857             if len(n) == 2:
 858                 packageName = n[0]
 859                 versionCode = n[1].split('.')[0]
 860                 if re.match('^-?[0-9]+$', versionCode) \
 861                    and common.is_valid_package_name(n[0]):
 862                     repo_file['packageName'] = packageName
 863                     repo_file['versionCode'] = int(versionCode)
 864             srcfilename = name + b'_src.tar.gz'
 865             if os.path.exists(os.path.join(repodir, srcfilename)):
 866                 repo_file['srcname'] = srcfilename.decode('utf-8')
 867             repo_file['size'] = stat.st_size
 868
 869             apkcache[name] = repo_file
 870             cachechanged = True
 871
 872         if use_date_from_file:
 873             timestamp = stat.st_ctime
 874             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 875         else:
 876             default_date_param = None
 877
 878         # Record in knownapks, getting the added date at the same time..
 879         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 880                                     default_date=default_date_param)
 881         if added:
 882             repo_file['added'] = added
 883
 884         repo_files.append(repo_file)
 885
 886     return repo_files, cachechanged
 887
 888
 889 def scan_apk_aapt(apk, apkfile):
 890     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 891     if p.returncode != 0:
 892         if options.delete_unknown:
 893             if os.path.exists(apkfile):
 894                 logging.error("Failed to get apk information, deleting " + apkfile)
 895                 os.remove(apkfile)
 896             else:
 897                 logging.error("Could not find {0} to remove it".format(apkfile))
 898         else:
 899             logging.error("Failed to get apk information, skipping " + apkfile)
 900         raise BuildException("Invalid APK")
 901     for line in p.output.splitlines():
 902         if line.startswith("package:"):
 903             try:
 904                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 905                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 906                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 907             except Exception as e:
 908                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
 909         elif line.startswith("application:"):
 910             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 911             # Keep path to non-dpi icon in case we need it
 912             match = re.match(APK_ICON_PAT_NODPI, line)
 913             if match:
 914                 apk['icons_src']['-1'] = match.group(1)
 915         elif line.startswith("launchable-activity:"):
 916             # Only use launchable-activity as fallback to application
 917             if not apk['name']:
 918                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 919             if '-1' not in apk['icons_src']:
 920                 match = re.match(APK_ICON_PAT_NODPI, line)
 921                 if match:
 922                     apk['icons_src']['-1'] = match.group(1)
 923         elif line.startswith("application-icon-"):
 924             match = re.match(APK_ICON_PAT, line)
 925             if match:
 926                 density = match.group(1)
 927                 path = match.group(2)
 928                 apk['icons_src'][density] = path
 929         elif line.startswith("sdkVersion:"):
 930             m = re.match(APK_SDK_VERSION_PAT, line)
 931             if m is None:
 932                 logging.error(line.replace('sdkVersion:', '')
 933                               + ' is not a valid minSdkVersion!')
 934             else:
 935                 apk['minSdkVersion'] = m.group(1)
 936                 # if target not set, default to min
 937                 if 'targetSdkVersion' not in apk:
 938                     apk['targetSdkVersion'] = m.group(1)
 939         elif line.startswith("targetSdkVersion:"):
 940             m = re.match(APK_SDK_VERSION_PAT, line)
 941             if m is None:
 942                 logging.error(line.replace('targetSdkVersion:', '')
 943                               + ' is not a valid targetSdkVersion!')
 944             else:
 945                 apk['targetSdkVersion'] = m.group(1)
 946         elif line.startswith("maxSdkVersion:"):
 947             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 948         elif line.startswith("native-code:"):
 949             apk['nativecode'] = []
 950             for arch in line[13:].split(' '):
 951                 apk['nativecode'].append(arch[1:-1])
 952         elif line.startswith('uses-permission:'):
 953             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 954             if perm_match['maxSdkVersion']:
 955                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 956             permission = UsesPermission(
 957                 perm_match['name'],
 958                 perm_match['maxSdkVersion']
 959             )
 960
 961             apk['uses-permission'].append(permission)
 962         elif line.startswith('uses-permission-sdk-23:'):
 963             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 964             if perm_match['maxSdkVersion']:
 965                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 966             permission_sdk_23 = UsesPermissionSdk23(
 967                 perm_match['name'],
 968                 perm_match['maxSdkVersion']
 969             )
 970
 971             apk['uses-permission-sdk-23'].append(permission_sdk_23)
 972
 973         elif line.startswith('uses-feature:'):
 974             feature = re.match(APK_FEATURE_PAT, line).group(1)
 975             # Filter out this, it's only added with the latest SDK tools and
 976             # causes problems for lots of apps.
 977             if feature != "android.hardware.screen.portrait" \
 978                     and feature != "android.hardware.screen.landscape":
 979                 if feature.startswith("android.feature."):
 980                     feature = feature[16:]
 981                 apk['features'].add(feature)
 982
 983
 984 def scan_apk_androguard(apk, apkfile):
 985     try:
 986         from androguard.core.bytecodes.apk import APK
 987         apkobject = APK(apkfile)
 988         if apkobject.is_valid_APK():
 989             arsc = apkobject.get_android_resources()
 990         else:
 991             if options.delete_unknown:
 992                 if os.path.exists(apkfile):
 993                     logging.error("Failed to get apk information, deleting " + apkfile)
 994                     os.remove(apkfile)
 995                 else:
 996                     logging.error("Could not find {0} to remove it".format(apkfile))
 997             else:
 998                 logging.error("Failed to get apk information, skipping " + apkfile)
 999             raise BuildException("Invaild APK")
1000     except ImportError:
1001         raise FDroidException("androguard library is not installed and aapt not present")
1002     except FileNotFoundError:
1003         logging.error("Could not open apk file for analysis")
1004         raise BuildException("Invalid APK")
1005
1006     apk['packageName'] = apkobject.get_package()
1007     apk['versionCode'] = int(apkobject.get_androidversion_code())
1008     apk['versionName'] = apkobject.get_androidversion_name()
1009     if apk['versionName'][0] == "@":
1010         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1011         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1012         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1013     apk['name'] = apkobject.get_app_name()
1014
1015     if apkobject.get_max_sdk_version() is not None:
1016         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1017     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1018     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1019
1020     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1021     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1022
1023     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1024
1025     for file in apkobject.get_files():
1026         d_re = density_re.match(file)
1027         if d_re:
1028             folder = d_re.group(1).split('-')
1029             if len(folder) > 1:
1030                 resolution = folder[1]
1031             else:
1032                 resolution = 'mdpi'
1033             density = screen_resolutions[resolution]
1034             apk['icons_src'][density] = d_re.group(0)
1035
1036     if apk['icons_src'].get('-1') is None:
1037         apk['icons_src']['-1'] = apk['icons_src']['160']
1038
1039     arch_re = re.compile("^lib/(.*)/.*$")
1040     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1041     if len(arch) >= 1:
1042         apk['nativecode'] = []
1043         apk['nativecode'].extend(sorted(list(arch)))
1044
1045     xml = apkobject.get_android_manifest_xml()
1046
1047     for item in xml.getElementsByTagName('uses-permission'):
1048         name = str(item.getAttribute("android:name"))
1049         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1050         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1051         permission = UsesPermission(
1052             name,
1053             maxSdkVersion
1054         )
1055         apk['uses-permission'].append(permission)
1056
1057     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1058         name = str(item.getAttribute("android:name"))
1059         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1060         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1061         permission_sdk_23 = UsesPermissionSdk23(
1062             name,
1063             maxSdkVersion
1064         )
1065         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1066
1067     for item in xml.getElementsByTagName('uses-feature'):
1068         feature = str(item.getAttribute("android:name"))
1069         if feature != "android.hardware.screen.portrait" \
1070                 and feature != "android.hardware.screen.landscape":
1071             if feature.startswith("android.feature."):
1072                 feature = feature[16:]
1073         apk['features'].append(feature)
1074
1075
1076 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
1077     """Scan the apk with the given filename in the given repo directory.
1078
1079     This also extracts the icons.
1080
1081     :param apkcache: current apk cache information
1082     :param apkfilename: the filename of the apk to scan
1083     :param repodir: repo directory to scan
1084     :param knownapks: known apks info
1085     :param use_date_from_apk: use date from APK (instead of current date)
1086                               for newly added APKs
1087     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1088      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1089     """
1090
1091     if ' ' in apkfilename:
1092         if options.rename_apks:
1093             newfilename = apkfilename.replace(' ', '_')
1094             os.rename(os.path.join(repodir, apkfilename),
1095                       os.path.join(repodir, newfilename))
1096             apkfilename = newfilename
1097         else:
1098             logging.critical("Spaces in filenames are not allowed.")
1099             return True, None, False
1100
1101     apkfile = os.path.join(repodir, apkfilename)
1102     shasum = sha256sum(apkfile)
1103
1104     cachechanged = False
1105     usecache = False
1106     if apkfilename in apkcache:
1107         apk = apkcache[apkfilename]
1108         if apk.get('hash') == shasum:
1109             logging.debug("Reading " + apkfilename + " from cache")
1110             usecache = True
1111         else:
1112             logging.debug("Ignoring stale cache data for " + apkfilename)
1113
1114     if not usecache:
1115         logging.debug("Processing " + apkfilename)
1116         apk = {}
1117         apk['hash'] = shasum
1118         apk['hashType'] = 'sha256'
1119         apk['uses-permission'] = []
1120         apk['uses-permission-sdk-23'] = []
1121         apk['features'] = []
1122         apk['icons_src'] = {}
1123         apk['icons'] = {}
1124         apk['antiFeatures'] = set()
1125         if has_old_openssl(apkfile):
1126             apk['antiFeatures'].add('KnownVuln')
1127
1128         try:
1129             if SdkToolsPopen(['aapt', 'version'], output=False):
1130                 scan_apk_aapt(apk, apkfile)
1131             else:
1132                 scan_apk_androguard(apk, apkfile)
1133         except BuildException:
1134             return True, None, False
1135
1136         if 'minSdkVersion' not in apk:
1137             logging.warn("No SDK version information found in {0}".format(apkfile))
1138             apk['minSdkVersion'] = 1
1139
1140         # Check for debuggable apks...
1141         if common.isApkAndDebuggable(apkfile):
1142             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1143
1144         # Get the signature (or md5 of, to be precise)...
1145         logging.debug('Getting signature of {0}'.format(apkfile))
1146         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1147         if not apk['sig']:
1148             logging.critical("Failed to get apk signature")
1149             return True, None, False
1150
1151         if options.rename_apks:
1152             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1153             std_short_name = os.path.join(repodir, n)
1154             if apkfile != std_short_name:
1155                 if os.path.exists(std_short_name):
1156                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1157                     if apkfile != std_long_name:
1158                         if os.path.exists(std_long_name):
1159                             dupdir = os.path.join('duplicates', repodir)
1160                             if not os.path.isdir(dupdir):
1161                                 os.makedirs(dupdir, exist_ok=True)
1162                             dupfile = os.path.join('duplicates', std_long_name)
1163                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1164                             os.rename(apkfile, dupfile)
1165                             return True, None, False
1166                         else:
1167                             os.rename(apkfile, std_long_name)
1168                     apkfile = std_long_name
1169                 else:
1170                     os.rename(apkfile, std_short_name)
1171                     apkfile = std_short_name
1172                 apkfilename = apkfile[len(repodir) + 1:]
1173
1174         apk['apkName'] = apkfilename
1175         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1176         if os.path.exists(os.path.join(repodir, srcfilename)):
1177             apk['srcname'] = srcfilename
1178         apk['size'] = os.path.getsize(apkfile)
1179
1180         apkzip = zipfile.ZipFile(apkfile, 'r')
1181
1182         # if an APK has files newer than the system time, suggest updating
1183         # the system clock.  This is useful for offline systems, used for
1184         # signing, which do not have another source of clock sync info. It
1185         # has to be more than 24 hours newer because ZIP/APK files do not
1186         # store timezone info
1187         manifest = apkzip.getinfo('AndroidManifest.xml')
1188         if manifest.date_time[1] == 0:  # month can't be zero
1189             logging.debug('AndroidManifest.xml has no date')
1190         else:
1191             dt_obj = datetime(*manifest.date_time)
1192             checkdt = dt_obj - timedelta(1)
1193             if datetime.today() < checkdt:
1194                 logging.warn('System clock is older than manifest in: '
1195                              + apkfilename
1196                              + '\nSet clock to that time using:\n'
1197                              + 'sudo date -s "' + str(dt_obj) + '"')
1198
1199         iconfilename = "%s.%s.png" % (
1200             apk['packageName'],
1201             apk['versionCode'])
1202
1203         # Extract the icon file...
1204         empty_densities = []
1205         for density in screen_densities:
1206             if density not in apk['icons_src']:
1207                 empty_densities.append(density)
1208                 continue
1209             iconsrc = apk['icons_src'][density]
1210             icon_dir = get_icon_dir(repodir, density)
1211             icondest = os.path.join(icon_dir, iconfilename)
1212
1213             try:
1214                 with open(icondest, 'wb') as f:
1215                     f.write(get_icon_bytes(apkzip, iconsrc))
1216                 apk['icons'][density] = iconfilename
1217             except (zipfile.BadZipFile, ValueError, KeyError) as e:
1218                 logging.warning("Error retrieving icon file: %s" % (icondest))
1219                 del apk['icons_src'][density]
1220                 empty_densities.append(density)
1221
1222         if '-1' in apk['icons_src']:
1223             iconsrc = apk['icons_src']['-1']
1224             iconpath = os.path.join(
1225                 get_icon_dir(repodir, '0'), iconfilename)
1226             with open(iconpath, 'wb') as f:
1227                 f.write(get_icon_bytes(apkzip, iconsrc))
1228             try:
1229                 im = Image.open(iconpath)
1230                 dpi = px_to_dpi(im.size[0])
1231                 for density in screen_densities:
1232                     if density in apk['icons']:
1233                         break
1234                     if density == screen_densities[-1] or dpi >= int(density):
1235                         apk['icons'][density] = iconfilename
1236                         shutil.move(iconpath,
1237                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
1238                         empty_densities.remove(density)
1239                         break
1240             except Exception as e:
1241                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1242
1243         if apk['icons']:
1244             apk['icon'] = iconfilename
1245
1246         apkzip.close()
1247
1248         # First try resizing down to not lose quality
1249         last_density = None
1250         for density in screen_densities:
1251             if density not in empty_densities:
1252                 last_density = density
1253                 continue
1254             if last_density is None:
1255                 continue
1256             logging.debug("Density %s not available, resizing down from %s"
1257                           % (density, last_density))
1258
1259             last_iconpath = os.path.join(
1260                 get_icon_dir(repodir, last_density), iconfilename)
1261             iconpath = os.path.join(
1262                 get_icon_dir(repodir, density), iconfilename)
1263             fp = None
1264             try:
1265                 fp = open(last_iconpath, 'rb')
1266                 im = Image.open(fp)
1267
1268                 size = dpi_to_px(density)
1269
1270                 im.thumbnail((size, size), Image.ANTIALIAS)
1271                 im.save(iconpath, "PNG")
1272                 empty_densities.remove(density)
1273             except Exception as e:
1274                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1275             finally:
1276                 if fp:
1277                     fp.close()
1278
1279         # Then just copy from the highest resolution available
1280         last_density = None
1281         for density in reversed(screen_densities):
1282             if density not in empty_densities:
1283                 last_density = density
1284                 continue
1285             if last_density is None:
1286                 continue
1287             logging.debug("Density %s not available, copying from lower density %s"
1288                           % (density, last_density))
1289
1290             shutil.copyfile(
1291                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1292                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1293
1294             empty_densities.remove(density)
1295
1296         for density in screen_densities:
1297             icon_dir = get_icon_dir(repodir, density)
1298             icondest = os.path.join(icon_dir, iconfilename)
1299             resize_icon(icondest, density)
1300
1301         # Copy from icons-mdpi to icons since mdpi is the baseline density
1302         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1303         if os.path.isfile(baseline):
1304             apk['icons']['0'] = iconfilename
1305             shutil.copyfile(baseline,
1306                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1307
1308         if use_date_from_apk and manifest.date_time[1] != 0:
1309             default_date_param = datetime(*manifest.date_time)
1310         else:
1311             default_date_param = None
1312
1313         # Record in known apks, getting the added date at the same time..
1314         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1315                                     default_date=default_date_param)
1316         if added:
1317             apk['added'] = added
1318
1319         apkcache[apkfilename] = apk
1320         cachechanged = True
1321
1322     return False, apk, cachechanged
1323
1324
1325 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1326     """Scan the apks in the given repo directory.
1327
1328     This also extracts the icons.
1329
1330     :param apkcache: current apk cache information
1331     :param repodir: repo directory to scan
1332     :param knownapks: known apks info
1333     :param use_date_from_apk: use date from APK (instead of current date)
1334                               for newly added APKs
1335     :returns: (apks, cachechanged) where apks is a list of apk information,
1336               and cachechanged is True if the apkcache got changed.
1337     """
1338
1339     cachechanged = False
1340
1341     for icon_dir in get_all_icon_dirs(repodir):
1342         if os.path.exists(icon_dir):
1343             if options.clean:
1344                 shutil.rmtree(icon_dir)
1345                 os.makedirs(icon_dir)
1346         else:
1347             os.makedirs(icon_dir)
1348
1349     apks = []
1350     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1351         apkfilename = apkfile[len(repodir) + 1:]
1352         (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1353         if skip:
1354             continue
1355         apks.append(apk)
1356
1357     return apks, cachechanged
1358
1359
1360 def apply_info_from_latest_apk(apps, apks):
1361     """
1362     Some information from the apks needs to be applied up to the application level.
1363     When doing this, we use the info from the most recent version's apk.
1364     We deal with figuring out when the app was added and last updated at the same time.
1365     """
1366     for appid, app in apps.items():
1367         bestver = UNSET_VERSION_CODE
1368         for apk in apks:
1369             if apk['packageName'] == appid:
1370                 if apk['versionCode'] > bestver:
1371                     bestver = apk['versionCode']
1372                     bestapk = apk
1373
1374                 if 'added' in apk:
1375                     if not app.added or apk['added'] < app.added:
1376                         app.added = apk['added']
1377                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1378                         app.lastUpdated = apk['added']
1379
1380         if not app.added:
1381             logging.debug("Don't know when " + appid + " was added")
1382         if not app.lastUpdated:
1383             logging.debug("Don't know when " + appid + " was last updated")
1384
1385         if bestver == UNSET_VERSION_CODE:
1386
1387             if app.Name is None:
1388                 app.Name = app.AutoName or appid
1389             app.icon = None
1390             logging.debug("Application " + appid + " has no packages")
1391         else:
1392             if app.Name is None:
1393                 app.Name = bestapk['name']
1394             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1395             if app.CurrentVersionCode is None:
1396                 app.CurrentVersionCode = str(bestver)
1397
1398
1399 def make_categories_txt(repodir, categories):
1400     '''Write a category list in the repo to allow quick access'''
1401     catdata = ''
1402     for cat in sorted(categories):
1403         catdata += cat + '\n'
1404     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1405         f.write(catdata)
1406
1407
1408 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1409
1410     for appid, app in apps.items():
1411
1412         if app.ArchivePolicy:
1413             keepversions = int(app.ArchivePolicy[:-9])
1414         else:
1415             keepversions = defaultkeepversions
1416
1417         def filter_apk_list_sorted(apk_list):
1418             res = []
1419             for apk in apk_list:
1420                 if apk['packageName'] == appid:
1421                     res.append(apk)
1422
1423             # Sort the apk list by version code. First is highest/newest.
1424             return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1425
1426         def move_file(from_dir, to_dir, filename, ignore_missing):
1427             from_path = os.path.join(from_dir, filename)
1428             if ignore_missing and not os.path.exists(from_path):
1429                 return
1430             to_path = os.path.join(to_dir, filename)
1431             shutil.move(from_path, to_path)
1432
1433         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1434                       .format(appid, len(apks), keepversions, len(archapks)))
1435
1436         if len(apks) > keepversions:
1437             apklist = filter_apk_list_sorted(apks)
1438             # Move back the ones we don't want.
1439             for apk in apklist[keepversions:]:
1440                 logging.info("Moving " + apk['apkName'] + " to archive")
1441                 move_file(repodir, archivedir, apk['apkName'], False)
1442                 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1443                 for density in all_screen_densities:
1444                     repo_icon_dir = get_icon_dir(repodir, density)
1445                     archive_icon_dir = get_icon_dir(archivedir, density)
1446                     if density not in apk['icons']:
1447                         continue
1448                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1449                 if 'srcname' in apk:
1450                     move_file(repodir, archivedir, apk['srcname'], False)
1451                 archapks.append(apk)
1452                 apks.remove(apk)
1453         elif len(apks) < keepversions and len(archapks) > 0:
1454             required = keepversions - len(apks)
1455             archapklist = filter_apk_list_sorted(archapks)
1456             # Move forward the ones we want again.
1457             for apk in archapklist[:required]:
1458                 logging.info("Moving " + apk['apkName'] + " from archive")
1459                 move_file(archivedir, repodir, apk['apkName'], False)
1460                 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1461                 for density in all_screen_densities:
1462                     repo_icon_dir = get_icon_dir(repodir, density)
1463                     archive_icon_dir = get_icon_dir(archivedir, density)
1464                     if density not in apk['icons']:
1465                         continue
1466                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1467                 if 'srcname' in apk:
1468                     move_file(archivedir, repodir, apk['srcname'], False)
1469                 archapks.remove(apk)
1470                 apks.append(apk)
1471
1472
1473 def add_apks_to_per_app_repos(repodir, apks):
1474     apks_per_app = dict()
1475     for apk in apks:
1476         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1477         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1478         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1479         apks_per_app[apk['packageName']] = apk
1480
1481         if not os.path.exists(apk['per_app_icons']):
1482             logging.info('Adding new repo for only ' + apk['packageName'])
1483             os.makedirs(apk['per_app_icons'])
1484
1485         apkpath = os.path.join(repodir, apk['apkName'])
1486         shutil.copy(apkpath, apk['per_app_repo'])
1487         apksigpath = apkpath + '.sig'
1488         if os.path.exists(apksigpath):
1489             shutil.copy(apksigpath, apk['per_app_repo'])
1490         apkascpath = apkpath + '.asc'
1491         if os.path.exists(apkascpath):
1492             shutil.copy(apkascpath, apk['per_app_repo'])
1493
1494
1495 config = None
1496 options = None
1497
1498
1499 def main():
1500
1501     global config, options
1502
1503     # Parse command line...
1504     parser = ArgumentParser()
1505     common.setup_global_opts(parser)
1506     parser.add_argument("--create-key", action="store_true", default=False,
1507                         help="Create a repo signing key in a keystore")
1508     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1509                         help="Create skeleton metadata files that are missing")
1510     parser.add_argument("--delete-unknown", action="store_true", default=False,
1511                         help="Delete APKs and/or OBBs without metadata from the repo")
1512     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1513                         help="Report on build data status")
1514     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1515                         help="Interactively ask about things that need updating.")
1516     parser.add_argument("-I", "--icons", action="store_true", default=False,
1517                         help="Resize all the icons exceeding the max pixel size and exit")
1518     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1519                         help="Specify editor to use in interactive mode. Default " +
1520                         "is /etc/alternatives/editor")
1521     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1522                         help="Update the wiki")
1523     parser.add_argument("--pretty", action="store_true", default=False,
1524                         help="Produce human-readable index.xml")
1525     parser.add_argument("--clean", action="store_true", default=False,
1526                         help="Clean update - don't uses caches, reprocess all apks")
1527     parser.add_argument("--nosign", action="store_true", default=False,
1528                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1529     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1530                         help="Use date from apk instead of current time for newly added apks")
1531     parser.add_argument("--rename-apks", action="store_true", default=False,
1532                         help="Rename APK files that do not match package.name_123.apk")
1533     metadata.add_metadata_arguments(parser)
1534     options = parser.parse_args()
1535     metadata.warnings_action = options.W
1536
1537     config = common.read_config(options)
1538
1539     if not ('jarsigner' in config and 'keytool' in config):
1540         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1541
1542     repodirs = ['repo']
1543     if config['archive_older'] != 0:
1544         repodirs.append('archive')
1545         if not os.path.exists('archive'):
1546             os.mkdir('archive')
1547
1548     if options.icons:
1549         resize_all_icons(repodirs)
1550         sys.exit(0)
1551
1552     if options.rename_apks:
1553         options.clean = True
1554
1555     # check that icons exist now, rather than fail at the end of `fdroid update`
1556     for k in ['repo_icon', 'archive_icon']:
1557         if k in config:
1558             if not os.path.exists(config[k]):
1559                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1560                 sys.exit(1)
1561
1562     # if the user asks to create a keystore, do it now, reusing whatever it can
1563     if options.create_key:
1564         if os.path.exists(config['keystore']):
1565             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1566             logging.critical("\t'" + config['keystore'] + "'")
1567             sys.exit(1)
1568
1569         if 'repo_keyalias' not in config:
1570             config['repo_keyalias'] = socket.getfqdn()
1571             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1572         if 'keydname' not in config:
1573             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1574             common.write_to_config(config, 'keydname', config['keydname'])
1575         if 'keystore' not in config:
1576             config['keystore'] = common.default_config['keystore']
1577             common.write_to_config(config, 'keystore', config['keystore'])
1578
1579         password = common.genpassword()
1580         if 'keystorepass' not in config:
1581             config['keystorepass'] = password
1582             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1583         if 'keypass' not in config:
1584             config['keypass'] = password
1585             common.write_to_config(config, 'keypass', config['keypass'])
1586         common.genkeystore(config)
1587
1588     # Get all apps...
1589     apps = metadata.read_metadata()
1590
1591     # Generate a list of categories...
1592     categories = set()
1593     for app in apps.values():
1594         categories.update(app.Categories)
1595
1596     # Read known apks data (will be updated and written back when we've finished)
1597     knownapks = common.KnownApks()
1598
1599     # Get APK cache
1600     apkcache = get_cache()
1601
1602     # Delete builds for disabled apps
1603     delete_disabled_builds(apps, apkcache, repodirs)
1604
1605     # Scan all apks in the main repo
1606     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1607
1608     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1609                                            options.use_date_from_apk)
1610     cachechanged = cachechanged or fcachechanged
1611     apks += files
1612     # Generate warnings for apk's with no metadata (or create skeleton
1613     # metadata files, if requested on the command line)
1614     newmetadata = False
1615     for apk in apks:
1616         if apk['packageName'] not in apps:
1617             if options.create_metadata:
1618                 if 'name' not in apk:
1619                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1620                     continue
1621                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1622                 f.write("License:Unknown\n")
1623                 f.write("Web Site:\n")
1624                 f.write("Source Code:\n")
1625                 f.write("Issue Tracker:\n")
1626                 f.write("Changelog:\n")
1627                 f.write("Summary:" + apk['name'] + "\n")
1628                 f.write("Description:\n")
1629                 f.write(apk['name'] + "\n")
1630                 f.write(".\n")
1631                 f.write("Name:" + apk['name'] + "\n")
1632                 f.close()
1633                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1634                 newmetadata = True
1635             else:
1636                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1637                 if options.delete_unknown:
1638                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1639                     rmf = os.path.join(repodirs[0], apk['apkName'])
1640                     if not os.path.exists(rmf):
1641                         logging.error("Could not find {0} to remove it".format(rmf))
1642                     else:
1643                         os.remove(rmf)
1644                 else:
1645                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1646
1647     # update the metadata with the newly created ones included
1648     if newmetadata:
1649         apps = metadata.read_metadata()
1650
1651     copy_triple_t_store_metadata(apps)
1652     insert_obbs(repodirs[0], apps, apks)
1653     insert_localized_app_metadata(apps)
1654
1655     # Scan the archive repo for apks as well
1656     if len(repodirs) > 1:
1657         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1658         if cc:
1659             cachechanged = True
1660     else:
1661         archapks = []
1662
1663     # Apply information from latest apks to the application and update dates
1664     apply_info_from_latest_apk(apps, apks + archapks)
1665
1666     # Sort the app list by name, then the web site doesn't have to by default.
1667     # (we had to wait until we'd scanned the apks to do this, because mostly the
1668     # name comes from there!)
1669     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1670
1671     # APKs are placed into multiple repos based on the app package, providing
1672     # per-app subscription feeds for nightly builds and things like it
1673     if config['per_app_repos']:
1674         add_apks_to_per_app_repos(repodirs[0], apks)
1675         for appid, app in apps.items():
1676             repodir = os.path.join(appid, 'fdroid', 'repo')
1677             appdict = dict()
1678             appdict[appid] = app
1679             if os.path.isdir(repodir):
1680                 index.make(appdict, [appid], apks, repodir, False)
1681             else:
1682                 logging.info('Skipping index generation for ' + appid)
1683         return
1684
1685     if len(repodirs) > 1:
1686         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1687
1688     # Make the index for the main repo...
1689     index.make(apps, sortedids, apks, repodirs[0], False)
1690     make_categories_txt(repodirs[0], categories)
1691
1692     # If there's an archive repo,  make the index for it. We already scanned it
1693     # earlier on.
1694     if len(repodirs) > 1:
1695         index.make(apps, sortedids, archapks, repodirs[1], True)
1696
1697     git_remote = config.get('binary_transparency_remote')
1698     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1699         btlog.make_binary_transparency_log(repodirs)
1700
1701     if config['update_stats']:
1702         # Update known apks info...
1703         knownapks.writeifchanged()
1704
1705         # Generate latest apps data for widget
1706         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1707             data = ''
1708             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1709                 for line in f:
1710                     appid = line.rstrip()
1711                     data += appid + "\t"
1712                     app = apps[appid]
1713                     data += app.Name + "\t"
1714                     if app.icon is not None:
1715                         data += app.icon + "\t"
1716                     data += app.License + "\n"
1717             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1718                 f.write(data)
1719
1720     if cachechanged:
1721         write_cache(apkcache)
1722
1723     # Update the wiki...
1724     if options.wiki:
1725         update_wiki(apps, sortedids, apks + archapks)
1726
1727     logging.info("Finished.")
1728
1729
1730 if __name__ == "__main__":
1731     main()