fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import btlog
  41 from . import common
  42 from . import index
  43 from . import metadata
  44 from .common import SdkToolsPopen
  45 from .exception import BuildException, FDroidException
  46
  47 METADATA_VERSION = 18
  48
  49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  50 UNSET_VERSION_CODE = -0x100000000
  51
  52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  59 APK_PERMISSION_PAT = \
  60     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  62
  63 screen_densities = ['640', '480', '320', '240', '160', '120']
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "undefined": '-1',
  72     "anydpi": '65534',
  73     "nodpi": '65535'
  74 }
  75
  76 all_screen_densities = ['0'] + screen_densities
  77
  78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  80
  81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  84                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  85
  86
  87 def dpi_to_px(density):
  88     return (int(density) * 48) / 160
  89
  90
  91 def px_to_dpi(px):
  92     return (int(px) * 160) / 48
  93
  94
  95 def get_icon_dir(repodir, density):
  96     if density == '0':
  97         return os.path.join(repodir, "icons")
  98     return os.path.join(repodir, "icons-%s" % density)
  99
 100
 101 def get_icon_dirs(repodir):
 102     for density in screen_densities:
 103         yield get_icon_dir(repodir, density)
 104
 105
 106 def get_all_icon_dirs(repodir):
 107     for density in all_screen_densities:
 108         yield get_icon_dir(repodir, density)
 109
 110
 111 def update_wiki(apps, sortedids, apks):
 112     """Update the wiki
 113
 114     :param apps: fully populated list of all applications
 115     :param apks: all apks, except...
 116     """
 117     logging.info("Updating wiki")
 118     wikicat = 'Apps'
 119     wikiredircat = 'App Redirects'
 120     import mwclient
 121     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 122                          path=config['wiki_path'])
 123     site.login(config['wiki_user'], config['wiki_password'])
 124     generated_pages = {}
 125     generated_redirects = {}
 126
 127     for appid in sortedids:
 128         app = metadata.App(apps[appid])
 129
 130         wikidata = ''
 131         if app.Disabled:
 132             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 133         if app.AntiFeatures:
 134             for af in app.AntiFeatures:
 135                 wikidata += '{{AntiFeature|' + af + '}}\n'
 136         if app.RequiresRoot:
 137             requiresroot = 'Yes'
 138         else:
 139             requiresroot = 'No'
 140         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 141             appid,
 142             app.Name,
 143             app.added.strftime('%Y-%m-%d') if app.added else '',
 144             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 145             app.SourceCode,
 146             app.IssueTracker,
 147             app.WebSite,
 148             app.Changelog,
 149             app.Donate,
 150             app.FlattrID,
 151             app.Bitcoin,
 152             app.Litecoin,
 153             app.License,
 154             requiresroot,
 155             app.AuthorName,
 156             app.AuthorEmail)
 157
 158         if app.Provides:
 159             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 160
 161         wikidata += app.Summary
 162         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 163
 164         wikidata += "=Description=\n"
 165         wikidata += metadata.description_wiki(app.Description) + "\n"
 166
 167         wikidata += "=Maintainer Notes=\n"
 168         if app.MaintainerNotes:
 169             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 170         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 171
 172         # Get a list of all packages for this application...
 173         apklist = []
 174         gotcurrentver = False
 175         cantupdate = False
 176         buildfails = False
 177         for apk in apks:
 178             if apk['packageName'] == appid:
 179                 if str(apk['versionCode']) == app.CurrentVersionCode:
 180                     gotcurrentver = True
 181                 apklist.append(apk)
 182         # Include ones we can't build, as a special case...
 183         for build in app.builds:
 184             if build.disable:
 185                 if build.versionCode == app.CurrentVersionCode:
 186                     cantupdate = True
 187                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 188                 apklist.append({'versionCode': int(build.versionCode),
 189                                 'versionName': build.versionName,
 190                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 191                                 })
 192             else:
 193                 builtit = False
 194                 for apk in apklist:
 195                     if apk['versionCode'] == int(build.versionCode):
 196                         builtit = True
 197                         break
 198                 if not builtit:
 199                     buildfails = True
 200                     apklist.append({'versionCode': int(build.versionCode),
 201                                     'versionName': build.versionName,
 202                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 203                                     })
 204         if app.CurrentVersionCode == '0':
 205             cantupdate = True
 206         # Sort with most recent first...
 207         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 208
 209         wikidata += "=Versions=\n"
 210         if len(apklist) == 0:
 211             wikidata += "We currently have no versions of this app available."
 212         elif not gotcurrentver:
 213             wikidata += "We don't have the current version of this app."
 214         else:
 215             wikidata += "We have the current version of this app."
 216         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 217         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 218         if len(app.NoSourceSince) > 0:
 219             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 220         if len(app.CurrentVersion) > 0:
 221             wikidata += "The current (recommended) version is " + app.CurrentVersion
 222             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 223         validapks = 0
 224         for apk in apklist:
 225             wikidata += "==" + apk['versionName'] + "==\n"
 226
 227             if 'buildproblem' in apk:
 228                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 229             else:
 230                 validapks += 1
 231                 wikidata += "This version is built and signed by "
 232                 if 'srcname' in apk:
 233                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 234                 else:
 235                     wikidata += "the original developer.\n\n"
 236             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 237
 238         wikidata += '\n[[Category:' + wikicat + ']]\n'
 239         if len(app.NoSourceSince) > 0:
 240             wikidata += '\n[[Category:Apps missing source code]]\n'
 241         if validapks == 0 and not app.Disabled:
 242             wikidata += '\n[[Category:Apps with no packages]]\n'
 243         if cantupdate and not app.Disabled:
 244             wikidata += "\n[[Category:Apps we cannot update]]\n"
 245         if buildfails and not app.Disabled:
 246             wikidata += "\n[[Category:Apps with failing builds]]\n"
 247         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 248             wikidata += '\n[[Category:Apps to Update]]\n'
 249         if app.Disabled:
 250             wikidata += '\n[[Category:Apps that are disabled]]\n'
 251         if app.UpdateCheckMode == 'None' and not app.Disabled:
 252             wikidata += '\n[[Category:Apps with no update check]]\n'
 253         for appcat in app.Categories:
 254             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 255
 256         # We can't have underscores in the page name, even if they're in
 257         # the package ID, because MediaWiki messes with them...
 258         pagename = appid.replace('_', ' ')
 259
 260         # Drop a trailing newline, because mediawiki is going to drop it anyway
 261         # and it we don't we'll think the page has changed when it hasn't...
 262         if wikidata.endswith('\n'):
 263             wikidata = wikidata[:-1]
 264
 265         generated_pages[pagename] = wikidata
 266
 267         # Make a redirect from the name to the ID too, unless there's
 268         # already an existing page with the name and it isn't a redirect.
 269         noclobber = False
 270         apppagename = app.Name.replace('_', ' ')
 271         apppagename = apppagename.replace('{', '')
 272         apppagename = apppagename.replace('}', ' ')
 273         apppagename = apppagename.replace(':', ' ')
 274         apppagename = apppagename.replace('[', ' ')
 275         apppagename = apppagename.replace(']', ' ')
 276         # Drop double spaces caused mostly by replacing ':' above
 277         apppagename = apppagename.replace('  ', ' ')
 278         for expagename in site.allpages(prefix=apppagename,
 279                                         filterredir='nonredirects',
 280                                         generator=False):
 281             if expagename == apppagename:
 282                 noclobber = True
 283         # Another reason not to make the redirect page is if the app name
 284         # is the same as it's ID, because that will overwrite the real page
 285         # with an redirect to itself! (Although it seems like an odd
 286         # scenario this happens a lot, e.g. where there is metadata but no
 287         # builds or binaries to extract a name from.
 288         if apppagename == pagename:
 289             noclobber = True
 290         if not noclobber:
 291             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 292
 293     for tcat, genp in [(wikicat, generated_pages),
 294                        (wikiredircat, generated_redirects)]:
 295         catpages = site.Pages['Category:' + tcat]
 296         existingpages = []
 297         for page in catpages:
 298             existingpages.append(page.name)
 299             if page.name in genp:
 300                 pagetxt = page.edit()
 301                 if pagetxt != genp[page.name]:
 302                     logging.debug("Updating modified page " + page.name)
 303                     page.save(genp[page.name], summary='Auto-updated')
 304                 else:
 305                     logging.debug("Page " + page.name + " is unchanged")
 306             else:
 307                 logging.warn("Deleting page " + page.name)
 308                 page.delete('No longer published')
 309         for pagename, text in genp.items():
 310             logging.debug("Checking " + pagename)
 311             if pagename not in existingpages:
 312                 logging.debug("Creating page " + pagename)
 313                 try:
 314                     newpage = site.Pages[pagename]
 315                     newpage.save(text, summary='Auto-created')
 316                 except Exception as e:
 317                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 318
 319     # Purge server cache to ensure counts are up to date
 320     site.pages['Repository Maintenance'].purge()
 321
 322
 323 def delete_disabled_builds(apps, apkcache, repodirs):
 324     """Delete disabled build outputs.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodirs: the repo directories to process
 329     """
 330     for appid, app in apps.items():
 331         for build in app['builds']:
 332             if not build.disable:
 333                 continue
 334             apkfilename = common.get_release_filename(app, build)
 335             iconfilename = "%s.%s.png" % (
 336                 appid,
 337                 build.versionCode)
 338             for repodir in repodirs:
 339                 files = [
 340                     os.path.join(repodir, apkfilename),
 341                     os.path.join(repodir, apkfilename + '.asc'),
 342                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 343                 ]
 344                 for density in all_screen_densities:
 345                     repo_dir = get_icon_dir(repodir, density)
 346                     files.append(os.path.join(repo_dir, iconfilename))
 347
 348                 for f in files:
 349                     if os.path.exists(f):
 350                         logging.info("Deleting disabled build output " + f)
 351                         os.remove(f)
 352             if apkfilename in apkcache:
 353                 del apkcache[apkfilename]
 354
 355
 356 def resize_icon(iconpath, density):
 357
 358     if not os.path.isfile(iconpath):
 359         return
 360
 361     fp = None
 362     try:
 363         fp = open(iconpath, 'rb')
 364         im = Image.open(fp)
 365         size = dpi_to_px(density)
 366
 367         if any(length > size for length in im.size):
 368             oldsize = im.size
 369             im.thumbnail((size, size), Image.ANTIALIAS)
 370             logging.debug("%s was too large at %s - new size is %s" % (
 371                 iconpath, oldsize, im.size))
 372             im.save(iconpath, "PNG")
 373
 374     except Exception as e:
 375         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 376
 377     finally:
 378         if fp:
 379             fp.close()
 380
 381
 382 def resize_all_icons(repodirs):
 383     """Resize all icons that exceed the max size
 384
 385     :param repodirs: the repo directories to process
 386     """
 387     for repodir in repodirs:
 388         for density in screen_densities:
 389             icon_dir = get_icon_dir(repodir, density)
 390             icon_glob = os.path.join(icon_dir, '*.png')
 391             for iconpath in glob.glob(icon_glob):
 392                 resize_icon(iconpath, density)
 393
 394
 395 def getsig(apkpath):
 396     """ Get the signing certificate of an apk. To get the same md5 has that
 397     Android gets, we encode the .RSA certificate in a specific format and pass
 398     it hex-encoded to the md5 digest algorithm.
 399
 400     :param apkpath: path to the apk
 401     :returns: A string containing the md5 of the signature of the apk or None
 402               if an error occurred.
 403     """
 404
 405     with zipfile.ZipFile(apkpath, 'r') as apk:
 406         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 407
 408         if len(certs) < 1:
 409             logging.error("Found no signing certificates on %s" % apkpath)
 410             return None
 411         if len(certs) > 1:
 412             logging.error("Found multiple signing certificates on %s" % apkpath)
 413             return None
 414
 415         cert = apk.read(certs[0])
 416
 417     cert_encoded = common.get_certificate(cert)
 418
 419     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 420
 421
 422 def get_cache_file():
 423     return os.path.join('tmp', 'apkcache')
 424
 425
 426 def get_cache():
 427     """
 428     Gather information about all the apk files in the repo directory,
 429     using cached data if possible.
 430     :return: apkcache
 431     """
 432     apkcachefile = get_cache_file()
 433     if not options.clean and os.path.exists(apkcachefile):
 434         with open(apkcachefile, 'rb') as cf:
 435             apkcache = pickle.load(cf, encoding='utf-8')
 436         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
 437             apkcache = {}
 438     else:
 439         apkcache = {}
 440
 441     return apkcache
 442
 443
 444 def write_cache(apkcache):
 445     apkcachefile = get_cache_file()
 446     cache_path = os.path.dirname(apkcachefile)
 447     if not os.path.exists(cache_path):
 448         os.makedirs(cache_path)
 449     apkcache["METADATA_VERSION"] = METADATA_VERSION
 450     with open(apkcachefile, 'wb') as cf:
 451         pickle.dump(apkcache, cf)
 452
 453
 454 def get_icon_bytes(apkzip, iconsrc):
 455     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 456     try:
 457         return apkzip.read(iconsrc)
 458     except KeyError:
 459         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 460
 461
 462 def sha256sum(filename):
 463     '''Calculate the sha256 of the given file'''
 464     sha = hashlib.sha256()
 465     with open(filename, 'rb') as f:
 466         while True:
 467             t = f.read(16384)
 468             if len(t) == 0:
 469                 break
 470             sha.update(t)
 471     return sha.hexdigest()
 472
 473
 474 def has_old_openssl(filename):
 475     '''checks for known vulnerable openssl versions in the APK'''
 476
 477     # statically load this pattern
 478     if not hasattr(has_old_openssl, "pattern"):
 479         has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 480
 481     with zipfile.ZipFile(filename) as zf:
 482         for name in zf.namelist():
 483             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 484                 lib = zf.open(name)
 485                 while True:
 486                     chunk = lib.read(4096)
 487                     if chunk == b'':
 488                         break
 489                     m = has_old_openssl.pattern.search(chunk)
 490                     if m:
 491                         version = m.group(1).decode('ascii')
 492                         if version.startswith('1.0.1') and version[5] >= 'r' \
 493                            or version.startswith('1.0.2') and version[5] >= 'f':
 494                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 495                         else:
 496                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 497                             return True
 498                         break
 499     return False
 500
 501
 502 def insert_obbs(repodir, apps, apks):
 503     """Scans the .obb files in a given repo directory and adds them to the
 504     relevant APK instances.  OBB files have versionCodes like APK
 505     files, and they are loosely associated.  If there is an OBB file
 506     present, then any APK with the same or higher versionCode will use
 507     that OBB file.  There are two OBB types: main and patch, each APK
 508     can only have only have one of each.
 509
 510     https://developer.android.com/google/play/expansion-files.html
 511
 512     :param repodir: repo directory to scan
 513     :param apps: list of current, valid apps
 514     :param apks: current information on all APKs
 515
 516     """
 517
 518     def obbWarnDelete(f, msg):
 519         logging.warning(msg + f)
 520         if options.delete_unknown:
 521             logging.error("Deleting unknown file: " + f)
 522             os.remove(f)
 523
 524     obbs = []
 525     java_Integer_MIN_VALUE = -pow(2, 31)
 526     currentPackageNames = apps.keys()
 527     for f in glob.glob(os.path.join(repodir, '*.obb')):
 528         obbfile = os.path.basename(f)
 529         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 530         chunks = obbfile.split('.')
 531         if chunks[0] != 'main' and chunks[0] != 'patch':
 532             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 533             continue
 534         if not re.match(r'^-?[0-9]+$', chunks[1]):
 535             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 536             continue
 537         versionCode = int(chunks[1])
 538         packagename = ".".join(chunks[2:-1])
 539
 540         highestVersionCode = java_Integer_MIN_VALUE
 541         if packagename not in currentPackageNames:
 542             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 543             continue
 544         for apk in apks:
 545             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 546                 highestVersionCode = apk['versionCode']
 547         if versionCode > highestVersionCode:
 548             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 549                           + ') than any APK: ')
 550             continue
 551         obbsha256 = sha256sum(f)
 552         obbs.append((packagename, versionCode, obbfile, obbsha256))
 553
 554     for apk in apks:
 555         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 556             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 557                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 558                     apk['obbMainFile'] = obbfile
 559                     apk['obbMainFileSha256'] = obbsha256
 560                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 561                     apk['obbPatchFile'] = obbfile
 562                     apk['obbPatchFileSha256'] = obbsha256
 563             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 564                 break
 565
 566
 567 def _get_localized_dict(app, locale):
 568     '''get the dict to add localized store metadata to'''
 569     if 'localized' not in app:
 570         app['localized'] = collections.OrderedDict()
 571     if locale not in app['localized']:
 572         app['localized'][locale] = collections.OrderedDict()
 573     return app['localized'][locale]
 574
 575
 576 def _set_localized_text_entry(app, locale, key, f):
 577     limit = config['char_limits'][key]
 578     localized = _get_localized_dict(app, locale)
 579     with open(f) as fp:
 580         text = fp.read()[:limit]
 581         if len(text) > 0:
 582             localized[key] = text
 583
 584
 585 def _set_author_entry(app, key, f):
 586     limit = config['char_limits']['author']
 587     with open(f) as fp:
 588         text = fp.read()[:limit]
 589         if len(text) > 0:
 590             app[key] = text
 591
 592
 593 def copy_triple_t_store_metadata(apps):
 594     """Include store metadata from the app's source repo
 595
 596     The Triple-T Gradle Play Publisher is a plugin that has a standard
 597     file layout for all of the metadata and graphics that the Google
 598     Play Store accepts.  Since F-Droid has the git repo, it can just
 599     pluck those files directly.  This method reads any text files into
 600     the app dict, then copies any graphics into the fdroid repo
 601     directory structure.
 602
 603     This needs to be run before insert_localized_app_metadata() so that
 604     the graphics files that are copied into the fdroid repo get
 605     properly indexed.
 606
 607     https://github.com/Triple-T/gradle-play-publisher#upload-images
 608     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 609
 610     """
 611
 612     if not os.path.isdir('build'):
 613         return  # nothing to do
 614
 615     for packageName, app in apps.items():
 616         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 617             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 618             for root, dirs, files in os.walk(d):
 619                 segments = root.split('/')
 620                 locale = segments[-2]
 621                 for f in files:
 622                     if f == 'fulldescription':
 623                         _set_localized_text_entry(app, locale, 'description',
 624                                                   os.path.join(root, f))
 625                         continue
 626                     elif f == 'shortdescription':
 627                         _set_localized_text_entry(app, locale, 'summary',
 628                                                   os.path.join(root, f))
 629                         continue
 630                     elif f == 'title':
 631                         _set_localized_text_entry(app, locale, 'name',
 632                                                   os.path.join(root, f))
 633                         continue
 634                     elif f == 'video':
 635                         _set_localized_text_entry(app, locale, 'video',
 636                                                   os.path.join(root, f))
 637                         continue
 638                     elif f == 'whatsnew':
 639                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 640                                                   os.path.join(root, f))
 641                         continue
 642                     elif f == 'contactEmail':
 643                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 644                         continue
 645                     elif f == 'contactPhone':
 646                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 647                         continue
 648                     elif f == 'contactWebsite':
 649                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 650                         continue
 651
 652                     base, extension = common.get_extension(f)
 653                     dirname = os.path.basename(root)
 654                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 655                         if segments[-2] == 'listing':
 656                             locale = segments[-3]
 657                         else:
 658                             locale = segments[-2]
 659                         destdir = os.path.join('repo', packageName, locale)
 660                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 661                         sourcefile = os.path.join(root, f)
 662                         destfile = os.path.join(destdir, dirname + '.' + extension)
 663                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 664                         shutil.copy(sourcefile, destfile)
 665
 666
 667 def insert_localized_app_metadata(apps):
 668     """scans standard locations for graphics and localized text
 669
 670     Scans for localized description files, store graphics, and
 671     screenshot PNG files in statically defined screenshots directory
 672     and adds them to the app metadata.  The screenshots and graphic
 673     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 674     and must be in the following layout:
 675     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 676
 677     repo/packageName/locale/featureGraphic.png
 678     repo/packageName/locale/phoneScreenshots/1.png
 679     repo/packageName/locale/phoneScreenshots/2.png
 680
 681     The changelog files must be text files named with the versionCode
 682     ending with ".txt" and must be in the following layout:
 683     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 684
 685     repo/packageName/locale/changelogs/12345.txt
 686
 687     This will scan the each app's source repo then the metadata/ dir
 688     for these standard locations of changelog files.  If it finds
 689     them, they will be added to the dict of all packages, with the
 690     versions in the metadata/ folder taking precendence over the what
 691     is in the app's source repo.
 692
 693     Where "packageName" is the app's packageName and "locale" is the locale
 694     of the graphics, e.g. what language they are in, using the IETF RFC5646
 695     format (en-US, fr-CA, es-MX, etc).
 696
 697     This will also scan the app's git for a fastlane folder, and the
 698     metadata/ folder and the apps' source repos for standard locations
 699     of graphic and screenshot files.  If it finds them, it will copy
 700     them into the repo.  The fastlane files follow this pattern:
 701     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 702
 703     """
 704
 705     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 706     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 707     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 708
 709     for srcd in sorted(sourcedirs):
 710         if not os.path.isdir(srcd):
 711             continue
 712         for root, dirs, files in os.walk(srcd):
 713             segments = root.split('/')
 714             packageName = segments[1]
 715             if packageName not in apps:
 716                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 717                 continue
 718             locale = segments[-1]
 719             destdir = os.path.join('repo', packageName, locale)
 720             for f in files:
 721                 if f in ('description.txt', 'full_description.txt'):
 722                     _set_localized_text_entry(apps[packageName], locale, 'description',
 723                                               os.path.join(root, f))
 724                     continue
 725                 elif f in ('summary.txt', 'short_description.txt'):
 726                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 727                                               os.path.join(root, f))
 728                     continue
 729                 elif f in ('name.txt', 'title.txt'):
 730                     _set_localized_text_entry(apps[packageName], locale, 'name',
 731                                               os.path.join(root, f))
 732                     continue
 733                 elif f == 'video.txt':
 734                     _set_localized_text_entry(apps[packageName], locale, 'video',
 735                                               os.path.join(root, f))
 736                     continue
 737                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 738                     locale = segments[-2]
 739                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 740                                               os.path.join(root, f))
 741                     continue
 742
 743                 base, extension = common.get_extension(f)
 744                 if locale == 'images':
 745                     locale = segments[-2]
 746                     destdir = os.path.join('repo', packageName, locale)
 747                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 748                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 749                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 750                     shutil.copy(os.path.join(root, f), destdir)
 751             for d in dirs:
 752                 if d in SCREENSHOT_DIRS:
 753                     for f in glob.glob(os.path.join(root, d, '*.*')):
 754                         _, extension = common.get_extension(f)
 755                         if extension in ALLOWED_EXTENSIONS:
 756                             screenshotdestdir = os.path.join(destdir, d)
 757                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 758                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 759                             shutil.copy(f, screenshotdestdir)
 760
 761     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 762     for d in repofiles:
 763         if not os.path.isdir(d):
 764             continue
 765         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 766             if not os.path.isfile(f):
 767                 continue
 768             segments = f.split('/')
 769             packageName = segments[1]
 770             locale = segments[2]
 771             screenshotdir = segments[3]
 772             filename = os.path.basename(f)
 773             base, extension = common.get_extension(filename)
 774
 775             if packageName not in apps:
 776                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 777                                 % (filename, packageName))
 778                 continue
 779             graphics = _get_localized_dict(apps[packageName], locale)
 780
 781             if extension not in ALLOWED_EXTENSIONS:
 782                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 783             elif base in GRAPHIC_NAMES:
 784                 # there can only be zero or one of these per locale
 785                 graphics[base] = filename
 786             elif screenshotdir in SCREENSHOT_DIRS:
 787                 # there can any number of these per locale
 788                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 789                 if screenshotdir not in graphics:
 790                     graphics[screenshotdir] = []
 791                 graphics[screenshotdir].append(filename)
 792             else:
 793                 logging.warning('Unsupported graphics file found: ' + f)
 794
 795
 796 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 797     """Scan a repo for all files with an extension except APK/OBB
 798
 799     :param apkcache: current cached info about all repo files
 800     :param repodir: repo directory to scan
 801     :param knownapks: list of all known files, as per metadata.read_metadata
 802     :param use_date_from_file: use date from file (instead of current date)
 803                                for newly added files
 804     """
 805
 806     cachechanged = False
 807     repo_files = []
 808     repodir = repodir.encode('utf-8')
 809     for name in os.listdir(repodir):
 810         file_extension = common.get_file_extension(name)
 811         if file_extension == 'apk' or file_extension == 'obb':
 812             continue
 813         filename = os.path.join(repodir, name)
 814         name_utf8 = name.decode('utf-8')
 815         if filename.endswith(b'_src.tar.gz'):
 816             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 817             continue
 818         if not common.is_repo_file(filename):
 819             continue
 820         stat = os.stat(filename)
 821         if stat.st_size == 0:
 822             raise FDroidException(filename + ' is zero size!')
 823
 824         shasum = sha256sum(filename)
 825         usecache = False
 826         if name in apkcache:
 827             repo_file = apkcache[name]
 828             # added time is cached as tuple but used here as datetime instance
 829             if 'added' in repo_file:
 830                 a = repo_file['added']
 831                 if isinstance(a, datetime):
 832                     repo_file['added'] = a
 833                 else:
 834                     repo_file['added'] = datetime(*a[:6])
 835             if repo_file.get('hash') == shasum:
 836                 logging.debug("Reading " + name_utf8 + " from cache")
 837                 usecache = True
 838             else:
 839                 logging.debug("Ignoring stale cache data for " + name)
 840
 841         if not usecache:
 842             logging.debug("Processing " + name_utf8)
 843             repo_file = collections.OrderedDict()
 844             repo_file['name'] = os.path.splitext(name_utf8)[0]
 845             # TODO rename apkname globally to something more generic
 846             repo_file['apkName'] = name_utf8
 847             repo_file['hash'] = shasum
 848             repo_file['hashType'] = 'sha256'
 849             repo_file['versionCode'] = 0
 850             repo_file['versionName'] = shasum
 851             # the static ID is the SHA256 unless it is set in the metadata
 852             repo_file['packageName'] = shasum
 853
 854             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 855             if m:
 856                 repo_file['packageName'] = m.group(1)
 857                 repo_file['versionCode'] = int(m.group(2))
 858             srcfilename = name + b'_src.tar.gz'
 859             if os.path.exists(os.path.join(repodir, srcfilename)):
 860                 repo_file['srcname'] = srcfilename.decode('utf-8')
 861             repo_file['size'] = stat.st_size
 862
 863             apkcache[name] = repo_file
 864             cachechanged = True
 865
 866         if use_date_from_file:
 867             timestamp = stat.st_ctime
 868             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 869         else:
 870             default_date_param = None
 871
 872         # Record in knownapks, getting the added date at the same time..
 873         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 874                                     default_date=default_date_param)
 875         if added:
 876             repo_file['added'] = added
 877
 878         repo_files.append(repo_file)
 879
 880     return repo_files, cachechanged
 881
 882
 883 def scan_apk_aapt(apk, apkfile):
 884     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 885     if p.returncode != 0:
 886         if options.delete_unknown:
 887             if os.path.exists(apkfile):
 888                 logging.error("Failed to get apk information, deleting " + apkfile)
 889                 os.remove(apkfile)
 890             else:
 891                 logging.error("Could not find {0} to remove it".format(apkfile))
 892         else:
 893             logging.error("Failed to get apk information, skipping " + apkfile)
 894         raise BuildException("Invalid APK")
 895     for line in p.output.splitlines():
 896         if line.startswith("package:"):
 897             try:
 898                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 899                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 900                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 901             except Exception as e:
 902                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
 903         elif line.startswith("application:"):
 904             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 905             # Keep path to non-dpi icon in case we need it
 906             match = re.match(APK_ICON_PAT_NODPI, line)
 907             if match:
 908                 apk['icons_src']['-1'] = match.group(1)
 909         elif line.startswith("launchable-activity:"):
 910             # Only use launchable-activity as fallback to application
 911             if not apk['name']:
 912                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 913             if '-1' not in apk['icons_src']:
 914                 match = re.match(APK_ICON_PAT_NODPI, line)
 915                 if match:
 916                     apk['icons_src']['-1'] = match.group(1)
 917         elif line.startswith("application-icon-"):
 918             match = re.match(APK_ICON_PAT, line)
 919             if match:
 920                 density = match.group(1)
 921                 path = match.group(2)
 922                 apk['icons_src'][density] = path
 923         elif line.startswith("sdkVersion:"):
 924             m = re.match(APK_SDK_VERSION_PAT, line)
 925             if m is None:
 926                 logging.error(line.replace('sdkVersion:', '')
 927                               + ' is not a valid minSdkVersion!')
 928             else:
 929                 apk['minSdkVersion'] = m.group(1)
 930                 # if target not set, default to min
 931                 if 'targetSdkVersion' not in apk:
 932                     apk['targetSdkVersion'] = m.group(1)
 933         elif line.startswith("targetSdkVersion:"):
 934             m = re.match(APK_SDK_VERSION_PAT, line)
 935             if m is None:
 936                 logging.error(line.replace('targetSdkVersion:', '')
 937                               + ' is not a valid targetSdkVersion!')
 938             else:
 939                 apk['targetSdkVersion'] = m.group(1)
 940         elif line.startswith("maxSdkVersion:"):
 941             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 942         elif line.startswith("native-code:"):
 943             apk['nativecode'] = []
 944             for arch in line[13:].split(' '):
 945                 apk['nativecode'].append(arch[1:-1])
 946         elif line.startswith('uses-permission:'):
 947             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 948             if perm_match['maxSdkVersion']:
 949                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 950             permission = UsesPermission(
 951                 perm_match['name'],
 952                 perm_match['maxSdkVersion']
 953             )
 954
 955             apk['uses-permission'].append(permission)
 956         elif line.startswith('uses-permission-sdk-23:'):
 957             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 958             if perm_match['maxSdkVersion']:
 959                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 960             permission_sdk_23 = UsesPermissionSdk23(
 961                 perm_match['name'],
 962                 perm_match['maxSdkVersion']
 963             )
 964
 965             apk['uses-permission-sdk-23'].append(permission_sdk_23)
 966
 967         elif line.startswith('uses-feature:'):
 968             feature = re.match(APK_FEATURE_PAT, line).group(1)
 969             # Filter out this, it's only added with the latest SDK tools and
 970             # causes problems for lots of apps.
 971             if feature != "android.hardware.screen.portrait" \
 972                     and feature != "android.hardware.screen.landscape":
 973                 if feature.startswith("android.feature."):
 974                     feature = feature[16:]
 975                 apk['features'].add(feature)
 976
 977
 978 def scan_apk_androguard(apk, apkfile):
 979     try:
 980         from androguard.core.bytecodes.apk import APK
 981         apkobject = APK(apkfile)
 982         if apkobject.is_valid_APK():
 983             arsc = apkobject.get_android_resources()
 984         else:
 985             if options.delete_unknown:
 986                 if os.path.exists(apkfile):
 987                     logging.error("Failed to get apk information, deleting " + apkfile)
 988                     os.remove(apkfile)
 989                 else:
 990                     logging.error("Could not find {0} to remove it".format(apkfile))
 991             else:
 992                 logging.error("Failed to get apk information, skipping " + apkfile)
 993             raise BuildException("Invaild APK")
 994     except ImportError:
 995         raise FDroidException("androguard library is not installed and aapt not present")
 996     except FileNotFoundError:
 997         logging.error("Could not open apk file for analysis")
 998         raise BuildException("Invalid APK")
 999
1000     apk['packageName'] = apkobject.get_package()
1001     apk['versionCode'] = int(apkobject.get_androidversion_code())
1002     apk['versionName'] = apkobject.get_androidversion_name()
1003     if apk['versionName'][0] == "@":
1004         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1005         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1006         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1007     apk['name'] = apkobject.get_app_name()
1008
1009     if apkobject.get_max_sdk_version() is not None:
1010         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1011     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1012     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1013
1014     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1015     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1016
1017     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1018
1019     for file in apkobject.get_files():
1020         d_re = density_re.match(file)
1021         if d_re:
1022             folder = d_re.group(1).split('-')
1023             if len(folder) > 1:
1024                 resolution = folder[1]
1025             else:
1026                 resolution = 'mdpi'
1027             density = screen_resolutions[resolution]
1028             apk['icons_src'][density] = d_re.group(0)
1029
1030     if apk['icons_src'].get('-1') is None:
1031         apk['icons_src']['-1'] = apk['icons_src']['160']
1032
1033     arch_re = re.compile("^lib/(.*)/.*$")
1034     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1035     if len(arch) >= 1:
1036         apk['nativecode'] = []
1037         apk['nativecode'].extend(sorted(list(arch)))
1038
1039     xml = apkobject.get_android_manifest_xml()
1040
1041     for item in xml.getElementsByTagName('uses-permission'):
1042         name = str(item.getAttribute("android:name"))
1043         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1044         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1045         permission = UsesPermission(
1046             name,
1047             maxSdkVersion
1048         )
1049         apk['uses-permission'].append(permission)
1050
1051     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1052         name = str(item.getAttribute("android:name"))
1053         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1054         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1055         permission_sdk_23 = UsesPermissionSdk23(
1056             name,
1057             maxSdkVersion
1058         )
1059         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1060
1061     for item in xml.getElementsByTagName('uses-feature'):
1062         feature = str(item.getAttribute("android:name"))
1063         if feature != "android.hardware.screen.portrait" \
1064                 and feature != "android.hardware.screen.landscape":
1065             if feature.startswith("android.feature."):
1066                 feature = feature[16:]
1067         apk['features'].append(feature)
1068
1069
1070 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
1071     """Scan the apk with the given filename in the given repo directory.
1072
1073     This also extracts the icons.
1074
1075     :param apkcache: current apk cache information
1076     :param apkfilename: the filename of the apk to scan
1077     :param repodir: repo directory to scan
1078     :param knownapks: known apks info
1079     :param use_date_from_apk: use date from APK (instead of current date)
1080                               for newly added APKs
1081     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1082      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1083     """
1084
1085     if ' ' in apkfilename:
1086         if options.rename_apks:
1087             newfilename = apkfilename.replace(' ', '_')
1088             os.rename(os.path.join(repodir, apkfilename),
1089                       os.path.join(repodir, newfilename))
1090             apkfilename = newfilename
1091         else:
1092             logging.critical("Spaces in filenames are not allowed.")
1093             return True, None, False
1094
1095     apkfile = os.path.join(repodir, apkfilename)
1096     shasum = sha256sum(apkfile)
1097
1098     cachechanged = False
1099     usecache = False
1100     if apkfilename in apkcache:
1101         apk = apkcache[apkfilename]
1102         if apk.get('hash') == shasum:
1103             logging.debug("Reading " + apkfilename + " from cache")
1104             usecache = True
1105         else:
1106             logging.debug("Ignoring stale cache data for " + apkfilename)
1107
1108     if not usecache:
1109         logging.debug("Processing " + apkfilename)
1110         apk = {}
1111         apk['hash'] = shasum
1112         apk['hashType'] = 'sha256'
1113         apk['uses-permission'] = []
1114         apk['uses-permission-sdk-23'] = []
1115         apk['features'] = []
1116         apk['icons_src'] = {}
1117         apk['icons'] = {}
1118         apk['antiFeatures'] = set()
1119
1120         try:
1121             if SdkToolsPopen(['aapt', 'version'], output=False):
1122                 scan_apk_aapt(apk, apkfile)
1123             else:
1124                 scan_apk_androguard(apk, apkfile)
1125         except BuildException:
1126             return True, None, False
1127
1128         if 'minSdkVersion' not in apk:
1129             logging.warn("No SDK version information found in {0}".format(apkfile))
1130             apk['minSdkVersion'] = 1
1131
1132         # Check for debuggable apks...
1133         if common.isApkAndDebuggable(apkfile):
1134             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1135
1136         # Get the signature (or md5 of, to be precise)...
1137         logging.debug('Getting signature of {0}'.format(apkfile))
1138         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1139         if not apk['sig']:
1140             logging.critical("Failed to get apk signature")
1141             return True, None, False
1142
1143         if options.rename_apks:
1144             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1145             std_short_name = os.path.join(repodir, n)
1146             if apkfile != std_short_name:
1147                 if os.path.exists(std_short_name):
1148                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1149                     if apkfile != std_long_name:
1150                         if os.path.exists(std_long_name):
1151                             dupdir = os.path.join('duplicates', repodir)
1152                             if not os.path.isdir(dupdir):
1153                                 os.makedirs(dupdir, exist_ok=True)
1154                             dupfile = os.path.join('duplicates', std_long_name)
1155                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1156                             os.rename(apkfile, dupfile)
1157                             return True, None, False
1158                         else:
1159                             os.rename(apkfile, std_long_name)
1160                     apkfile = std_long_name
1161                 else:
1162                     os.rename(apkfile, std_short_name)
1163                     apkfile = std_short_name
1164                 apkfilename = apkfile[len(repodir) + 1:]
1165
1166         apk['apkName'] = apkfilename
1167         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1168         if os.path.exists(os.path.join(repodir, srcfilename)):
1169             apk['srcname'] = srcfilename
1170         apk['size'] = os.path.getsize(apkfile)
1171
1172         # verify the jar signature is correct
1173         if not common.verify_apk_signature(apkfile):
1174             return True, None, False
1175
1176         if has_old_openssl(apkfile):
1177             apk['antiFeatures'].add('KnownVuln')
1178
1179         apkzip = zipfile.ZipFile(apkfile, 'r')
1180
1181         # if an APK has files newer than the system time, suggest updating
1182         # the system clock.  This is useful for offline systems, used for
1183         # signing, which do not have another source of clock sync info. It
1184         # has to be more than 24 hours newer because ZIP/APK files do not
1185         # store timezone info
1186         manifest = apkzip.getinfo('AndroidManifest.xml')
1187         if manifest.date_time[1] == 0:  # month can't be zero
1188             logging.debug('AndroidManifest.xml has no date')
1189         else:
1190             dt_obj = datetime(*manifest.date_time)
1191             checkdt = dt_obj - timedelta(1)
1192             if datetime.today() < checkdt:
1193                 logging.warn('System clock is older than manifest in: '
1194                              + apkfilename
1195                              + '\nSet clock to that time using:\n'
1196                              + 'sudo date -s "' + str(dt_obj) + '"')
1197
1198         iconfilename = "%s.%s.png" % (
1199             apk['packageName'],
1200             apk['versionCode'])
1201
1202         # Extract the icon file...
1203         empty_densities = []
1204         for density in screen_densities:
1205             if density not in apk['icons_src']:
1206                 empty_densities.append(density)
1207                 continue
1208             iconsrc = apk['icons_src'][density]
1209             icon_dir = get_icon_dir(repodir, density)
1210             icondest = os.path.join(icon_dir, iconfilename)
1211
1212             try:
1213                 with open(icondest, 'wb') as f:
1214                     f.write(get_icon_bytes(apkzip, iconsrc))
1215                 apk['icons'][density] = iconfilename
1216             except (zipfile.BadZipFile, ValueError, KeyError) as e:
1217                 logging.warning("Error retrieving icon file: %s" % (icondest))
1218                 del apk['icons_src'][density]
1219                 empty_densities.append(density)
1220
1221         if '-1' in apk['icons_src']:
1222             iconsrc = apk['icons_src']['-1']
1223             iconpath = os.path.join(
1224                 get_icon_dir(repodir, '0'), iconfilename)
1225             with open(iconpath, 'wb') as f:
1226                 f.write(get_icon_bytes(apkzip, iconsrc))
1227             try:
1228                 im = Image.open(iconpath)
1229                 dpi = px_to_dpi(im.size[0])
1230                 for density in screen_densities:
1231                     if density in apk['icons']:
1232                         break
1233                     if density == screen_densities[-1] or dpi >= int(density):
1234                         apk['icons'][density] = iconfilename
1235                         shutil.move(iconpath,
1236                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
1237                         empty_densities.remove(density)
1238                         break
1239             except Exception as e:
1240                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1241
1242         if apk['icons']:
1243             apk['icon'] = iconfilename
1244
1245         apkzip.close()
1246
1247         # First try resizing down to not lose quality
1248         last_density = None
1249         for density in screen_densities:
1250             if density not in empty_densities:
1251                 last_density = density
1252                 continue
1253             if last_density is None:
1254                 continue
1255             logging.debug("Density %s not available, resizing down from %s"
1256                           % (density, last_density))
1257
1258             last_iconpath = os.path.join(
1259                 get_icon_dir(repodir, last_density), iconfilename)
1260             iconpath = os.path.join(
1261                 get_icon_dir(repodir, density), iconfilename)
1262             fp = None
1263             try:
1264                 fp = open(last_iconpath, 'rb')
1265                 im = Image.open(fp)
1266
1267                 size = dpi_to_px(density)
1268
1269                 im.thumbnail((size, size), Image.ANTIALIAS)
1270                 im.save(iconpath, "PNG")
1271                 empty_densities.remove(density)
1272             except Exception as e:
1273                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1274             finally:
1275                 if fp:
1276                     fp.close()
1277
1278         # Then just copy from the highest resolution available
1279         last_density = None
1280         for density in reversed(screen_densities):
1281             if density not in empty_densities:
1282                 last_density = density
1283                 continue
1284             if last_density is None:
1285                 continue
1286             logging.debug("Density %s not available, copying from lower density %s"
1287                           % (density, last_density))
1288
1289             shutil.copyfile(
1290                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1291                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1292
1293             empty_densities.remove(density)
1294
1295         for density in screen_densities:
1296             icon_dir = get_icon_dir(repodir, density)
1297             icondest = os.path.join(icon_dir, iconfilename)
1298             resize_icon(icondest, density)
1299
1300         # Copy from icons-mdpi to icons since mdpi is the baseline density
1301         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1302         if os.path.isfile(baseline):
1303             apk['icons']['0'] = iconfilename
1304             shutil.copyfile(baseline,
1305                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1306
1307         if use_date_from_apk and manifest.date_time[1] != 0:
1308             default_date_param = datetime(*manifest.date_time)
1309         else:
1310             default_date_param = None
1311
1312         # Record in known apks, getting the added date at the same time..
1313         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1314                                     default_date=default_date_param)
1315         if added:
1316             apk['added'] = added
1317
1318         apkcache[apkfilename] = apk
1319         cachechanged = True
1320
1321     return False, apk, cachechanged
1322
1323
1324 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1325     """Scan the apks in the given repo directory.
1326
1327     This also extracts the icons.
1328
1329     :param apkcache: current apk cache information
1330     :param repodir: repo directory to scan
1331     :param knownapks: known apks info
1332     :param use_date_from_apk: use date from APK (instead of current date)
1333                               for newly added APKs
1334     :returns: (apks, cachechanged) where apks is a list of apk information,
1335               and cachechanged is True if the apkcache got changed.
1336     """
1337
1338     cachechanged = False
1339
1340     for icon_dir in get_all_icon_dirs(repodir):
1341         if os.path.exists(icon_dir):
1342             if options.clean:
1343                 shutil.rmtree(icon_dir)
1344                 os.makedirs(icon_dir)
1345         else:
1346             os.makedirs(icon_dir)
1347
1348     apks = []
1349     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1350         apkfilename = apkfile[len(repodir) + 1:]
1351         (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1352         if skip:
1353             continue
1354         apks.append(apk)
1355
1356     return apks, cachechanged
1357
1358
1359 def apply_info_from_latest_apk(apps, apks):
1360     """
1361     Some information from the apks needs to be applied up to the application level.
1362     When doing this, we use the info from the most recent version's apk.
1363     We deal with figuring out when the app was added and last updated at the same time.
1364     """
1365     for appid, app in apps.items():
1366         bestver = UNSET_VERSION_CODE
1367         for apk in apks:
1368             if apk['packageName'] == appid:
1369                 if apk['versionCode'] > bestver:
1370                     bestver = apk['versionCode']
1371                     bestapk = apk
1372
1373                 if 'added' in apk:
1374                     if not app.added or apk['added'] < app.added:
1375                         app.added = apk['added']
1376                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1377                         app.lastUpdated = apk['added']
1378
1379         if not app.added:
1380             logging.debug("Don't know when " + appid + " was added")
1381         if not app.lastUpdated:
1382             logging.debug("Don't know when " + appid + " was last updated")
1383
1384         if bestver == UNSET_VERSION_CODE:
1385
1386             if app.Name is None:
1387                 app.Name = app.AutoName or appid
1388             app.icon = None
1389             logging.debug("Application " + appid + " has no packages")
1390         else:
1391             if app.Name is None:
1392                 app.Name = bestapk['name']
1393             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1394             if app.CurrentVersionCode is None:
1395                 app.CurrentVersionCode = str(bestver)
1396
1397
1398 def make_categories_txt(repodir, categories):
1399     '''Write a category list in the repo to allow quick access'''
1400     catdata = ''
1401     for cat in sorted(categories):
1402         catdata += cat + '\n'
1403     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1404         f.write(catdata)
1405
1406
1407 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1408
1409     for appid, app in apps.items():
1410
1411         if app.ArchivePolicy:
1412             keepversions = int(app.ArchivePolicy[:-9])
1413         else:
1414             keepversions = defaultkeepversions
1415
1416         def filter_apk_list_sorted(apk_list):
1417             res = []
1418             for apk in apk_list:
1419                 if apk['packageName'] == appid:
1420                     res.append(apk)
1421
1422             # Sort the apk list by version code. First is highest/newest.
1423             return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1424
1425         def move_file(from_dir, to_dir, filename, ignore_missing):
1426             from_path = os.path.join(from_dir, filename)
1427             if ignore_missing and not os.path.exists(from_path):
1428                 return
1429             to_path = os.path.join(to_dir, filename)
1430             shutil.move(from_path, to_path)
1431
1432         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1433                       .format(appid, len(apks), keepversions, len(archapks)))
1434
1435         if len(apks) > keepversions:
1436             apklist = filter_apk_list_sorted(apks)
1437             # Move back the ones we don't want.
1438             for apk in apklist[keepversions:]:
1439                 logging.info("Moving " + apk['apkName'] + " to archive")
1440                 move_file(repodir, archivedir, apk['apkName'], False)
1441                 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1442                 for density in all_screen_densities:
1443                     repo_icon_dir = get_icon_dir(repodir, density)
1444                     archive_icon_dir = get_icon_dir(archivedir, density)
1445                     if density not in apk['icons']:
1446                         continue
1447                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1448                 if 'srcname' in apk:
1449                     move_file(repodir, archivedir, apk['srcname'], False)
1450                 archapks.append(apk)
1451                 apks.remove(apk)
1452         elif len(apks) < keepversions and len(archapks) > 0:
1453             required = keepversions - len(apks)
1454             archapklist = filter_apk_list_sorted(archapks)
1455             # Move forward the ones we want again.
1456             for apk in archapklist[:required]:
1457                 logging.info("Moving " + apk['apkName'] + " from archive")
1458                 move_file(archivedir, repodir, apk['apkName'], False)
1459                 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1460                 for density in all_screen_densities:
1461                     repo_icon_dir = get_icon_dir(repodir, density)
1462                     archive_icon_dir = get_icon_dir(archivedir, density)
1463                     if density not in apk['icons']:
1464                         continue
1465                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1466                 if 'srcname' in apk:
1467                     move_file(archivedir, repodir, apk['srcname'], False)
1468                 archapks.remove(apk)
1469                 apks.append(apk)
1470
1471
1472 def add_apks_to_per_app_repos(repodir, apks):
1473     apks_per_app = dict()
1474     for apk in apks:
1475         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1476         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1477         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1478         apks_per_app[apk['packageName']] = apk
1479
1480         if not os.path.exists(apk['per_app_icons']):
1481             logging.info('Adding new repo for only ' + apk['packageName'])
1482             os.makedirs(apk['per_app_icons'])
1483
1484         apkpath = os.path.join(repodir, apk['apkName'])
1485         shutil.copy(apkpath, apk['per_app_repo'])
1486         apksigpath = apkpath + '.sig'
1487         if os.path.exists(apksigpath):
1488             shutil.copy(apksigpath, apk['per_app_repo'])
1489         apkascpath = apkpath + '.asc'
1490         if os.path.exists(apkascpath):
1491             shutil.copy(apkascpath, apk['per_app_repo'])
1492
1493
1494 config = None
1495 options = None
1496
1497
1498 def main():
1499
1500     global config, options
1501
1502     # Parse command line...
1503     parser = ArgumentParser()
1504     common.setup_global_opts(parser)
1505     parser.add_argument("--create-key", action="store_true", default=False,
1506                         help="Create a repo signing key in a keystore")
1507     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1508                         help="Create skeleton metadata files that are missing")
1509     parser.add_argument("--delete-unknown", action="store_true", default=False,
1510                         help="Delete APKs and/or OBBs without metadata from the repo")
1511     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1512                         help="Report on build data status")
1513     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1514                         help="Interactively ask about things that need updating.")
1515     parser.add_argument("-I", "--icons", action="store_true", default=False,
1516                         help="Resize all the icons exceeding the max pixel size and exit")
1517     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1518                         help="Specify editor to use in interactive mode. Default " +
1519                         "is /etc/alternatives/editor")
1520     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1521                         help="Update the wiki")
1522     parser.add_argument("--pretty", action="store_true", default=False,
1523                         help="Produce human-readable index.xml")
1524     parser.add_argument("--clean", action="store_true", default=False,
1525                         help="Clean update - don't uses caches, reprocess all apks")
1526     parser.add_argument("--nosign", action="store_true", default=False,
1527                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1528     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1529                         help="Use date from apk instead of current time for newly added apks")
1530     parser.add_argument("--rename-apks", action="store_true", default=False,
1531                         help="Rename APK files that do not match package.name_123.apk")
1532     metadata.add_metadata_arguments(parser)
1533     options = parser.parse_args()
1534     metadata.warnings_action = options.W
1535
1536     config = common.read_config(options)
1537
1538     if not ('jarsigner' in config and 'keytool' in config):
1539         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1540
1541     repodirs = ['repo']
1542     if config['archive_older'] != 0:
1543         repodirs.append('archive')
1544         if not os.path.exists('archive'):
1545             os.mkdir('archive')
1546
1547     if options.icons:
1548         resize_all_icons(repodirs)
1549         sys.exit(0)
1550
1551     if options.rename_apks:
1552         options.clean = True
1553
1554     # check that icons exist now, rather than fail at the end of `fdroid update`
1555     for k in ['repo_icon', 'archive_icon']:
1556         if k in config:
1557             if not os.path.exists(config[k]):
1558                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1559                 sys.exit(1)
1560
1561     # if the user asks to create a keystore, do it now, reusing whatever it can
1562     if options.create_key:
1563         if os.path.exists(config['keystore']):
1564             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1565             logging.critical("\t'" + config['keystore'] + "'")
1566             sys.exit(1)
1567
1568         if 'repo_keyalias' not in config:
1569             config['repo_keyalias'] = socket.getfqdn()
1570             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1571         if 'keydname' not in config:
1572             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1573             common.write_to_config(config, 'keydname', config['keydname'])
1574         if 'keystore' not in config:
1575             config['keystore'] = common.default_config['keystore']
1576             common.write_to_config(config, 'keystore', config['keystore'])
1577
1578         password = common.genpassword()
1579         if 'keystorepass' not in config:
1580             config['keystorepass'] = password
1581             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1582         if 'keypass' not in config:
1583             config['keypass'] = password
1584             common.write_to_config(config, 'keypass', config['keypass'])
1585         common.genkeystore(config)
1586
1587     # Get all apps...
1588     apps = metadata.read_metadata()
1589
1590     # Generate a list of categories...
1591     categories = set()
1592     for app in apps.values():
1593         categories.update(app.Categories)
1594
1595     # Read known apks data (will be updated and written back when we've finished)
1596     knownapks = common.KnownApks()
1597
1598     # Get APK cache
1599     apkcache = get_cache()
1600
1601     # Delete builds for disabled apps
1602     delete_disabled_builds(apps, apkcache, repodirs)
1603
1604     # Scan all apks in the main repo
1605     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1606
1607     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1608                                            options.use_date_from_apk)
1609     cachechanged = cachechanged or fcachechanged
1610     apks += files
1611     # Generate warnings for apk's with no metadata (or create skeleton
1612     # metadata files, if requested on the command line)
1613     newmetadata = False
1614     for apk in apks:
1615         if apk['packageName'] not in apps:
1616             if options.create_metadata:
1617                 if 'name' not in apk:
1618                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1619                     continue
1620                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1621                 f.write("License:Unknown\n")
1622                 f.write("Web Site:\n")
1623                 f.write("Source Code:\n")
1624                 f.write("Issue Tracker:\n")
1625                 f.write("Changelog:\n")
1626                 f.write("Summary:" + apk['name'] + "\n")
1627                 f.write("Description:\n")
1628                 f.write(apk['name'] + "\n")
1629                 f.write(".\n")
1630                 f.write("Name:" + apk['name'] + "\n")
1631                 f.close()
1632                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1633                 newmetadata = True
1634             else:
1635                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1636                 if options.delete_unknown:
1637                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1638                     rmf = os.path.join(repodirs[0], apk['apkName'])
1639                     if not os.path.exists(rmf):
1640                         logging.error("Could not find {0} to remove it".format(rmf))
1641                     else:
1642                         os.remove(rmf)
1643                 else:
1644                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1645
1646     # update the metadata with the newly created ones included
1647     if newmetadata:
1648         apps = metadata.read_metadata()
1649
1650     copy_triple_t_store_metadata(apps)
1651     insert_obbs(repodirs[0], apps, apks)
1652     insert_localized_app_metadata(apps)
1653
1654     # Scan the archive repo for apks as well
1655     if len(repodirs) > 1:
1656         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1657         if cc:
1658             cachechanged = True
1659     else:
1660         archapks = []
1661
1662     # Apply information from latest apks to the application and update dates
1663     apply_info_from_latest_apk(apps, apks + archapks)
1664
1665     # Sort the app list by name, then the web site doesn't have to by default.
1666     # (we had to wait until we'd scanned the apks to do this, because mostly the
1667     # name comes from there!)
1668     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1669
1670     # APKs are placed into multiple repos based on the app package, providing
1671     # per-app subscription feeds for nightly builds and things like it
1672     if config['per_app_repos']:
1673         add_apks_to_per_app_repos(repodirs[0], apks)
1674         for appid, app in apps.items():
1675             repodir = os.path.join(appid, 'fdroid', 'repo')
1676             appdict = dict()
1677             appdict[appid] = app
1678             if os.path.isdir(repodir):
1679                 index.make(appdict, [appid], apks, repodir, False)
1680             else:
1681                 logging.info('Skipping index generation for ' + appid)
1682         return
1683
1684     if len(repodirs) > 1:
1685         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1686
1687     # Make the index for the main repo...
1688     index.make(apps, sortedids, apks, repodirs[0], False)
1689     make_categories_txt(repodirs[0], categories)
1690
1691     # If there's an archive repo,  make the index for it. We already scanned it
1692     # earlier on.
1693     if len(repodirs) > 1:
1694         index.make(apps, sortedids, archapks, repodirs[1], True)
1695
1696     git_remote = config.get('binary_transparency_remote')
1697     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1698         btlog.make_binary_transparency_log(repodirs)
1699
1700     if config['update_stats']:
1701         # Update known apks info...
1702         knownapks.writeifchanged()
1703
1704         # Generate latest apps data for widget
1705         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1706             data = ''
1707             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1708                 for line in f:
1709                     appid = line.rstrip()
1710                     data += appid + "\t"
1711                     app = apps[appid]
1712                     data += app.Name + "\t"
1713                     if app.icon is not None:
1714                         data += app.icon + "\t"
1715                     data += app.License + "\n"
1716             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1717                 f.write(data)
1718
1719     if cachechanged:
1720         write_cache(apkcache)
1721
1722     # Update the wiki...
1723     if options.wiki:
1724         update_wiki(apps, sortedids, apks + archapks)
1725
1726     logging.info("Finished.")
1727
1728
1729 if __name__ == "__main__":
1730     main()