fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import btlog
  41 from . import common
  42 from . import index
  43 from . import metadata
  44 from .common import SdkToolsPopen
  45
  46 METADATA_VERSION = 18
  47
  48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  49 UNSET_VERSION_CODE = -0x100000000
  50
  51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['640', '480', '320', '240', '160', '120']
  63
  64 all_screen_densities = ['0'] + screen_densities
  65
  66 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  67 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  68
  69 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  70 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  71 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  72                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  73
  74
  75 def dpi_to_px(density):
  76     return (int(density) * 48) / 160
  77
  78
  79 def px_to_dpi(px):
  80     return (int(px) * 160) / 48
  81
  82
  83 def get_icon_dir(repodir, density):
  84     if density == '0':
  85         return os.path.join(repodir, "icons")
  86     return os.path.join(repodir, "icons-%s" % density)
  87
  88
  89 def get_icon_dirs(repodir):
  90     for density in screen_densities:
  91         yield get_icon_dir(repodir, density)
  92
  93
  94 def get_all_icon_dirs(repodir):
  95     for density in all_screen_densities:
  96         yield get_icon_dir(repodir, density)
  97
  98
  99 def update_wiki(apps, sortedids, apks):
 100     """Update the wiki
 101
 102     :param apps: fully populated list of all applications
 103     :param apks: all apks, except...
 104     """
 105     logging.info("Updating wiki")
 106     wikicat = 'Apps'
 107     wikiredircat = 'App Redirects'
 108     import mwclient
 109     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 110                          path=config['wiki_path'])
 111     site.login(config['wiki_user'], config['wiki_password'])
 112     generated_pages = {}
 113     generated_redirects = {}
 114
 115     for appid in sortedids:
 116         app = metadata.App(apps[appid])
 117
 118         wikidata = ''
 119         if app.Disabled:
 120             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 121         if app.AntiFeatures:
 122             for af in app.AntiFeatures:
 123                 wikidata += '{{AntiFeature|' + af + '}}\n'
 124         if app.RequiresRoot:
 125             requiresroot = 'Yes'
 126         else:
 127             requiresroot = 'No'
 128         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 129             appid,
 130             app.Name,
 131             app.added.strftime('%Y-%m-%d') if app.added else '',
 132             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 133             app.SourceCode,
 134             app.IssueTracker,
 135             app.WebSite,
 136             app.Changelog,
 137             app.Donate,
 138             app.FlattrID,
 139             app.Bitcoin,
 140             app.Litecoin,
 141             app.License,
 142             requiresroot,
 143             app.AuthorName,
 144             app.AuthorEmail)
 145
 146         if app.Provides:
 147             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 148
 149         wikidata += app.Summary
 150         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 151
 152         wikidata += "=Description=\n"
 153         wikidata += metadata.description_wiki(app.Description) + "\n"
 154
 155         wikidata += "=Maintainer Notes=\n"
 156         if app.MaintainerNotes:
 157             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 158         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 159
 160         # Get a list of all packages for this application...
 161         apklist = []
 162         gotcurrentver = False
 163         cantupdate = False
 164         buildfails = False
 165         for apk in apks:
 166             if apk['packageName'] == appid:
 167                 if str(apk['versionCode']) == app.CurrentVersionCode:
 168                     gotcurrentver = True
 169                 apklist.append(apk)
 170         # Include ones we can't build, as a special case...
 171         for build in app.builds:
 172             if build.disable:
 173                 if build.versionCode == app.CurrentVersionCode:
 174                     cantupdate = True
 175                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 176                 apklist.append({'versionCode': int(build.versionCode),
 177                                 'versionName': build.versionName,
 178                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 179                                 })
 180             else:
 181                 builtit = False
 182                 for apk in apklist:
 183                     if apk['versionCode'] == int(build.versionCode):
 184                         builtit = True
 185                         break
 186                 if not builtit:
 187                     buildfails = True
 188                     apklist.append({'versionCode': int(build.versionCode),
 189                                     'versionName': build.versionName,
 190                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 191                                     })
 192         if app.CurrentVersionCode == '0':
 193             cantupdate = True
 194         # Sort with most recent first...
 195         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 196
 197         wikidata += "=Versions=\n"
 198         if len(apklist) == 0:
 199             wikidata += "We currently have no versions of this app available."
 200         elif not gotcurrentver:
 201             wikidata += "We don't have the current version of this app."
 202         else:
 203             wikidata += "We have the current version of this app."
 204         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 205         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 208         if len(app.CurrentVersion) > 0:
 209             wikidata += "The current (recommended) version is " + app.CurrentVersion
 210             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 211         validapks = 0
 212         for apk in apklist:
 213             wikidata += "==" + apk['versionName'] + "==\n"
 214
 215             if 'buildproblem' in apk:
 216                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 217             else:
 218                 validapks += 1
 219                 wikidata += "This version is built and signed by "
 220                 if 'srcname' in apk:
 221                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 222                 else:
 223                     wikidata += "the original developer.\n\n"
 224             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 225
 226         wikidata += '\n[[Category:' + wikicat + ']]\n'
 227         if len(app.NoSourceSince) > 0:
 228             wikidata += '\n[[Category:Apps missing source code]]\n'
 229         if validapks == 0 and not app.Disabled:
 230             wikidata += '\n[[Category:Apps with no packages]]\n'
 231         if cantupdate and not app.Disabled:
 232             wikidata += "\n[[Category:Apps we cannot update]]\n"
 233         if buildfails and not app.Disabled:
 234             wikidata += "\n[[Category:Apps with failing builds]]\n"
 235         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 236             wikidata += '\n[[Category:Apps to Update]]\n'
 237         if app.Disabled:
 238             wikidata += '\n[[Category:Apps that are disabled]]\n'
 239         if app.UpdateCheckMode == 'None' and not app.Disabled:
 240             wikidata += '\n[[Category:Apps with no update check]]\n'
 241         for appcat in app.Categories:
 242             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 243
 244         # We can't have underscores in the page name, even if they're in
 245         # the package ID, because MediaWiki messes with them...
 246         pagename = appid.replace('_', ' ')
 247
 248         # Drop a trailing newline, because mediawiki is going to drop it anyway
 249         # and it we don't we'll think the page has changed when it hasn't...
 250         if wikidata.endswith('\n'):
 251             wikidata = wikidata[:-1]
 252
 253         generated_pages[pagename] = wikidata
 254
 255         # Make a redirect from the name to the ID too, unless there's
 256         # already an existing page with the name and it isn't a redirect.
 257         noclobber = False
 258         apppagename = app.Name.replace('_', ' ')
 259         apppagename = apppagename.replace('{', '')
 260         apppagename = apppagename.replace('}', ' ')
 261         apppagename = apppagename.replace(':', ' ')
 262         apppagename = apppagename.replace('[', ' ')
 263         apppagename = apppagename.replace(']', ' ')
 264         # Drop double spaces caused mostly by replacing ':' above
 265         apppagename = apppagename.replace('  ', ' ')
 266         for expagename in site.allpages(prefix=apppagename,
 267                                         filterredir='nonredirects',
 268                                         generator=False):
 269             if expagename == apppagename:
 270                 noclobber = True
 271         # Another reason not to make the redirect page is if the app name
 272         # is the same as it's ID, because that will overwrite the real page
 273         # with an redirect to itself! (Although it seems like an odd
 274         # scenario this happens a lot, e.g. where there is metadata but no
 275         # builds or binaries to extract a name from.
 276         if apppagename == pagename:
 277             noclobber = True
 278         if not noclobber:
 279             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 280
 281     for tcat, genp in [(wikicat, generated_pages),
 282                        (wikiredircat, generated_redirects)]:
 283         catpages = site.Pages['Category:' + tcat]
 284         existingpages = []
 285         for page in catpages:
 286             existingpages.append(page.name)
 287             if page.name in genp:
 288                 pagetxt = page.edit()
 289                 if pagetxt != genp[page.name]:
 290                     logging.debug("Updating modified page " + page.name)
 291                     page.save(genp[page.name], summary='Auto-updated')
 292                 else:
 293                     logging.debug("Page " + page.name + " is unchanged")
 294             else:
 295                 logging.warn("Deleting page " + page.name)
 296                 page.delete('No longer published')
 297         for pagename, text in genp.items():
 298             logging.debug("Checking " + pagename)
 299             if pagename not in existingpages:
 300                 logging.debug("Creating page " + pagename)
 301                 try:
 302                     newpage = site.Pages[pagename]
 303                     newpage.save(text, summary='Auto-created')
 304                 except Exception as e:
 305                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 306
 307     # Purge server cache to ensure counts are up to date
 308     site.pages['Repository Maintenance'].purge()
 309
 310
 311 def delete_disabled_builds(apps, apkcache, repodirs):
 312     """Delete disabled build outputs.
 313
 314     :param apps: list of all applications, as per metadata.read_metadata
 315     :param apkcache: current apk cache information
 316     :param repodirs: the repo directories to process
 317     """
 318     for appid, app in apps.items():
 319         for build in app['builds']:
 320             if not build.disable:
 321                 continue
 322             apkfilename = common.get_release_filename(app, build)
 323             iconfilename = "%s.%s.png" % (
 324                 appid,
 325                 build.versionCode)
 326             for repodir in repodirs:
 327                 files = [
 328                     os.path.join(repodir, apkfilename),
 329                     os.path.join(repodir, apkfilename + '.asc'),
 330                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 331                 ]
 332                 for density in all_screen_densities:
 333                     repo_dir = get_icon_dir(repodir, density)
 334                     files.append(os.path.join(repo_dir, iconfilename))
 335
 336                 for f in files:
 337                     if os.path.exists(f):
 338                         logging.info("Deleting disabled build output " + f)
 339                         os.remove(f)
 340             if apkfilename in apkcache:
 341                 del apkcache[apkfilename]
 342
 343
 344 def resize_icon(iconpath, density):
 345
 346     if not os.path.isfile(iconpath):
 347         return
 348
 349     fp = None
 350     try:
 351         fp = open(iconpath, 'rb')
 352         im = Image.open(fp)
 353         size = dpi_to_px(density)
 354
 355         if any(length > size for length in im.size):
 356             oldsize = im.size
 357             im.thumbnail((size, size), Image.ANTIALIAS)
 358             logging.debug("%s was too large at %s - new size is %s" % (
 359                 iconpath, oldsize, im.size))
 360             im.save(iconpath, "PNG")
 361
 362     except Exception as e:
 363         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 364
 365     finally:
 366         if fp:
 367             fp.close()
 368
 369
 370 def resize_all_icons(repodirs):
 371     """Resize all icons that exceed the max size
 372
 373     :param repodirs: the repo directories to process
 374     """
 375     for repodir in repodirs:
 376         for density in screen_densities:
 377             icon_dir = get_icon_dir(repodir, density)
 378             icon_glob = os.path.join(icon_dir, '*.png')
 379             for iconpath in glob.glob(icon_glob):
 380                 resize_icon(iconpath, density)
 381
 382
 383 def getsig(apkpath):
 384     """ Get the signing certificate of an apk. To get the same md5 has that
 385     Android gets, we encode the .RSA certificate in a specific format and pass
 386     it hex-encoded to the md5 digest algorithm.
 387
 388     :param apkpath: path to the apk
 389     :returns: A string containing the md5 of the signature of the apk or None
 390               if an error occurred.
 391     """
 392
 393     # verify the jar signature is correct
 394     if not common.verify_apk_signature(apkpath):
 395         return None
 396
 397     with zipfile.ZipFile(apkpath, 'r') as apk:
 398         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 399
 400         if len(certs) < 1:
 401             logging.error("Found no signing certificates on %s" % apkpath)
 402             return None
 403         if len(certs) > 1:
 404             logging.error("Found multiple signing certificates on %s" % apkpath)
 405             return None
 406
 407         cert = apk.read(certs[0])
 408
 409     cert_encoded = common.get_certificate(cert)
 410
 411     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 412
 413
 414 def get_cache_file():
 415     return os.path.join('tmp', 'apkcache')
 416
 417
 418 def get_cache():
 419     """
 420     Gather information about all the apk files in the repo directory,
 421     using cached data if possible.
 422     :return: apkcache
 423     """
 424     apkcachefile = get_cache_file()
 425     if not options.clean and os.path.exists(apkcachefile):
 426         with open(apkcachefile, 'rb') as cf:
 427             apkcache = pickle.load(cf, encoding='utf-8')
 428         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
 429             apkcache = {}
 430     else:
 431         apkcache = {}
 432
 433     return apkcache
 434
 435
 436 def write_cache(apkcache):
 437     apkcachefile = get_cache_file()
 438     cache_path = os.path.dirname(apkcachefile)
 439     if not os.path.exists(cache_path):
 440         os.makedirs(cache_path)
 441     apkcache["METADATA_VERSION"] = METADATA_VERSION
 442     with open(apkcachefile, 'wb') as cf:
 443         pickle.dump(apkcache, cf)
 444
 445
 446 def get_icon_bytes(apkzip, iconsrc):
 447     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 448     try:
 449         return apkzip.read(iconsrc)
 450     except KeyError:
 451         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 452
 453
 454 def sha256sum(filename):
 455     '''Calculate the sha256 of the given file'''
 456     sha = hashlib.sha256()
 457     with open(filename, 'rb') as f:
 458         while True:
 459             t = f.read(16384)
 460             if len(t) == 0:
 461                 break
 462             sha.update(t)
 463     return sha.hexdigest()
 464
 465
 466 def has_old_openssl(filename):
 467     '''checks for known vulnerable openssl versions in the APK'''
 468
 469     # statically load this pattern
 470     if not hasattr(has_old_openssl, "pattern"):
 471         has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 472
 473     with zipfile.ZipFile(filename) as zf:
 474         for name in zf.namelist():
 475             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 476                 lib = zf.open(name)
 477                 while True:
 478                     chunk = lib.read(4096)
 479                     if chunk == b'':
 480                         break
 481                     m = has_old_openssl.pattern.search(chunk)
 482                     if m:
 483                         version = m.group(1).decode('ascii')
 484                         if version.startswith('1.0.1') and version[5] >= 'r' \
 485                            or version.startswith('1.0.2') and version[5] >= 'f':
 486                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 487                         else:
 488                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 489                             return True
 490                         break
 491     return False
 492
 493
 494 def insert_obbs(repodir, apps, apks):
 495     """Scans the .obb files in a given repo directory and adds them to the
 496     relevant APK instances.  OBB files have versionCodes like APK
 497     files, and they are loosely associated.  If there is an OBB file
 498     present, then any APK with the same or higher versionCode will use
 499     that OBB file.  There are two OBB types: main and patch, each APK
 500     can only have only have one of each.
 501
 502     https://developer.android.com/google/play/expansion-files.html
 503
 504     :param repodir: repo directory to scan
 505     :param apps: list of current, valid apps
 506     :param apks: current information on all APKs
 507
 508     """
 509
 510     def obbWarnDelete(f, msg):
 511         logging.warning(msg + f)
 512         if options.delete_unknown:
 513             logging.error("Deleting unknown file: " + f)
 514             os.remove(f)
 515
 516     obbs = []
 517     java_Integer_MIN_VALUE = -pow(2, 31)
 518     currentPackageNames = apps.keys()
 519     for f in glob.glob(os.path.join(repodir, '*.obb')):
 520         obbfile = os.path.basename(f)
 521         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 522         chunks = obbfile.split('.')
 523         if chunks[0] != 'main' and chunks[0] != 'patch':
 524             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 525             continue
 526         if not re.match(r'^-?[0-9]+$', chunks[1]):
 527             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 528             continue
 529         versionCode = int(chunks[1])
 530         packagename = ".".join(chunks[2:-1])
 531
 532         highestVersionCode = java_Integer_MIN_VALUE
 533         if packagename not in currentPackageNames:
 534             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 535             continue
 536         for apk in apks:
 537             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 538                 highestVersionCode = apk['versionCode']
 539         if versionCode > highestVersionCode:
 540             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 541                           + ') than any APK: ')
 542             continue
 543         obbsha256 = sha256sum(f)
 544         obbs.append((packagename, versionCode, obbfile, obbsha256))
 545
 546     for apk in apks:
 547         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 548             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 549                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 550                     apk['obbMainFile'] = obbfile
 551                     apk['obbMainFileSha256'] = obbsha256
 552                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 553                     apk['obbPatchFile'] = obbfile
 554                     apk['obbPatchFileSha256'] = obbsha256
 555             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 556                 break
 557
 558
 559 def _get_localized_dict(app, locale):
 560     '''get the dict to add localized store metadata to'''
 561     if 'localized' not in app:
 562         app['localized'] = collections.OrderedDict()
 563     if locale not in app['localized']:
 564         app['localized'][locale] = collections.OrderedDict()
 565     return app['localized'][locale]
 566
 567
 568 def _set_localized_text_entry(app, locale, key, f):
 569     limit = config['char_limits'][key]
 570     localized = _get_localized_dict(app, locale)
 571     with open(f) as fp:
 572         text = fp.read()[:limit]
 573         if len(text) > 0:
 574             localized[key] = text
 575
 576
 577 def _set_author_entry(app, key, f):
 578     limit = config['char_limits']['author']
 579     with open(f) as fp:
 580         text = fp.read()[:limit]
 581         if len(text) > 0:
 582             app[key] = text
 583
 584
 585 def copy_triple_t_store_metadata(apps):
 586     """Include store metadata from the app's source repo
 587
 588     The Triple-T Gradle Play Publisher is a plugin that has a standard
 589     file layout for all of the metadata and graphics that the Google
 590     Play Store accepts.  Since F-Droid has the git repo, it can just
 591     pluck those files directly.  This method reads any text files into
 592     the app dict, then copies any graphics into the fdroid repo
 593     directory structure.
 594
 595     This needs to be run before insert_localized_app_metadata() so that
 596     the graphics files that are copied into the fdroid repo get
 597     properly indexed.
 598
 599     https://github.com/Triple-T/gradle-play-publisher#upload-images
 600     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 601
 602     """
 603
 604     if not os.path.isdir('build'):
 605         return  # nothing to do
 606
 607     for packageName, app in apps.items():
 608         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 609             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 610             for root, dirs, files in os.walk(d):
 611                 segments = root.split('/')
 612                 locale = segments[-2]
 613                 for f in files:
 614                     if f == 'fulldescription':
 615                         _set_localized_text_entry(app, locale, 'description',
 616                                                   os.path.join(root, f))
 617                         continue
 618                     elif f == 'shortdescription':
 619                         _set_localized_text_entry(app, locale, 'summary',
 620                                                   os.path.join(root, f))
 621                         continue
 622                     elif f == 'title':
 623                         _set_localized_text_entry(app, locale, 'name',
 624                                                   os.path.join(root, f))
 625                         continue
 626                     elif f == 'video':
 627                         _set_localized_text_entry(app, locale, 'video',
 628                                                   os.path.join(root, f))
 629                         continue
 630                     elif f == 'whatsnew':
 631                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 632                                                   os.path.join(root, f))
 633                         continue
 634                     elif f == 'contactEmail':
 635                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 636                         continue
 637                     elif f == 'contactPhone':
 638                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 639                         continue
 640                     elif f == 'contactWebsite':
 641                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 642                         continue
 643
 644                     base, extension = common.get_extension(f)
 645                     dirname = os.path.basename(root)
 646                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 647                         if segments[-2] == 'listing':
 648                             locale = segments[-3]
 649                         else:
 650                             locale = segments[-2]
 651                         destdir = os.path.join('repo', packageName, locale)
 652                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 653                         sourcefile = os.path.join(root, f)
 654                         destfile = os.path.join(destdir, dirname + '.' + extension)
 655                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 656                         shutil.copy(sourcefile, destfile)
 657
 658
 659 def insert_localized_app_metadata(apps):
 660     """scans standard locations for graphics and localized text
 661
 662     Scans for localized description files, store graphics, and
 663     screenshot PNG files in statically defined screenshots directory
 664     and adds them to the app metadata.  The screenshots and graphic
 665     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 666     and must be in the following layout:
 667     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 668     # TODO mention that the 'localized' section is not in metadata.yml, so key names are like Java vars: camelCase with first letter lowercase.
 669     repo/packageName/locale/featureGraphic.png
 670     repo/packageName/locale/phoneScreenshots/1.png
 671     repo/packageName/locale/phoneScreenshots/2.png
 672
 673     The changelog files must be text files named with the versionCode
 674     ending with ".txt" and must be in the following layout:
 675     https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#changelogs-whats-new
 676
 677     repo/packageName/locale/changelogs/12345.txt
 678
 679     This will scan the each app's source repo then the metadata/ dir
 680     for these standard locations of changelog files.  If it finds
 681     them, they will be added to the dict of all packages, with the
 682     versions in the metadata/ folder taking precendence over the what
 683     is in the app's source repo.
 684
 685     Where "packageName" is the app's packageName and "locale" is the locale
 686     of the graphics, e.g. what language they are in, using the IETF RFC5646
 687     format (en-US, fr-CA, es-MX, etc).
 688
 689     This will also scan the app's git for a fastlane folder, and the
 690     metadata/ folder and the apps' source repos for standard locations
 691     of graphic and screenshot files.  If it finds them, it will copy
 692     them into the repo.  The fastlane files follow this pattern:
 693     https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#images-and-screenshots
 694
 695     """
 696
 697     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 698     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 699
 700     for d in sorted(sourcedirs):
 701         if not os.path.isdir(d):
 702             continue
 703         for root, dirs, files in os.walk(d):
 704             segments = root.split('/')
 705             packageName = segments[1]
 706             if packageName not in apps:
 707                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 708                 continue
 709             locale = segments[-1]
 710             destdir = os.path.join('repo', packageName, locale)
 711             for f in files:
 712                 if f == 'full_description.txt':
 713                     _set_localized_text_entry(apps[packageName], locale, 'description',
 714                                               os.path.join(root, f))
 715                     continue
 716                 elif f == 'short_description.txt':
 717                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 718                                               os.path.join(root, f))
 719                     continue
 720                 elif f == 'title.txt':
 721                     _set_localized_text_entry(apps[packageName], locale, 'name',
 722                                               os.path.join(root, f))
 723                     continue
 724                 elif f == 'video.txt':
 725                     _set_localized_text_entry(apps[packageName], locale, 'video',
 726                                               os.path.join(root, f))
 727                     continue
 728                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 729                     _set_localized_text_entry(apps[packageName], segments[-2], 'whatsNew',
 730                                               os.path.join(root, f))
 731                     continue
 732
 733                 base, extension = common.get_extension(f)
 734                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 735                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 736                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 737                     shutil.copy(os.path.join(root, f), destdir)
 738             for d in dirs:
 739                 if d in SCREENSHOT_DIRS:
 740                     for f in glob.glob(os.path.join(root, d, '*.*')):
 741                         _, extension = common.get_extension(f)
 742                         if extension in ALLOWED_EXTENSIONS:
 743                             screenshotdestdir = os.path.join(destdir, d)
 744                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 745                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 746                             shutil.copy(f, screenshotdestdir)
 747
 748     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 749     for d in repofiles:
 750         if not os.path.isdir(d):
 751             continue
 752         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 753             if not os.path.isfile(f):
 754                 continue
 755             segments = f.split('/')
 756             packageName = segments[1]
 757             locale = segments[2]
 758             screenshotdir = segments[3]
 759             filename = os.path.basename(f)
 760             base, extension = common.get_extension(filename)
 761
 762             if packageName not in apps:
 763                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 764                                 % (filename, packageName))
 765                 continue
 766             graphics = _get_localized_dict(apps[packageName], locale)
 767
 768             if extension not in ALLOWED_EXTENSIONS:
 769                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 770             elif base in GRAPHIC_NAMES:
 771                 # there can only be zero or one of these per locale
 772                 graphics[base] = filename
 773             elif screenshotdir in SCREENSHOT_DIRS:
 774                 # there can any number of these per locale
 775                 logging.debug('adding ' + base + ':' + f)
 776                 if screenshotdir not in graphics:
 777                     graphics[screenshotdir] = []
 778                 graphics[screenshotdir].append(filename)
 779             else:
 780                 logging.warning('Unsupported graphics file found: ' + f)
 781
 782
 783 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 784     """Scan a repo for all files with an extension except APK/OBB
 785
 786     :param apkcache: current cached info about all repo files
 787     :param repodir: repo directory to scan
 788     :param knownapks: list of all known files, as per metadata.read_metadata
 789     :param use_date_from_file: use date from file (instead of current date)
 790                                for newly added files
 791     """
 792
 793     cachechanged = False
 794     repo_files = []
 795     repodir = repodir.encode('utf-8')
 796     for name in os.listdir(repodir):
 797         file_extension = common.get_file_extension(name)
 798         if file_extension == 'apk' or file_extension == 'obb':
 799             continue
 800         filename = os.path.join(repodir, name)
 801         name_utf8 = name.decode('utf-8')
 802         if filename.endswith(b'_src.tar.gz'):
 803             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 804             continue
 805         if not common.is_repo_file(filename):
 806             continue
 807         stat = os.stat(filename)
 808         if stat.st_size == 0:
 809             logging.error(filename + ' is zero size!')
 810             sys.exit(1)
 811
 812         shasum = sha256sum(filename)
 813         usecache = False
 814         if name in apkcache:
 815             repo_file = apkcache[name]
 816             # added time is cached as tuple but used here as datetime instance
 817             if 'added' in repo_file:
 818                 a = repo_file['added']
 819                 if isinstance(a, datetime):
 820                     repo_file['added'] = a
 821                 else:
 822                     repo_file['added'] = datetime(*a[:6])
 823             if repo_file.get('hash') == shasum:
 824                 logging.debug("Reading " + name_utf8 + " from cache")
 825                 usecache = True
 826             else:
 827                 logging.debug("Ignoring stale cache data for " + name)
 828
 829         if not usecache:
 830             logging.debug("Processing " + name_utf8)
 831             repo_file = collections.OrderedDict()
 832             # TODO rename apkname globally to something more generic
 833             repo_file['name'] = name_utf8
 834             repo_file['apkName'] = name_utf8
 835             repo_file['hash'] = shasum
 836             repo_file['hashType'] = 'sha256'
 837             repo_file['versionCode'] = 0
 838             repo_file['versionName'] = shasum
 839             # the static ID is the SHA256 unless it is set in the metadata
 840             repo_file['packageName'] = shasum
 841             n = name_utf8.split('_')
 842             if len(n) == 2:
 843                 packageName = n[0]
 844                 versionCode = n[1].split('.')[0]
 845                 if re.match('^-?[0-9]+$', versionCode) \
 846                    and common.is_valid_package_name(name_utf8.split('_')[0]):
 847                     repo_file['packageName'] = packageName
 848                     repo_file['versionCode'] = int(versionCode)
 849             srcfilename = name + b'_src.tar.gz'
 850             if os.path.exists(os.path.join(repodir, srcfilename)):
 851                 repo_file['srcname'] = srcfilename.decode('utf-8')
 852             repo_file['size'] = stat.st_size
 853
 854             apkcache[name] = repo_file
 855             cachechanged = True
 856
 857         if use_date_from_file:
 858             timestamp = stat.st_ctime
 859             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 860         else:
 861             default_date_param = None
 862
 863         # Record in knownapks, getting the added date at the same time..
 864         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 865                                     default_date=default_date_param)
 866         if added:
 867             repo_file['added'] = added
 868
 869         repo_files.append(repo_file)
 870
 871     return repo_files, cachechanged
 872
 873
 874 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
 875     """Scan the apk with the given filename in the given repo directory.
 876
 877     This also extracts the icons.
 878
 879     :param apkcache: current apk cache information
 880     :param apkfilename: the filename of the apk to scan
 881     :param repodir: repo directory to scan
 882     :param knownapks: known apks info
 883     :param use_date_from_apk: use date from APK (instead of current date)
 884                               for newly added APKs
 885     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
 886      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
 887     """
 888
 889     if ' ' in apkfilename:
 890         logging.critical("Spaces in filenames are not allowed.")
 891         sys.exit(1)
 892
 893     apkfile = os.path.join(repodir, apkfilename)
 894     shasum = sha256sum(apkfile)
 895
 896     cachechanged = False
 897     usecache = False
 898     if apkfilename in apkcache:
 899         apk = apkcache[apkfilename]
 900         if apk.get('hash') == shasum:
 901             logging.debug("Reading " + apkfilename + " from cache")
 902             usecache = True
 903         else:
 904             logging.debug("Ignoring stale cache data for " + apkfilename)
 905
 906     if not usecache:
 907         logging.debug("Processing " + apkfilename)
 908         apk = {}
 909         apk['apkName'] = apkfilename
 910         apk['hash'] = shasum
 911         apk['hashType'] = 'sha256'
 912         srcfilename = apkfilename[:-4] + "_src.tar.gz"
 913         if os.path.exists(os.path.join(repodir, srcfilename)):
 914             apk['srcname'] = srcfilename
 915         apk['size'] = os.path.getsize(apkfile)
 916         apk['uses-permission'] = []
 917         apk['uses-permission-sdk-23'] = []
 918         apk['features'] = []
 919         apk['icons_src'] = {}
 920         apk['icons'] = {}
 921         apk['antiFeatures'] = set()
 922         if has_old_openssl(apkfile):
 923             apk['antiFeatures'].add('KnownVuln')
 924         p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 925         if p.returncode != 0:
 926             if options.delete_unknown:
 927                 if os.path.exists(apkfile):
 928                     logging.error("Failed to get apk information, deleting " + apkfile)
 929                     os.remove(apkfile)
 930                 else:
 931                     logging.error("Could not find {0} to remove it".format(apkfile))
 932             else:
 933                 logging.error("Failed to get apk information, skipping " + apkfile)
 934             return True, None, False
 935         for line in p.output.splitlines():
 936             if line.startswith("package:"):
 937                 try:
 938                     apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 939                     apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 940                     apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 941                 except Exception as e:
 942                     logging.error("Package matching failed: " + str(e))
 943                     logging.info("Line was: " + line)
 944                     sys.exit(1)
 945             elif line.startswith("application:"):
 946                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 947                 # Keep path to non-dpi icon in case we need it
 948                 match = re.match(APK_ICON_PAT_NODPI, line)
 949                 if match:
 950                     apk['icons_src']['-1'] = match.group(1)
 951             elif line.startswith("launchable-activity:"):
 952                 # Only use launchable-activity as fallback to application
 953                 if not apk['name']:
 954                     apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 955                 if '-1' not in apk['icons_src']:
 956                     match = re.match(APK_ICON_PAT_NODPI, line)
 957                     if match:
 958                         apk['icons_src']['-1'] = match.group(1)
 959             elif line.startswith("application-icon-"):
 960                 match = re.match(APK_ICON_PAT, line)
 961                 if match:
 962                     density = match.group(1)
 963                     path = match.group(2)
 964                     apk['icons_src'][density] = path
 965             elif line.startswith("sdkVersion:"):
 966                 m = re.match(APK_SDK_VERSION_PAT, line)
 967                 if m is None:
 968                     logging.error(line.replace('sdkVersion:', '')
 969                                   + ' is not a valid minSdkVersion!')
 970                 else:
 971                     apk['minSdkVersion'] = m.group(1)
 972                     # if target not set, default to min
 973                     if 'targetSdkVersion' not in apk:
 974                         apk['targetSdkVersion'] = m.group(1)
 975             elif line.startswith("targetSdkVersion:"):
 976                 m = re.match(APK_SDK_VERSION_PAT, line)
 977                 if m is None:
 978                     logging.error(line.replace('targetSdkVersion:', '')
 979                                   + ' is not a valid targetSdkVersion!')
 980                 else:
 981                     apk['targetSdkVersion'] = m.group(1)
 982             elif line.startswith("maxSdkVersion:"):
 983                 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 984             elif line.startswith("native-code:"):
 985                 apk['nativecode'] = []
 986                 for arch in line[13:].split(' '):
 987                     apk['nativecode'].append(arch[1:-1])
 988             elif line.startswith('uses-permission:'):
 989                 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 990                 if perm_match['maxSdkVersion']:
 991                     perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 992                 permission = UsesPermission(
 993                     perm_match['name'],
 994                     perm_match['maxSdkVersion']
 995                 )
 996
 997                 apk['uses-permission'].append(permission)
 998             elif line.startswith('uses-permission-sdk-23:'):
 999                 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1000                 if perm_match['maxSdkVersion']:
1001                     perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1002                 permission_sdk_23 = UsesPermissionSdk23(
1003                     perm_match['name'],
1004                     perm_match['maxSdkVersion']
1005                 )
1006
1007                 apk['uses-permission-sdk-23'].append(permission_sdk_23)
1008
1009             elif line.startswith('uses-feature:'):
1010                 feature = re.match(APK_FEATURE_PAT, line).group(1)
1011                 # Filter out this, it's only added with the latest SDK tools and
1012                 # causes problems for lots of apps.
1013                 if feature != "android.hardware.screen.portrait" \
1014                         and feature != "android.hardware.screen.landscape":
1015                     if feature.startswith("android.feature."):
1016                         feature = feature[16:]
1017                     apk['features'].add(feature)
1018
1019         if 'minSdkVersion' not in apk:
1020             logging.warn("No SDK version information found in {0}".format(apkfile))
1021             apk['minSdkVersion'] = 1
1022
1023         # Check for debuggable apks...
1024         if common.isApkAndDebuggable(apkfile, config):
1025             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1026
1027         # Get the signature (or md5 of, to be precise)...
1028         logging.debug('Getting signature of {0}'.format(apkfile))
1029         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
1030         if not apk['sig']:
1031             logging.critical("Failed to get apk signature")
1032             sys.exit(1)
1033
1034         apkzip = zipfile.ZipFile(apkfile, 'r')
1035
1036         # if an APK has files newer than the system time, suggest updating
1037         # the system clock.  This is useful for offline systems, used for
1038         # signing, which do not have another source of clock sync info. It
1039         # has to be more than 24 hours newer because ZIP/APK files do not
1040         # store timezone info
1041         manifest = apkzip.getinfo('AndroidManifest.xml')
1042         if manifest.date_time[1] == 0:  # month can't be zero
1043             logging.debug('AndroidManifest.xml has no date')
1044         else:
1045             dt_obj = datetime(*manifest.date_time)
1046             checkdt = dt_obj - timedelta(1)
1047             if datetime.today() < checkdt:
1048                 logging.warn('System clock is older than manifest in: '
1049                              + apkfilename
1050                              + '\nSet clock to that time using:\n'
1051                              + 'sudo date -s "' + str(dt_obj) + '"')
1052
1053         iconfilename = "%s.%s.png" % (
1054             apk['packageName'],
1055             apk['versionCode'])
1056
1057         # Extract the icon file...
1058         empty_densities = []
1059         for density in screen_densities:
1060             if density not in apk['icons_src']:
1061                 empty_densities.append(density)
1062                 continue
1063             iconsrc = apk['icons_src'][density]
1064             icon_dir = get_icon_dir(repodir, density)
1065             icondest = os.path.join(icon_dir, iconfilename)
1066
1067             try:
1068                 with open(icondest, 'wb') as f:
1069                     f.write(get_icon_bytes(apkzip, iconsrc))
1070                 apk['icons'][density] = iconfilename
1071
1072             except Exception as e:
1073                 logging.warn("Error retrieving icon file: %s" % (e))
1074                 del apk['icons'][density]
1075                 del apk['icons_src'][density]
1076                 empty_densities.append(density)
1077
1078         if '-1' in apk['icons_src']:
1079             iconsrc = apk['icons_src']['-1']
1080             iconpath = os.path.join(
1081                 get_icon_dir(repodir, '0'), iconfilename)
1082             with open(iconpath, 'wb') as f:
1083                 f.write(get_icon_bytes(apkzip, iconsrc))
1084             try:
1085                 im = Image.open(iconpath)
1086                 dpi = px_to_dpi(im.size[0])
1087                 for density in screen_densities:
1088                     if density in apk['icons']:
1089                         break
1090                     if density == screen_densities[-1] or dpi >= int(density):
1091                         apk['icons'][density] = iconfilename
1092                         shutil.move(iconpath,
1093                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
1094                         empty_densities.remove(density)
1095                         break
1096             except Exception as e:
1097                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
1098
1099         if apk['icons']:
1100             apk['icon'] = iconfilename
1101
1102         apkzip.close()
1103
1104         # First try resizing down to not lose quality
1105         last_density = None
1106         for density in screen_densities:
1107             if density not in empty_densities:
1108                 last_density = density
1109                 continue
1110             if last_density is None:
1111                 continue
1112             logging.debug("Density %s not available, resizing down from %s"
1113                           % (density, last_density))
1114
1115             last_iconpath = os.path.join(
1116                 get_icon_dir(repodir, last_density), iconfilename)
1117             iconpath = os.path.join(
1118                 get_icon_dir(repodir, density), iconfilename)
1119             fp = None
1120             try:
1121                 fp = open(last_iconpath, 'rb')
1122                 im = Image.open(fp)
1123
1124                 size = dpi_to_px(density)
1125
1126                 im.thumbnail((size, size), Image.ANTIALIAS)
1127                 im.save(iconpath, "PNG")
1128                 empty_densities.remove(density)
1129             except Exception as e:
1130                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1131             finally:
1132                 if fp:
1133                     fp.close()
1134
1135         # Then just copy from the highest resolution available
1136         last_density = None
1137         for density in reversed(screen_densities):
1138             if density not in empty_densities:
1139                 last_density = density
1140                 continue
1141             if last_density is None:
1142                 continue
1143             logging.debug("Density %s not available, copying from lower density %s"
1144                           % (density, last_density))
1145
1146             shutil.copyfile(
1147                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1148                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1149
1150             empty_densities.remove(density)
1151
1152         for density in screen_densities:
1153             icon_dir = get_icon_dir(repodir, density)
1154             icondest = os.path.join(icon_dir, iconfilename)
1155             resize_icon(icondest, density)
1156
1157         # Copy from icons-mdpi to icons since mdpi is the baseline density
1158         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1159         if os.path.isfile(baseline):
1160             apk['icons']['0'] = iconfilename
1161             shutil.copyfile(baseline,
1162                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1163
1164         if use_date_from_apk and manifest.date_time[1] != 0:
1165             default_date_param = datetime(*manifest.date_time)
1166         else:
1167             default_date_param = None
1168
1169         # Record in known apks, getting the added date at the same time..
1170         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1171                                     default_date=default_date_param)
1172         if added:
1173             apk['added'] = added
1174
1175         apkcache[apkfilename] = apk
1176         cachechanged = True
1177
1178     return False, apk, cachechanged
1179
1180
1181 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1182     """Scan the apks in the given repo directory.
1183
1184     This also extracts the icons.
1185
1186     :param apkcache: current apk cache information
1187     :param repodir: repo directory to scan
1188     :param knownapks: known apks info
1189     :param use_date_from_apk: use date from APK (instead of current date)
1190                               for newly added APKs
1191     :returns: (apks, cachechanged) where apks is a list of apk information,
1192               and cachechanged is True if the apkcache got changed.
1193     """
1194
1195     cachechanged = False
1196
1197     for icon_dir in get_all_icon_dirs(repodir):
1198         if os.path.exists(icon_dir):
1199             if options.clean:
1200                 shutil.rmtree(icon_dir)
1201                 os.makedirs(icon_dir)
1202         else:
1203             os.makedirs(icon_dir)
1204
1205     apks = []
1206     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1207         apkfilename = apkfile[len(repodir) + 1:]
1208         (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1209         if skip:
1210             continue
1211         apks.append(apk)
1212
1213     return apks, cachechanged
1214
1215
1216 def apply_info_from_latest_apk(apps, apks):
1217     """
1218     Some information from the apks needs to be applied up to the application level.
1219     When doing this, we use the info from the most recent version's apk.
1220     We deal with figuring out when the app was added and last updated at the same time.
1221     """
1222     for appid, app in apps.items():
1223         bestver = UNSET_VERSION_CODE
1224         for apk in apks:
1225             if apk['packageName'] == appid:
1226                 if apk['versionCode'] > bestver:
1227                     bestver = apk['versionCode']
1228                     bestapk = apk
1229
1230                 if 'added' in apk:
1231                     if not app.added or apk['added'] < app.added:
1232                         app.added = apk['added']
1233                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1234                         app.lastUpdated = apk['added']
1235
1236         if not app.added:
1237             logging.debug("Don't know when " + appid + " was added")
1238         if not app.lastUpdated:
1239             logging.debug("Don't know when " + appid + " was last updated")
1240
1241         if bestver == UNSET_VERSION_CODE:
1242
1243             if app.Name is None:
1244                 app.Name = app.AutoName or appid
1245             app.icon = None
1246             logging.debug("Application " + appid + " has no packages")
1247         else:
1248             if app.Name is None:
1249                 app.Name = bestapk['name']
1250             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1251             if app.CurrentVersionCode is None:
1252                 app.CurrentVersionCode = str(bestver)
1253
1254
1255 def make_categories_txt(repodir, categories):
1256     '''Write a category list in the repo to allow quick access'''
1257     catdata = ''
1258     for cat in sorted(categories):
1259         catdata += cat + '\n'
1260     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1261         f.write(catdata)
1262
1263
1264 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1265
1266     for appid, app in apps.items():
1267
1268         if app.ArchivePolicy:
1269             keepversions = int(app.ArchivePolicy[:-9])
1270         else:
1271             keepversions = defaultkeepversions
1272
1273         def filter_apk_list_sorted(apk_list):
1274             res = []
1275             for apk in apk_list:
1276                 if apk['packageName'] == appid:
1277                     res.append(apk)
1278
1279             # Sort the apk list by version code. First is highest/newest.
1280             return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1281
1282         def move_file(from_dir, to_dir, filename, ignore_missing):
1283             from_path = os.path.join(from_dir, filename)
1284             if ignore_missing and not os.path.exists(from_path):
1285                 return
1286             to_path = os.path.join(to_dir, filename)
1287             shutil.move(from_path, to_path)
1288
1289         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1290                       .format(appid, len(apks), keepversions, len(archapks)))
1291
1292         if len(apks) > keepversions:
1293             apklist = filter_apk_list_sorted(apks)
1294             # Move back the ones we don't want.
1295             for apk in apklist[keepversions:]:
1296                 logging.info("Moving " + apk['apkName'] + " to archive")
1297                 move_file(repodir, archivedir, apk['apkName'], False)
1298                 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1299                 for density in all_screen_densities:
1300                     repo_icon_dir = get_icon_dir(repodir, density)
1301                     archive_icon_dir = get_icon_dir(archivedir, density)
1302                     if density not in apk['icons']:
1303                         continue
1304                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1305                 if 'srcname' in apk:
1306                     move_file(repodir, archivedir, apk['srcname'], False)
1307                 archapks.append(apk)
1308                 apks.remove(apk)
1309         elif len(apks) < keepversions and len(archapks) > 0:
1310             required = keepversions - len(apks)
1311             archapklist = filter_apk_list_sorted(archapks)
1312             # Move forward the ones we want again.
1313             for apk in archapklist[:required]:
1314                 logging.info("Moving " + apk['apkName'] + " from archive")
1315                 move_file(archivedir, repodir, apk['apkName'], False)
1316                 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1317                 for density in all_screen_densities:
1318                     repo_icon_dir = get_icon_dir(repodir, density)
1319                     archive_icon_dir = get_icon_dir(archivedir, density)
1320                     if density not in apk['icons']:
1321                         continue
1322                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1323                 if 'srcname' in apk:
1324                     move_file(archivedir, repodir, apk['srcname'], False)
1325                 archapks.remove(apk)
1326                 apks.append(apk)
1327
1328
1329 def add_apks_to_per_app_repos(repodir, apks):
1330     apks_per_app = dict()
1331     for apk in apks:
1332         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1333         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1334         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1335         apks_per_app[apk['packageName']] = apk
1336
1337         if not os.path.exists(apk['per_app_icons']):
1338             logging.info('Adding new repo for only ' + apk['packageName'])
1339             os.makedirs(apk['per_app_icons'])
1340
1341         apkpath = os.path.join(repodir, apk['apkName'])
1342         shutil.copy(apkpath, apk['per_app_repo'])
1343         apksigpath = apkpath + '.sig'
1344         if os.path.exists(apksigpath):
1345             shutil.copy(apksigpath, apk['per_app_repo'])
1346         apkascpath = apkpath + '.asc'
1347         if os.path.exists(apkascpath):
1348             shutil.copy(apkascpath, apk['per_app_repo'])
1349
1350
1351 config = None
1352 options = None
1353
1354
1355 def main():
1356
1357     global config, options
1358
1359     # Parse command line...
1360     parser = ArgumentParser()
1361     common.setup_global_opts(parser)
1362     parser.add_argument("--create-key", action="store_true", default=False,
1363                         help="Create a repo signing key in a keystore")
1364     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1365                         help="Create skeleton metadata files that are missing")
1366     parser.add_argument("--delete-unknown", action="store_true", default=False,
1367                         help="Delete APKs and/or OBBs without metadata from the repo")
1368     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1369                         help="Report on build data status")
1370     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1371                         help="Interactively ask about things that need updating.")
1372     parser.add_argument("-I", "--icons", action="store_true", default=False,
1373                         help="Resize all the icons exceeding the max pixel size and exit")
1374     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1375                         help="Specify editor to use in interactive mode. Default " +
1376                         "is /etc/alternatives/editor")
1377     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1378                         help="Update the wiki")
1379     parser.add_argument("--pretty", action="store_true", default=False,
1380                         help="Produce human-readable index.xml")
1381     parser.add_argument("--clean", action="store_true", default=False,
1382                         help="Clean update - don't uses caches, reprocess all apks")
1383     parser.add_argument("--nosign", action="store_true", default=False,
1384                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1385     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1386                         help="Use date from apk instead of current time for newly added apks")
1387     metadata.add_metadata_arguments(parser)
1388     options = parser.parse_args()
1389     metadata.warnings_action = options.W
1390
1391     config = common.read_config(options)
1392
1393     if not ('jarsigner' in config and 'keytool' in config):
1394         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1395         sys.exit(1)
1396
1397     repodirs = ['repo']
1398     if config['archive_older'] != 0:
1399         repodirs.append('archive')
1400         if not os.path.exists('archive'):
1401             os.mkdir('archive')
1402
1403     if options.icons:
1404         resize_all_icons(repodirs)
1405         sys.exit(0)
1406
1407     # check that icons exist now, rather than fail at the end of `fdroid update`
1408     for k in ['repo_icon', 'archive_icon']:
1409         if k in config:
1410             if not os.path.exists(config[k]):
1411                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1412                 sys.exit(1)
1413
1414     # if the user asks to create a keystore, do it now, reusing whatever it can
1415     if options.create_key:
1416         if os.path.exists(config['keystore']):
1417             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1418             logging.critical("\t'" + config['keystore'] + "'")
1419             sys.exit(1)
1420
1421         if 'repo_keyalias' not in config:
1422             config['repo_keyalias'] = socket.getfqdn()
1423             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1424         if 'keydname' not in config:
1425             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1426             common.write_to_config(config, 'keydname', config['keydname'])
1427         if 'keystore' not in config:
1428             config['keystore'] = common.default_config.keystore
1429             common.write_to_config(config, 'keystore', config['keystore'])
1430
1431         password = common.genpassword()
1432         if 'keystorepass' not in config:
1433             config['keystorepass'] = password
1434             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1435         if 'keypass' not in config:
1436             config['keypass'] = password
1437             common.write_to_config(config, 'keypass', config['keypass'])
1438         common.genkeystore(config)
1439
1440     # Get all apps...
1441     apps = metadata.read_metadata()
1442
1443     # Generate a list of categories...
1444     categories = set()
1445     for app in apps.values():
1446         categories.update(app.Categories)
1447
1448     # Read known apks data (will be updated and written back when we've finished)
1449     knownapks = common.KnownApks()
1450
1451     # Get APK cache
1452     apkcache = get_cache()
1453
1454     # Delete builds for disabled apps
1455     delete_disabled_builds(apps, apkcache, repodirs)
1456
1457     # Scan all apks in the main repo
1458     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1459
1460     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1461                                            options.use_date_from_apk)
1462     cachechanged = cachechanged or fcachechanged
1463     apks += files
1464     # Generate warnings for apk's with no metadata (or create skeleton
1465     # metadata files, if requested on the command line)
1466     newmetadata = False
1467     for apk in apks:
1468         if apk['packageName'] not in apps:
1469             if options.create_metadata:
1470                 if 'name' not in apk:
1471                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1472                     continue
1473                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1474                 f.write("License:Unknown\n")
1475                 f.write("Web Site:\n")
1476                 f.write("Source Code:\n")
1477                 f.write("Issue Tracker:\n")
1478                 f.write("Changelog:\n")
1479                 f.write("Summary:" + apk['name'] + "\n")
1480                 f.write("Description:\n")
1481                 f.write(apk['name'] + "\n")
1482                 f.write(".\n")
1483                 f.write("Name:" + apk['name'] + "\n")
1484                 f.close()
1485                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1486                 newmetadata = True
1487             else:
1488                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1489                 if options.delete_unknown:
1490                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1491                     rmf = os.path.join(repodirs[0], apk['apkName'])
1492                     if not os.path.exists(rmf):
1493                         logging.error("Could not find {0} to remove it".format(rmf))
1494                     else:
1495                         os.remove(rmf)
1496                 else:
1497                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1498
1499     # update the metadata with the newly created ones included
1500     if newmetadata:
1501         apps = metadata.read_metadata()
1502
1503     copy_triple_t_store_metadata(apps)
1504     insert_obbs(repodirs[0], apps, apks)
1505     insert_localized_app_metadata(apps)
1506
1507     # Scan the archive repo for apks as well
1508     if len(repodirs) > 1:
1509         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1510         if cc:
1511             cachechanged = True
1512     else:
1513         archapks = []
1514
1515     # Apply information from latest apks to the application and update dates
1516     apply_info_from_latest_apk(apps, apks + archapks)
1517
1518     # Sort the app list by name, then the web site doesn't have to by default.
1519     # (we had to wait until we'd scanned the apks to do this, because mostly the
1520     # name comes from there!)
1521     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1522
1523     # APKs are placed into multiple repos based on the app package, providing
1524     # per-app subscription feeds for nightly builds and things like it
1525     if config['per_app_repos']:
1526         add_apks_to_per_app_repos(repodirs[0], apks)
1527         for appid, app in apps.items():
1528             repodir = os.path.join(appid, 'fdroid', 'repo')
1529             appdict = dict()
1530             appdict[appid] = app
1531             if os.path.isdir(repodir):
1532                 index.make(appdict, [appid], apks, repodir, False)
1533             else:
1534                 logging.info('Skipping index generation for ' + appid)
1535         return
1536
1537     if len(repodirs) > 1:
1538         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1539
1540     # Make the index for the main repo...
1541     index.make(apps, sortedids, apks, repodirs[0], False)
1542     make_categories_txt(repodirs[0], categories)
1543
1544     # If there's an archive repo,  make the index for it. We already scanned it
1545     # earlier on.
1546     if len(repodirs) > 1:
1547         index.make(apps, sortedids, archapks, repodirs[1], True)
1548
1549     git_remote = config.get('binary_transparency_remote')
1550     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1551         btlog.make_binary_transparency_log(repodirs)
1552
1553     if config['update_stats']:
1554         # Update known apks info...
1555         knownapks.writeifchanged()
1556
1557         # Generate latest apps data for widget
1558         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1559             data = ''
1560             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1561                 for line in f:
1562                     appid = line.rstrip()
1563                     data += appid + "\t"
1564                     app = apps[appid]
1565                     data += app.Name + "\t"
1566                     if app.icon is not None:
1567                         data += app.icon + "\t"
1568                     data += app.License + "\n"
1569             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1570                 f.write(data)
1571
1572     if cachechanged:
1573         write_cache(apkcache)
1574
1575     # Update the wiki...
1576     if options.wiki:
1577         update_wiki(apps, sortedids, apks + archapks)
1578
1579     logging.info("Finished.")
1580
1581
1582 if __name__ == "__main__":
1583     main()