fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import json
  27 import re
  28 import socket
  29 import zipfile
  30 import hashlib
  31 import pickle
  32 import platform
  33 from datetime import datetime, timedelta
  34 from argparse import ArgumentParser
  35
  36 import collections
  37 from pyasn1.error import PyAsn1Error
  38 from pyasn1.codec.der import decoder, encoder
  39 from pyasn1_modules import rfc2315
  40 from binascii import hexlify
  41
  42 from PIL import Image
  43 import logging
  44
  45 from . import common
  46 from . import index
  47 from . import metadata
  48 from .common import FDroidPopen, SdkToolsPopen
  49
  50 METADATA_VERSION = 18
  51
  52 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  53 UNSET_VERSION_CODE = -0x100000000
  54
  55 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  56 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  57 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  58 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  59 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  60 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  61 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  62 APK_PERMISSION_PAT = \
  63     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  64 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  65
  66 screen_densities = ['640', '480', '320', '240', '160', '120']
  67
  68 all_screen_densities = ['0'] + screen_densities
  69
  70 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  71 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  72
  73
  74 def dpi_to_px(density):
  75     return (int(density) * 48) / 160
  76
  77
  78 def px_to_dpi(px):
  79     return (int(px) * 160) / 48
  80
  81
  82 def get_icon_dir(repodir, density):
  83     if density == '0':
  84         return os.path.join(repodir, "icons")
  85     return os.path.join(repodir, "icons-%s" % density)
  86
  87
  88 def get_icon_dirs(repodir):
  89     for density in screen_densities:
  90         yield get_icon_dir(repodir, density)
  91
  92
  93 def get_all_icon_dirs(repodir):
  94     for density in all_screen_densities:
  95         yield get_icon_dir(repodir, density)
  96
  97
  98 def update_wiki(apps, sortedids, apks):
  99     """Update the wiki
 100
 101     :param apps: fully populated list of all applications
 102     :param apks: all apks, except...
 103     """
 104     logging.info("Updating wiki")
 105     wikicat = 'Apps'
 106     wikiredircat = 'App Redirects'
 107     import mwclient
 108     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 109                          path=config['wiki_path'])
 110     site.login(config['wiki_user'], config['wiki_password'])
 111     generated_pages = {}
 112     generated_redirects = {}
 113
 114     for appid in sortedids:
 115         app = metadata.App(apps[appid])
 116
 117         wikidata = ''
 118         if app.Disabled:
 119             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 120         if app.AntiFeatures:
 121             for af in app.AntiFeatures:
 122                 wikidata += '{{AntiFeature|' + af + '}}\n'
 123         if app.RequiresRoot:
 124             requiresroot = 'Yes'
 125         else:
 126             requiresroot = 'No'
 127         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 128             appid,
 129             app.Name,
 130             app.added.strftime('%Y-%m-%d') if app.added else '',
 131             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 132             app.SourceCode,
 133             app.IssueTracker,
 134             app.WebSite,
 135             app.Changelog,
 136             app.Donate,
 137             app.FlattrID,
 138             app.Bitcoin,
 139             app.Litecoin,
 140             app.License,
 141             requiresroot,
 142             app.AuthorName,
 143             app.AuthorEmail)
 144
 145         if app.Provides:
 146             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 147
 148         wikidata += app.Summary
 149         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 150
 151         wikidata += "=Description=\n"
 152         wikidata += metadata.description_wiki(app.Description) + "\n"
 153
 154         wikidata += "=Maintainer Notes=\n"
 155         if app.MaintainerNotes:
 156             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 157         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 158
 159         # Get a list of all packages for this application...
 160         apklist = []
 161         gotcurrentver = False
 162         cantupdate = False
 163         buildfails = False
 164         for apk in apks:
 165             if apk['packageName'] == appid:
 166                 if str(apk['versionCode']) == app.CurrentVersionCode:
 167                     gotcurrentver = True
 168                 apklist.append(apk)
 169         # Include ones we can't build, as a special case...
 170         for build in app.builds:
 171             if build.disable:
 172                 if build.versionCode == app.CurrentVersionCode:
 173                     cantupdate = True
 174                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 175                 apklist.append({'versionCode': int(build.versionCode),
 176                                 'versionName': build.versionName,
 177                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 178                                 })
 179             else:
 180                 builtit = False
 181                 for apk in apklist:
 182                     if apk['versionCode'] == int(build.versionCode):
 183                         builtit = True
 184                         break
 185                 if not builtit:
 186                     buildfails = True
 187                     apklist.append({'versionCode': int(build.versionCode),
 188                                     'versionName': build.versionName,
 189                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 190                                     })
 191         if app.CurrentVersionCode == '0':
 192             cantupdate = True
 193         # Sort with most recent first...
 194         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 195
 196         wikidata += "=Versions=\n"
 197         if len(apklist) == 0:
 198             wikidata += "We currently have no versions of this app available."
 199         elif not gotcurrentver:
 200             wikidata += "We don't have the current version of this app."
 201         else:
 202             wikidata += "We have the current version of this app."
 203         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 204         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 205         if len(app.NoSourceSince) > 0:
 206             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 207         if len(app.CurrentVersion) > 0:
 208             wikidata += "The current (recommended) version is " + app.CurrentVersion
 209             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 210         validapks = 0
 211         for apk in apklist:
 212             wikidata += "==" + apk['versionName'] + "==\n"
 213
 214             if 'buildproblem' in apk:
 215                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 216             else:
 217                 validapks += 1
 218                 wikidata += "This version is built and signed by "
 219                 if 'srcname' in apk:
 220                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 221                 else:
 222                     wikidata += "the original developer.\n\n"
 223             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 224
 225         wikidata += '\n[[Category:' + wikicat + ']]\n'
 226         if len(app.NoSourceSince) > 0:
 227             wikidata += '\n[[Category:Apps missing source code]]\n'
 228         if validapks == 0 and not app.Disabled:
 229             wikidata += '\n[[Category:Apps with no packages]]\n'
 230         if cantupdate and not app.Disabled:
 231             wikidata += "\n[[Category:Apps we cannot update]]\n"
 232         if buildfails and not app.Disabled:
 233             wikidata += "\n[[Category:Apps with failing builds]]\n"
 234         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 235             wikidata += '\n[[Category:Apps to Update]]\n'
 236         if app.Disabled:
 237             wikidata += '\n[[Category:Apps that are disabled]]\n'
 238         if app.UpdateCheckMode == 'None' and not app.Disabled:
 239             wikidata += '\n[[Category:Apps with no update check]]\n'
 240         for appcat in app.Categories:
 241             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 242
 243         # We can't have underscores in the page name, even if they're in
 244         # the package ID, because MediaWiki messes with them...
 245         pagename = appid.replace('_', ' ')
 246
 247         # Drop a trailing newline, because mediawiki is going to drop it anyway
 248         # and it we don't we'll think the page has changed when it hasn't...
 249         if wikidata.endswith('\n'):
 250             wikidata = wikidata[:-1]
 251
 252         generated_pages[pagename] = wikidata
 253
 254         # Make a redirect from the name to the ID too, unless there's
 255         # already an existing page with the name and it isn't a redirect.
 256         noclobber = False
 257         apppagename = app.Name.replace('_', ' ')
 258         apppagename = apppagename.replace('{', '')
 259         apppagename = apppagename.replace('}', ' ')
 260         apppagename = apppagename.replace(':', ' ')
 261         apppagename = apppagename.replace('[', ' ')
 262         apppagename = apppagename.replace(']', ' ')
 263         # Drop double spaces caused mostly by replacing ':' above
 264         apppagename = apppagename.replace('  ', ' ')
 265         for expagename in site.allpages(prefix=apppagename,
 266                                         filterredir='nonredirects',
 267                                         generator=False):
 268             if expagename == apppagename:
 269                 noclobber = True
 270         # Another reason not to make the redirect page is if the app name
 271         # is the same as it's ID, because that will overwrite the real page
 272         # with an redirect to itself! (Although it seems like an odd
 273         # scenario this happens a lot, e.g. where there is metadata but no
 274         # builds or binaries to extract a name from.
 275         if apppagename == pagename:
 276             noclobber = True
 277         if not noclobber:
 278             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 279
 280     for tcat, genp in [(wikicat, generated_pages),
 281                        (wikiredircat, generated_redirects)]:
 282         catpages = site.Pages['Category:' + tcat]
 283         existingpages = []
 284         for page in catpages:
 285             existingpages.append(page.name)
 286             if page.name in genp:
 287                 pagetxt = page.edit()
 288                 if pagetxt != genp[page.name]:
 289                     logging.debug("Updating modified page " + page.name)
 290                     page.save(genp[page.name], summary='Auto-updated')
 291                 else:
 292                     logging.debug("Page " + page.name + " is unchanged")
 293             else:
 294                 logging.warn("Deleting page " + page.name)
 295                 page.delete('No longer published')
 296         for pagename, text in genp.items():
 297             logging.debug("Checking " + pagename)
 298             if pagename not in existingpages:
 299                 logging.debug("Creating page " + pagename)
 300                 try:
 301                     newpage = site.Pages[pagename]
 302                     newpage.save(text, summary='Auto-created')
 303                 except Exception as e:
 304                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 305
 306     # Purge server cache to ensure counts are up to date
 307     site.pages['Repository Maintenance'].purge()
 308
 309
 310 def delete_disabled_builds(apps, apkcache, repodirs):
 311     """Delete disabled build outputs.
 312
 313     :param apps: list of all applications, as per metadata.read_metadata
 314     :param apkcache: current apk cache information
 315     :param repodirs: the repo directories to process
 316     """
 317     for appid, app in apps.items():
 318         for build in app['builds']:
 319             if not build.disable:
 320                 continue
 321             apkfilename = appid + '_' + str(build.versionCode) + '.apk'
 322             iconfilename = "%s.%s.png" % (
 323                 appid,
 324                 build.versionCode)
 325             for repodir in repodirs:
 326                 files = [
 327                     os.path.join(repodir, apkfilename),
 328                     os.path.join(repodir, apkfilename + '.asc'),
 329                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 330                 ]
 331                 for density in all_screen_densities:
 332                     repo_dir = get_icon_dir(repodir, density)
 333                     files.append(os.path.join(repo_dir, iconfilename))
 334
 335                 for f in files:
 336                     if os.path.exists(f):
 337                         logging.info("Deleting disabled build output " + f)
 338                         os.remove(f)
 339             if apkfilename in apkcache:
 340                 del apkcache[apkfilename]
 341
 342
 343 def resize_icon(iconpath, density):
 344
 345     if not os.path.isfile(iconpath):
 346         return
 347
 348     fp = None
 349     try:
 350         fp = open(iconpath, 'rb')
 351         im = Image.open(fp)
 352         size = dpi_to_px(density)
 353
 354         if any(length > size for length in im.size):
 355             oldsize = im.size
 356             im.thumbnail((size, size), Image.ANTIALIAS)
 357             logging.debug("%s was too large at %s - new size is %s" % (
 358                 iconpath, oldsize, im.size))
 359             im.save(iconpath, "PNG")
 360
 361     except Exception as e:
 362         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 363
 364     finally:
 365         if fp:
 366             fp.close()
 367
 368
 369 def resize_all_icons(repodirs):
 370     """Resize all icons that exceed the max size
 371
 372     :param repodirs: the repo directories to process
 373     """
 374     for repodir in repodirs:
 375         for density in screen_densities:
 376             icon_dir = get_icon_dir(repodir, density)
 377             icon_glob = os.path.join(icon_dir, '*.png')
 378             for iconpath in glob.glob(icon_glob):
 379                 resize_icon(iconpath, density)
 380
 381
 382 # A signature block file with a .DSA, .RSA, or .EC extension
 383 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 384
 385
 386 def getsig(apkpath):
 387     """ Get the signing certificate of an apk. To get the same md5 has that
 388     Android gets, we encode the .RSA certificate in a specific format and pass
 389     it hex-encoded to the md5 digest algorithm.
 390
 391     :param apkpath: path to the apk
 392     :returns: A string containing the md5 of the signature of the apk or None
 393               if an error occurred.
 394     """
 395
 396     cert = None
 397
 398     # verify the jar signature is correct
 399     args = [config['jarsigner'], '-verify', apkpath]
 400     p = FDroidPopen(args)
 401     if p.returncode != 0:
 402         logging.critical(apkpath + " has a bad signature!")
 403         return None
 404
 405     with zipfile.ZipFile(apkpath, 'r') as apk:
 406
 407         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 408
 409         if len(certs) < 1:
 410             logging.error("Found no signing certificates on %s" % apkpath)
 411             return None
 412         if len(certs) > 1:
 413             logging.error("Found multiple signing certificates on %s" % apkpath)
 414             return None
 415
 416         cert = apk.read(certs[0])
 417
 418     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 419     if content.getComponentByName('contentType') != rfc2315.signedData:
 420         logging.error("Unexpected format.")
 421         return None
 422
 423     content = decoder.decode(content.getComponentByName('content'),
 424                              asn1Spec=rfc2315.SignedData())[0]
 425     try:
 426         certificates = content.getComponentByName('certificates')
 427     except PyAsn1Error:
 428         logging.error("Certificates not found.")
 429         return None
 430
 431     cert_encoded = encoder.encode(certificates)[4:]
 432
 433     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 434
 435
 436 def get_cache_file():
 437     return os.path.join('tmp', 'apkcache')
 438
 439
 440 def get_cache():
 441     """
 442     Gather information about all the apk files in the repo directory,
 443     using cached data if possible.
 444     :return: apkcache
 445     """
 446     apkcachefile = get_cache_file()
 447     if not options.clean and os.path.exists(apkcachefile):
 448         with open(apkcachefile, 'rb') as cf:
 449             apkcache = pickle.load(cf, encoding='utf-8')
 450         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
 451             apkcache = {}
 452     else:
 453         apkcache = {}
 454
 455     return apkcache
 456
 457
 458 def write_cache(apkcache):
 459     apkcachefile = get_cache_file()
 460     cache_path = os.path.dirname(apkcachefile)
 461     if not os.path.exists(cache_path):
 462         os.makedirs(cache_path)
 463     apkcache["METADATA_VERSION"] = METADATA_VERSION
 464     with open(apkcachefile, 'wb') as cf:
 465         pickle.dump(apkcache, cf)
 466
 467
 468 def get_icon_bytes(apkzip, iconsrc):
 469     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 470     try:
 471         return apkzip.read(iconsrc)
 472     except KeyError:
 473         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 474
 475
 476 def sha256sum(filename):
 477     '''Calculate the sha256 of the given file'''
 478     sha = hashlib.sha256()
 479     with open(filename, 'rb') as f:
 480         while True:
 481             t = f.read(16384)
 482             if len(t) == 0:
 483                 break
 484             sha.update(t)
 485     return sha.hexdigest()
 486
 487
 488 def has_old_openssl(filename):
 489     '''checks for known vulnerable openssl versions in the APK'''
 490
 491     # statically load this pattern
 492     if not hasattr(has_old_openssl, "pattern"):
 493         has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 494
 495     with zipfile.ZipFile(filename) as zf:
 496         for name in zf.namelist():
 497             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 498                 lib = zf.open(name)
 499                 while True:
 500                     chunk = lib.read(4096)
 501                     if chunk == b'':
 502                         break
 503                     m = has_old_openssl.pattern.search(chunk)
 504                     if m:
 505                         version = m.group(1).decode('ascii')
 506                         if version.startswith('1.0.1') and version[5] >= 'r' \
 507                            or version.startswith('1.0.2') and version[5] >= 'f':
 508                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 509                         else:
 510                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 511                             return True
 512                         break
 513     return False
 514
 515
 516 def insert_obbs(repodir, apps, apks):
 517     """Scans the .obb files in a given repo directory and adds them to the
 518     relevant APK instances.  OBB files have versionCodes like APK
 519     files, and they are loosely associated.  If there is an OBB file
 520     present, then any APK with the same or higher versionCode will use
 521     that OBB file.  There are two OBB types: main and patch, each APK
 522     can only have only have one of each.
 523
 524     https://developer.android.com/google/play/expansion-files.html
 525
 526     :param repodir: repo directory to scan
 527     :param apps: list of current, valid apps
 528     :param apks: current information on all APKs
 529
 530     """
 531
 532     def obbWarnDelete(f, msg):
 533         logging.warning(msg + f)
 534         if options.delete_unknown:
 535             logging.error("Deleting unknown file: " + f)
 536             os.remove(f)
 537
 538     obbs = []
 539     java_Integer_MIN_VALUE = -pow(2, 31)
 540     currentPackageNames = apps.keys()
 541     for f in glob.glob(os.path.join(repodir, '*.obb')):
 542         obbfile = os.path.basename(f)
 543         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 544         chunks = obbfile.split('.')
 545         if chunks[0] != 'main' and chunks[0] != 'patch':
 546             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 547             continue
 548         if not re.match(r'^-?[0-9]+$', chunks[1]):
 549             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 550             continue
 551         versionCode = int(chunks[1])
 552         packagename = ".".join(chunks[2:-1])
 553
 554         highestVersionCode = java_Integer_MIN_VALUE
 555         if packagename not in currentPackageNames:
 556             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 557             continue
 558         for apk in apks:
 559             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 560                 highestVersionCode = apk['versionCode']
 561         if versionCode > highestVersionCode:
 562             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 563                           + ') than any APK: ')
 564             continue
 565         obbsha256 = sha256sum(f)
 566         obbs.append((packagename, versionCode, obbfile, obbsha256))
 567
 568     for apk in apks:
 569         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 570             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 571                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 572                     apk['obbMainFile'] = obbfile
 573                     apk['obbMainFileSha256'] = obbsha256
 574                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 575                     apk['obbPatchFile'] = obbfile
 576                     apk['obbPatchFileSha256'] = obbsha256
 577             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 578                 break
 579
 580
 581 def insert_graphics(repodir, apps):
 582     """Scans for screenshot PNG files in statically defined screenshots
 583     directory and adds them to the app metadata.  The screenshots and
 584     graphic must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 585     and must be in the following layout:
 586
 587     repo/packageName/locale/featureGraphic.png
 588     repo/packageName/locale/phoneScreenshots/1.png
 589     repo/packageName/locale/phoneScreenshots/2.png
 590
 591     Where "packageName" is the app's packageName and "locale" is the locale
 592     of the graphics, e.g. what language they are in, using the IETF RFC5646
 593     format (en-US, fr-CA, es-MX, etc).  This is following this pattern:
 594     https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#images-and-screenshots
 595
 596     This will also scan the metadata/ folder and the apps' source repos
 597     for standard locations of graphic and screenshot files.  If it finds
 598     them, it will copy them into the repo.
 599
 600     :param repodir: repo directory to scan
 601
 602     """
 603
 604     allowed_extensions = ('png', 'jpg', 'jpeg')
 605     graphicnames = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
 606     screenshotdirs = ('phoneScreenshots', 'sevenInchScreenshots',
 607                       'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
 608
 609     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z][A-Z-.@]*'))
 610     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*'))
 611
 612     for d in sorted(sourcedirs):
 613         if not os.path.isdir(d):
 614             continue
 615         for root, dirs, files in os.walk(d):
 616             segments = root.split('/')
 617             destdir = os.path.join('repo', segments[1], segments[-1])  # repo/packageName/locale
 618             for f in files:
 619                 base, extension = common.get_extension(f)
 620                 if base in graphicnames and extension in allowed_extensions:
 621                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 622                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 623                     shutil.copy(os.path.join(root, f), destdir)
 624             for d in dirs:
 625                 if d in screenshotdirs:
 626                     for f in glob.glob(os.path.join(root, d, '*.*')):
 627                         _, extension = common.get_extension(f)
 628                         if extension in allowed_extensions:
 629                             screenshotdestdir = os.path.join(destdir, d)
 630                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 631                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 632                             shutil.copy(f, screenshotdestdir)
 633
 634     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 635     for d in repofiles:
 636         if not os.path.isdir(d):
 637             continue
 638         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 639             if not os.path.isfile(f):
 640                 continue
 641             segments = f.split('/')
 642             packageName = segments[1]
 643             locale = segments[2]
 644             screenshotdir = segments[3]
 645             filename = os.path.basename(f)
 646             base, extension = common.get_extension(filename)
 647
 648             if packageName not in apps:
 649                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 650                                 % (filename, packageName))
 651                 continue
 652             if 'localized' not in apps[packageName]:
 653                 apps[packageName]['localized'] = collections.OrderedDict()
 654             if locale not in apps[packageName]['localized']:
 655                 apps[packageName]['localized'][locale] = collections.OrderedDict()
 656             graphics = apps[packageName]['localized'][locale]
 657
 658             if extension not in allowed_extensions:
 659                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 660             elif base in graphicnames:
 661                 # there can only be zero or one of these per locale
 662                 graphics[base] = filename
 663             elif screenshotdir in screenshotdirs:
 664                 # there can any number of these per locale
 665                 logging.debug('adding ' + base + ':' + f)
 666                 if screenshotdir not in graphics:
 667                     graphics[screenshotdir] = []
 668                 graphics[screenshotdir].append(filename)
 669             else:
 670                 logging.warning('Unsupported graphics file found: ' + f)
 671
 672
 673 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 674     """Scan a repo for all files with an extension except APK/OBB
 675
 676     :param apkcache: current cached info about all repo files
 677     :param repodir: repo directory to scan
 678     :param knownapks: list of all known files, as per metadata.read_metadata
 679     :param use_date_from_file: use date from file (instead of current date)
 680                                for newly added files
 681     """
 682
 683     cachechanged = False
 684     repo_files = []
 685     for name in os.listdir(repodir):
 686         file_extension = common.get_file_extension(name)
 687         if file_extension == 'apk' or file_extension == 'obb':
 688             continue
 689         filename = os.path.join(repodir, name)
 690         if filename.endswith('_src.tar.gz'):
 691             logging.debug('skipping source tarball: ' + filename)
 692             continue
 693         if not common.is_repo_file(filename):
 694             continue
 695         stat = os.stat(filename)
 696         if stat.st_size == 0:
 697             logging.error(filename + ' is zero size!')
 698             sys.exit(1)
 699
 700         shasum = sha256sum(filename)
 701         usecache = False
 702         if name in apkcache:
 703             repo_file = apkcache[name]
 704             # added time is cached as tuple but used here as datetime instance
 705             if 'added' in repo_file:
 706                 a = repo_file['added']
 707                 if isinstance(a, datetime):
 708                     repo_file['added'] = a
 709                 else:
 710                     repo_file['added'] = datetime(*a[:6])
 711             if repo_file['hash'] == shasum:
 712                 logging.debug("Reading " + name + " from cache")
 713                 usecache = True
 714             else:
 715                 logging.debug("Ignoring stale cache data for " + name)
 716
 717         if not usecache:
 718             logging.debug("Processing " + name)
 719             repo_file = {}
 720             # TODO rename apkname globally to something more generic
 721             repo_file['name'] = name
 722             repo_file['apkName'] = name
 723             repo_file['hash'] = shasum
 724             repo_file['hashType'] = 'sha256'
 725             repo_file['versionCode'] = 0
 726             repo_file['versionName'] = shasum
 727             # the static ID is the SHA256 unless it is set in the metadata
 728             repo_file['packageName'] = shasum
 729             n = name.split('_')
 730             if len(n) == 2:
 731                 packageName = n[0]
 732                 versionCode = n[1].split('.')[0]
 733                 if re.match(r'^-?[0-9]+$', versionCode) \
 734                    and common.is_valid_package_name(name.split('_')[0]):
 735                     repo_file['packageName'] = packageName
 736                     repo_file['versionCode'] = int(versionCode)
 737             srcfilename = name + "_src.tar.gz"
 738             if os.path.exists(os.path.join(repodir, srcfilename)):
 739                 repo_file['srcname'] = srcfilename
 740             repo_file['size'] = stat.st_size
 741
 742             apkcache[name] = repo_file
 743             cachechanged = True
 744
 745         if use_date_from_file:
 746             timestamp = stat.st_ctime
 747             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 748         else:
 749             default_date_param = None
 750
 751         # Record in knownapks, getting the added date at the same time..
 752         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 753                                     default_date=default_date_param)
 754         if added:
 755             repo_file['added'] = added
 756
 757         repo_files.append(repo_file)
 758
 759     return repo_files, cachechanged
 760
 761
 762 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
 763     """Scan the apk with the given filename in the given repo directory.
 764
 765     This also extracts the icons.
 766
 767     :param apkcache: current apk cache information
 768     :param apkfilename: the filename of the apk to scan
 769     :param repodir: repo directory to scan
 770     :param knownapks: known apks info
 771     :param use_date_from_apk: use date from APK (instead of current date)
 772                               for newly added APKs
 773     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
 774      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
 775     """
 776
 777     if ' ' in apkfilename:
 778         logging.critical("Spaces in filenames are not allowed.")
 779         sys.exit(1)
 780
 781     apkfile = os.path.join(repodir, apkfilename)
 782     shasum = sha256sum(apkfile)
 783
 784     cachechanged = False
 785     usecache = False
 786     if apkfilename in apkcache:
 787         apk = apkcache[apkfilename]
 788         if apk['hash'] == shasum:
 789             logging.debug("Reading " + apkfilename + " from cache")
 790             usecache = True
 791         else:
 792             logging.debug("Ignoring stale cache data for " + apkfilename)
 793
 794     if not usecache:
 795         logging.debug("Processing " + apkfilename)
 796         apk = {}
 797         apk['apkName'] = apkfilename
 798         apk['hash'] = shasum
 799         apk['hashType'] = 'sha256'
 800         srcfilename = apkfilename[:-4] + "_src.tar.gz"
 801         if os.path.exists(os.path.join(repodir, srcfilename)):
 802             apk['srcname'] = srcfilename
 803         apk['size'] = os.path.getsize(apkfile)
 804         apk['uses-permission'] = set()
 805         apk['uses-permission-sdk-23'] = set()
 806         apk['features'] = set()
 807         apk['icons_src'] = {}
 808         apk['icons'] = {}
 809         apk['antiFeatures'] = set()
 810         if has_old_openssl(apkfile):
 811             apk['antiFeatures'].add('KnownVuln')
 812         p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 813         if p.returncode != 0:
 814             if options.delete_unknown:
 815                 if os.path.exists(apkfile):
 816                     logging.error("Failed to get apk information, deleting " + apkfile)
 817                     os.remove(apkfile)
 818                 else:
 819                     logging.error("Could not find {0} to remove it".format(apkfile))
 820             else:
 821                 logging.error("Failed to get apk information, skipping " + apkfile)
 822             return True
 823         for line in p.output.splitlines():
 824             if line.startswith("package:"):
 825                 try:
 826                     apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 827                     apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 828                     apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 829                 except Exception as e:
 830                     logging.error("Package matching failed: " + str(e))
 831                     logging.info("Line was: " + line)
 832                     sys.exit(1)
 833             elif line.startswith("application:"):
 834                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 835                 # Keep path to non-dpi icon in case we need it
 836                 match = re.match(APK_ICON_PAT_NODPI, line)
 837                 if match:
 838                     apk['icons_src']['-1'] = match.group(1)
 839             elif line.startswith("launchable-activity:"):
 840                 # Only use launchable-activity as fallback to application
 841                 if not apk['name']:
 842                     apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 843                 if '-1' not in apk['icons_src']:
 844                     match = re.match(APK_ICON_PAT_NODPI, line)
 845                     if match:
 846                         apk['icons_src']['-1'] = match.group(1)
 847             elif line.startswith("application-icon-"):
 848                 match = re.match(APK_ICON_PAT, line)
 849                 if match:
 850                     density = match.group(1)
 851                     path = match.group(2)
 852                     apk['icons_src'][density] = path
 853             elif line.startswith("sdkVersion:"):
 854                 m = re.match(APK_SDK_VERSION_PAT, line)
 855                 if m is None:
 856                     logging.error(line.replace('sdkVersion:', '')
 857                                   + ' is not a valid minSdkVersion!')
 858                 else:
 859                     apk['minSdkVersion'] = m.group(1)
 860                     # if target not set, default to min
 861                     if 'targetSdkVersion' not in apk:
 862                         apk['targetSdkVersion'] = m.group(1)
 863             elif line.startswith("targetSdkVersion:"):
 864                 m = re.match(APK_SDK_VERSION_PAT, line)
 865                 if m is None:
 866                     logging.error(line.replace('targetSdkVersion:', '')
 867                                   + ' is not a valid targetSdkVersion!')
 868                 else:
 869                     apk['targetSdkVersion'] = m.group(1)
 870             elif line.startswith("maxSdkVersion:"):
 871                 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 872             elif line.startswith("native-code:"):
 873                 apk['nativecode'] = []
 874                 for arch in line[13:].split(' '):
 875                     apk['nativecode'].append(arch[1:-1])
 876             elif line.startswith('uses-permission:'):
 877                 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 878                 if perm_match['maxSdkVersion']:
 879                     perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 880                 permission = UsesPermission(
 881                     perm_match['name'],
 882                     perm_match['maxSdkVersion']
 883                 )
 884
 885                 apk['uses-permission'].add(permission)
 886             elif line.startswith('uses-permission-sdk-23:'):
 887                 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 888                 if perm_match['maxSdkVersion']:
 889                     perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 890                 permission_sdk_23 = UsesPermissionSdk23(
 891                     perm_match['name'],
 892                     perm_match['maxSdkVersion']
 893                 )
 894
 895                 apk['uses-permission-sdk-23'].add(permission_sdk_23)
 896
 897             elif line.startswith('uses-feature:'):
 898                 feature = re.match(APK_FEATURE_PAT, line).group(1)
 899                 # Filter out this, it's only added with the latest SDK tools and
 900                 # causes problems for lots of apps.
 901                 if feature != "android.hardware.screen.portrait" \
 902                         and feature != "android.hardware.screen.landscape":
 903                     if feature.startswith("android.feature."):
 904                         feature = feature[16:]
 905                     apk['features'].add(feature)
 906
 907         if 'minSdkVersion' not in apk:
 908             logging.warn("No SDK version information found in {0}".format(apkfile))
 909             apk['minSdkVersion'] = 1
 910
 911         # Check for debuggable apks...
 912         if common.isApkAndDebuggable(apkfile, config):
 913             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 914
 915         # Get the signature (or md5 of, to be precise)...
 916         logging.debug('Getting signature of {0}'.format(apkfile))
 917         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 918         if not apk['sig']:
 919             logging.critical("Failed to get apk signature")
 920             sys.exit(1)
 921
 922         apkzip = zipfile.ZipFile(apkfile, 'r')
 923
 924         # if an APK has files newer than the system time, suggest updating
 925         # the system clock.  This is useful for offline systems, used for
 926         # signing, which do not have another source of clock sync info. It
 927         # has to be more than 24 hours newer because ZIP/APK files do not
 928         # store timezone info
 929         manifest = apkzip.getinfo('AndroidManifest.xml')
 930         if manifest.date_time[1] == 0:  # month can't be zero
 931             logging.debug('AndroidManifest.xml has no date')
 932         else:
 933             dt_obj = datetime(*manifest.date_time)
 934             checkdt = dt_obj - timedelta(1)
 935             if datetime.today() < checkdt:
 936                 logging.warn('System clock is older than manifest in: '
 937                              + apkfilename
 938                              + '\nSet clock to that time using:\n'
 939                              + 'sudo date -s "' + str(dt_obj) + '"')
 940
 941         iconfilename = "%s.%s.png" % (
 942             apk['packageName'],
 943             apk['versionCode'])
 944
 945         # Extract the icon file...
 946         empty_densities = []
 947         for density in screen_densities:
 948             if density not in apk['icons_src']:
 949                 empty_densities.append(density)
 950                 continue
 951             iconsrc = apk['icons_src'][density]
 952             icon_dir = get_icon_dir(repodir, density)
 953             icondest = os.path.join(icon_dir, iconfilename)
 954
 955             try:
 956                 with open(icondest, 'wb') as f:
 957                     f.write(get_icon_bytes(apkzip, iconsrc))
 958                 apk['icons'][density] = iconfilename
 959
 960             except Exception as e:
 961                 logging.warn("Error retrieving icon file: %s" % (e))
 962                 del apk['icons'][density]
 963                 del apk['icons_src'][density]
 964                 empty_densities.append(density)
 965
 966         if '-1' in apk['icons_src']:
 967             iconsrc = apk['icons_src']['-1']
 968             iconpath = os.path.join(
 969                 get_icon_dir(repodir, '0'), iconfilename)
 970             with open(iconpath, 'wb') as f:
 971                 f.write(get_icon_bytes(apkzip, iconsrc))
 972             try:
 973                 im = Image.open(iconpath)
 974                 dpi = px_to_dpi(im.size[0])
 975                 for density in screen_densities:
 976                     if density in apk['icons']:
 977                         break
 978                     if density == screen_densities[-1] or dpi >= int(density):
 979                         apk['icons'][density] = iconfilename
 980                         shutil.move(iconpath,
 981                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 982                         empty_densities.remove(density)
 983                         break
 984             except Exception as e:
 985                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 986
 987         if apk['icons']:
 988             apk['icon'] = iconfilename
 989
 990         apkzip.close()
 991
 992         # First try resizing down to not lose quality
 993         last_density = None
 994         for density in screen_densities:
 995             if density not in empty_densities:
 996                 last_density = density
 997                 continue
 998             if last_density is None:
 999                 continue
1000             logging.debug("Density %s not available, resizing down from %s"
1001                           % (density, last_density))
1002
1003             last_iconpath = os.path.join(
1004                 get_icon_dir(repodir, last_density), iconfilename)
1005             iconpath = os.path.join(
1006                 get_icon_dir(repodir, density), iconfilename)
1007             fp = None
1008             try:
1009                 fp = open(last_iconpath, 'rb')
1010                 im = Image.open(fp)
1011
1012                 size = dpi_to_px(density)
1013
1014                 im.thumbnail((size, size), Image.ANTIALIAS)
1015                 im.save(iconpath, "PNG")
1016                 empty_densities.remove(density)
1017             except Exception as e:
1018                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
1019             finally:
1020                 if fp:
1021                     fp.close()
1022
1023         # Then just copy from the highest resolution available
1024         last_density = None
1025         for density in reversed(screen_densities):
1026             if density not in empty_densities:
1027                 last_density = density
1028                 continue
1029             if last_density is None:
1030                 continue
1031             logging.debug("Density %s not available, copying from lower density %s"
1032                           % (density, last_density))
1033
1034             shutil.copyfile(
1035                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1036                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1037
1038             empty_densities.remove(density)
1039
1040         for density in screen_densities:
1041             icon_dir = get_icon_dir(repodir, density)
1042             icondest = os.path.join(icon_dir, iconfilename)
1043             resize_icon(icondest, density)
1044
1045         # Copy from icons-mdpi to icons since mdpi is the baseline density
1046         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1047         if os.path.isfile(baseline):
1048             apk['icons']['0'] = iconfilename
1049             shutil.copyfile(baseline,
1050                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1051
1052         if use_date_from_apk and manifest.date_time[1] != 0:
1053             default_date_param = datetime(*manifest.date_time)
1054         else:
1055             default_date_param = None
1056
1057         # Record in known apks, getting the added date at the same time..
1058         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1059                                     default_date=default_date_param)
1060         if added:
1061             apk['added'] = added
1062
1063         apkcache[apkfilename] = apk
1064         cachechanged = True
1065
1066     return False, apk, cachechanged
1067
1068
1069 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1070     """Scan the apks in the given repo directory.
1071
1072     This also extracts the icons.
1073
1074     :param apkcache: current apk cache information
1075     :param repodir: repo directory to scan
1076     :param knownapks: known apks info
1077     :param use_date_from_apk: use date from APK (instead of current date)
1078                               for newly added APKs
1079     :returns: (apks, cachechanged) where apks is a list of apk information,
1080               and cachechanged is True if the apkcache got changed.
1081     """
1082
1083     cachechanged = False
1084
1085     for icon_dir in get_all_icon_dirs(repodir):
1086         if os.path.exists(icon_dir):
1087             if options.clean:
1088                 shutil.rmtree(icon_dir)
1089                 os.makedirs(icon_dir)
1090         else:
1091             os.makedirs(icon_dir)
1092
1093     apks = []
1094     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
1095         apkfilename = apkfile[len(repodir) + 1:]
1096         (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1097         if skip:
1098             continue
1099         apks.append(apk)
1100
1101     return apks, cachechanged
1102
1103
1104 def apply_info_from_latest_apk(apps, apks):
1105     """
1106     Some information from the apks needs to be applied up to the application level.
1107     When doing this, we use the info from the most recent version's apk.
1108     We deal with figuring out when the app was added and last updated at the same time.
1109     """
1110     for appid, app in apps.items():
1111         bestver = UNSET_VERSION_CODE
1112         for apk in apks:
1113             if apk['packageName'] == appid:
1114                 if apk['versionCode'] > bestver:
1115                     bestver = apk['versionCode']
1116                     bestapk = apk
1117
1118                 if 'added' in apk:
1119                     if not app.added or apk['added'] < app.added:
1120                         app.added = apk['added']
1121                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1122                         app.lastUpdated = apk['added']
1123
1124         if not app.added:
1125             logging.debug("Don't know when " + appid + " was added")
1126         if not app.lastUpdated:
1127             logging.debug("Don't know when " + appid + " was last updated")
1128
1129         if bestver == UNSET_VERSION_CODE:
1130
1131             if app.Name is None:
1132                 app.Name = app.AutoName or appid
1133             app.icon = None
1134             logging.debug("Application " + appid + " has no packages")
1135         else:
1136             if app.Name is None:
1137                 app.Name = bestapk['name']
1138             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1139             if app.CurrentVersionCode is None:
1140                 app.CurrentVersionCode = str(bestver)
1141
1142
1143 def make_categories_txt(repodir, categories):
1144     '''Write a category list in the repo to allow quick access'''
1145     catdata = ''
1146     for cat in sorted(categories):
1147         catdata += cat + '\n'
1148     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1149         f.write(catdata)
1150
1151
1152 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1153
1154     for appid, app in apps.items():
1155
1156         if app.ArchivePolicy:
1157             keepversions = int(app.ArchivePolicy[:-9])
1158         else:
1159             keepversions = defaultkeepversions
1160
1161         def filter_apk_list_sorted(apk_list):
1162             res = []
1163             for apk in apk_list:
1164                 if apk['packageName'] == appid:
1165                     res.append(apk)
1166
1167             # Sort the apk list by version code. First is highest/newest.
1168             return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1169
1170         def move_file(from_dir, to_dir, filename, ignore_missing):
1171             from_path = os.path.join(from_dir, filename)
1172             if ignore_missing and not os.path.exists(from_path):
1173                 return
1174             to_path = os.path.join(to_dir, filename)
1175             shutil.move(from_path, to_path)
1176
1177         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1178                       .format(appid, len(apks), keepversions, len(archapks)))
1179
1180         if len(apks) > keepversions:
1181             apklist = filter_apk_list_sorted(apks)
1182             # Move back the ones we don't want.
1183             for apk in apklist[keepversions:]:
1184                 logging.info("Moving " + apk['apkName'] + " to archive")
1185                 move_file(repodir, archivedir, apk['apkName'], False)
1186                 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1187                 for density in all_screen_densities:
1188                     repo_icon_dir = get_icon_dir(repodir, density)
1189                     archive_icon_dir = get_icon_dir(archivedir, density)
1190                     if density not in apk['icons']:
1191                         continue
1192                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1193                 if 'srcname' in apk:
1194                     move_file(repodir, archivedir, apk['srcname'], False)
1195                 archapks.append(apk)
1196                 apks.remove(apk)
1197         elif len(apks) < keepversions and len(archapks) > 0:
1198             required = keepversions - len(apks)
1199             archapklist = filter_apk_list_sorted(archapks)
1200             # Move forward the ones we want again.
1201             for apk in archapklist[:required]:
1202                 logging.info("Moving " + apk['apkName'] + " from archive")
1203                 move_file(archivedir, repodir, apk['apkName'], False)
1204                 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1205                 for density in all_screen_densities:
1206                     repo_icon_dir = get_icon_dir(repodir, density)
1207                     archive_icon_dir = get_icon_dir(archivedir, density)
1208                     if density not in apk['icons']:
1209                         continue
1210                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1211                 if 'srcname' in apk:
1212                     move_file(archivedir, repodir, apk['srcname'], False)
1213                 archapks.remove(apk)
1214                 apks.append(apk)
1215
1216
1217 def add_apks_to_per_app_repos(repodir, apks):
1218     apks_per_app = dict()
1219     for apk in apks:
1220         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1221         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1222         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1223         apks_per_app[apk['packageName']] = apk
1224
1225         if not os.path.exists(apk['per_app_icons']):
1226             logging.info('Adding new repo for only ' + apk['packageName'])
1227             os.makedirs(apk['per_app_icons'])
1228
1229         apkpath = os.path.join(repodir, apk['apkName'])
1230         shutil.copy(apkpath, apk['per_app_repo'])
1231         apksigpath = apkpath + '.sig'
1232         if os.path.exists(apksigpath):
1233             shutil.copy(apksigpath, apk['per_app_repo'])
1234         apkascpath = apkpath + '.asc'
1235         if os.path.exists(apkascpath):
1236             shutil.copy(apkascpath, apk['per_app_repo'])
1237
1238
1239 def make_binary_transparency_log(repodirs):
1240     '''Log the indexes in a standalone git repo to serve as a "binary
1241     transparency" log.
1242
1243     see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
1244
1245     '''
1246
1247     import git
1248     btrepo = 'binary_transparency'
1249     if os.path.exists(os.path.join(btrepo, '.git')):
1250         gitrepo = git.Repo(btrepo)
1251     else:
1252         if not os.path.exists(btrepo):
1253             os.mkdir(btrepo)
1254         gitrepo = git.Repo.init(btrepo)
1255
1256         gitconfig = gitrepo.config_writer()
1257         gitconfig.set_value('user', 'name', 'fdroid update')
1258         gitconfig.set_value('user', 'email', 'fdroid@' + platform.node())
1259
1260         url = config['repo_url'].rstrip('/')
1261         with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
1262             fp.write("""
1263 # Binary Transparency Log for %s
1264
1265 """ % url[:url.rindex('/')])  # strip '/repo'
1266         gitrepo.index.add(['README.md', ])
1267         gitrepo.index.commit('add README')
1268
1269     for repodir in repodirs:
1270         cpdir = os.path.join(btrepo, repodir)
1271         if not os.path.exists(cpdir):
1272             os.mkdir(cpdir)
1273         for f in ('index.xml', 'index-v1.json'):
1274             dest = os.path.join(cpdir, f)
1275             shutil.copyfile(os.path.join(repodir, f), dest)
1276             gitrepo.index.add([os.path.join(repodir, f), ])
1277         for f in ('index.jar', 'index-v1.jar'):
1278             repof = os.path.join(repodir, f)
1279             dest = os.path.join(cpdir, f)
1280             jarin = zipfile.ZipFile(repof, 'r')
1281             jarout = zipfile.ZipFile(dest, 'w')
1282             for info in jarin.infolist():
1283                 if info.filename.startswith('META-INF/'):
1284                     jarout.writestr(info, jarin.read(info.filename))
1285             jarout.close()
1286             jarin.close()
1287             gitrepo.index.add([repof, ])
1288
1289         files = []
1290         for root, dirs, filenames in os.walk(repodir):
1291             for f in filenames:
1292                 files.append(os.path.relpath(os.path.join(root, f), repodir))
1293         output = collections.OrderedDict()
1294         for f in sorted(files):
1295             repofile = os.path.join(repodir, f)
1296             stat = os.stat(repofile)
1297             output[f] = (
1298                 stat.st_size,
1299                 stat.st_ctime_ns,
1300                 stat.st_mtime_ns,
1301                 stat.st_mode,
1302                 stat.st_uid,
1303                 stat.st_gid,
1304             )
1305         fslogfile = os.path.join(cpdir, 'filesystemlog.json')
1306         with open(fslogfile, 'w') as fp:
1307             json.dump(output, fp, indent=2)
1308         gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
1309
1310     gitrepo.index.commit('fdroid update')
1311
1312
1313 config = None
1314 options = None
1315
1316
1317 def main():
1318
1319     global config, options
1320
1321     # Parse command line...
1322     parser = ArgumentParser()
1323     common.setup_global_opts(parser)
1324     parser.add_argument("--create-key", action="store_true", default=False,
1325                         help="Create a repo signing key in a keystore")
1326     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1327                         help="Create skeleton metadata files that are missing")
1328     parser.add_argument("--delete-unknown", action="store_true", default=False,
1329                         help="Delete APKs and/or OBBs without metadata from the repo")
1330     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1331                         help="Report on build data status")
1332     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1333                         help="Interactively ask about things that need updating.")
1334     parser.add_argument("-I", "--icons", action="store_true", default=False,
1335                         help="Resize all the icons exceeding the max pixel size and exit")
1336     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1337                         help="Specify editor to use in interactive mode. Default " +
1338                         "is /etc/alternatives/editor")
1339     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1340                         help="Update the wiki")
1341     parser.add_argument("--pretty", action="store_true", default=False,
1342                         help="Produce human-readable index.xml")
1343     parser.add_argument("--clean", action="store_true", default=False,
1344                         help="Clean update - don't uses caches, reprocess all apks")
1345     parser.add_argument("--nosign", action="store_true", default=False,
1346                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1347     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1348                         help="Use date from apk instead of current time for newly added apks")
1349     metadata.add_metadata_arguments(parser)
1350     options = parser.parse_args()
1351     metadata.warnings_action = options.W
1352
1353     config = common.read_config(options)
1354
1355     if not ('jarsigner' in config and 'keytool' in config):
1356         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1357         sys.exit(1)
1358
1359     repodirs = ['repo']
1360     if config['archive_older'] != 0:
1361         repodirs.append('archive')
1362         if not os.path.exists('archive'):
1363             os.mkdir('archive')
1364
1365     if options.icons:
1366         resize_all_icons(repodirs)
1367         sys.exit(0)
1368
1369     # check that icons exist now, rather than fail at the end of `fdroid update`
1370     for k in ['repo_icon', 'archive_icon']:
1371         if k in config:
1372             if not os.path.exists(config[k]):
1373                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1374                 sys.exit(1)
1375
1376     # if the user asks to create a keystore, do it now, reusing whatever it can
1377     if options.create_key:
1378         if os.path.exists(config['keystore']):
1379             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1380             logging.critical("\t'" + config['keystore'] + "'")
1381             sys.exit(1)
1382
1383         if 'repo_keyalias' not in config:
1384             config['repo_keyalias'] = socket.getfqdn()
1385             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1386         if 'keydname' not in config:
1387             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1388             common.write_to_config(config, 'keydname', config['keydname'])
1389         if 'keystore' not in config:
1390             config['keystore'] = common.default_config.keystore
1391             common.write_to_config(config, 'keystore', config['keystore'])
1392
1393         password = common.genpassword()
1394         if 'keystorepass' not in config:
1395             config['keystorepass'] = password
1396             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1397         if 'keypass' not in config:
1398             config['keypass'] = password
1399             common.write_to_config(config, 'keypass', config['keypass'])
1400         common.genkeystore(config)
1401
1402     # Get all apps...
1403     apps = metadata.read_metadata()
1404
1405     # Generate a list of categories...
1406     categories = set()
1407     for app in apps.values():
1408         categories.update(app.Categories)
1409
1410     # Read known apks data (will be updated and written back when we've finished)
1411     knownapks = common.KnownApks()
1412
1413     # Get APK cache
1414     apkcache = get_cache()
1415
1416     # Delete builds for disabled apps
1417     delete_disabled_builds(apps, apkcache, repodirs)
1418
1419     # Scan all apks in the main repo
1420     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1421
1422     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1423                                            options.use_date_from_apk)
1424     cachechanged = cachechanged or fcachechanged
1425     apks += files
1426     # Generate warnings for apk's with no metadata (or create skeleton
1427     # metadata files, if requested on the command line)
1428     newmetadata = False
1429     for apk in apks:
1430         if apk['packageName'] not in apps:
1431             if options.create_metadata:
1432                 if 'name' not in apk:
1433                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1434                     continue
1435                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1436                 f.write("License:Unknown\n")
1437                 f.write("Web Site:\n")
1438                 f.write("Source Code:\n")
1439                 f.write("Issue Tracker:\n")
1440                 f.write("Changelog:\n")
1441                 f.write("Summary:" + apk['name'] + "\n")
1442                 f.write("Description:\n")
1443                 f.write(apk['name'] + "\n")
1444                 f.write(".\n")
1445                 f.write("Name:" + apk['name'] + "\n")
1446                 f.close()
1447                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1448                 newmetadata = True
1449             else:
1450                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1451                 if options.delete_unknown:
1452                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1453                     rmf = os.path.join(repodirs[0], apk['apkName'])
1454                     if not os.path.exists(rmf):
1455                         logging.error("Could not find {0} to remove it".format(rmf))
1456                     else:
1457                         os.remove(rmf)
1458                 else:
1459                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1460
1461     # update the metadata with the newly created ones included
1462     if newmetadata:
1463         apps = metadata.read_metadata()
1464
1465     insert_obbs(repodirs[0], apps, apks)
1466     insert_graphics(repodirs[0], apps)
1467
1468     # Scan the archive repo for apks as well
1469     if len(repodirs) > 1:
1470         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1471         if cc:
1472             cachechanged = True
1473     else:
1474         archapks = []
1475
1476     # Apply information from latest apks to the application and update dates
1477     apply_info_from_latest_apk(apps, apks + archapks)
1478
1479     # Sort the app list by name, then the web site doesn't have to by default.
1480     # (we had to wait until we'd scanned the apks to do this, because mostly the
1481     # name comes from there!)
1482     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1483
1484     # APKs are placed into multiple repos based on the app package, providing
1485     # per-app subscription feeds for nightly builds and things like it
1486     if config['per_app_repos']:
1487         add_apks_to_per_app_repos(repodirs[0], apks)
1488         for appid, app in apps.items():
1489             repodir = os.path.join(appid, 'fdroid', 'repo')
1490             appdict = dict()
1491             appdict[appid] = app
1492             if os.path.isdir(repodir):
1493                 index.make(appdict, [appid], apks, repodir, False)
1494             else:
1495                 logging.info('Skipping index generation for ' + appid)
1496         return
1497
1498     if len(repodirs) > 1:
1499         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1500
1501     # Make the index for the main repo...
1502     index.make(apps, sortedids, apks, repodirs[0], False)
1503     make_categories_txt(repodirs[0], categories)
1504
1505     # If there's an archive repo,  make the index for it. We already scanned it
1506     # earlier on.
1507     if len(repodirs) > 1:
1508         index.make(apps, sortedids, archapks, repodirs[1], True)
1509
1510     if config.get('binary_transparency_remote'):
1511         make_binary_transparency_log(repodirs)
1512
1513     if config['update_stats']:
1514         # Update known apks info...
1515         knownapks.writeifchanged()
1516
1517         # Generate latest apps data for widget
1518         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1519             data = ''
1520             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1521                 for line in f:
1522                     appid = line.rstrip()
1523                     data += appid + "\t"
1524                     app = apps[appid]
1525                     data += app.Name + "\t"
1526                     if app.icon is not None:
1527                         data += app.icon + "\t"
1528                     data += app.License + "\n"
1529             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1530                 f.write(data)
1531
1532     if cachechanged:
1533         write_cache(apkcache)
1534
1535     # Update the wiki...
1536     if options.wiki:
1537         update_wiki(apps, sortedids, apks + archapks)
1538
1539     logging.info("Finished.")
1540
1541
1542 if __name__ == "__main__":
1543     main()