fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in sorted(app.AntiFeatures):
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.LiberapayID,
 153             app.Bitcoin,
 154             app.Litecoin,
 155             app.License,
 156             requiresroot,
 157             app.AuthorName,
 158             app.AuthorEmail)
 159
 160         if app.Provides:
 161             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 162
 163         wikidata += app.Summary
 164         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 165
 166         wikidata += "=Description=\n"
 167         wikidata += metadata.description_wiki(app.Description) + "\n"
 168
 169         wikidata += "=Maintainer Notes=\n"
 170         if app.MaintainerNotes:
 171             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 172         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 173
 174         # Get a list of all packages for this application...
 175         apklist = []
 176         gotcurrentver = False
 177         cantupdate = False
 178         buildfails = False
 179         for apk in apks:
 180             if apk['packageName'] == appid:
 181                 if str(apk['versionCode']) == app.CurrentVersionCode:
 182                     gotcurrentver = True
 183                 apklist.append(apk)
 184         # Include ones we can't build, as a special case...
 185         for build in app.builds:
 186             if build.disable:
 187                 if build.versionCode == app.CurrentVersionCode:
 188                     cantupdate = True
 189                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 190                 apklist.append({'versionCode': int(build.versionCode),
 191                                 'versionName': build.versionName,
 192                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 193                                 })
 194             else:
 195                 builtit = False
 196                 for apk in apklist:
 197                     if apk['versionCode'] == int(build.versionCode):
 198                         builtit = True
 199                         break
 200                 if not builtit:
 201                     buildfails = True
 202                     apklist.append({'versionCode': int(build.versionCode),
 203                                     'versionName': build.versionName,
 204                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 205                                     })
 206         if app.CurrentVersionCode == '0':
 207             cantupdate = True
 208         # Sort with most recent first...
 209         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 210
 211         wikidata += "=Versions=\n"
 212         if len(apklist) == 0:
 213             wikidata += "We currently have no versions of this app available."
 214         elif not gotcurrentver:
 215             wikidata += "We don't have the current version of this app."
 216         else:
 217             wikidata += "We have the current version of this app."
 218         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 219         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 220         if len(app.NoSourceSince) > 0:
 221             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 222         if len(app.CurrentVersion) > 0:
 223             wikidata += "The current (recommended) version is " + app.CurrentVersion
 224             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 225         validapks = 0
 226         for apk in apklist:
 227             wikidata += "==" + apk['versionName'] + "==\n"
 228
 229             if 'buildproblem' in apk:
 230                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 231             else:
 232                 validapks += 1
 233                 wikidata += "This version is built and signed by "
 234                 if 'srcname' in apk:
 235                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 236                 else:
 237                     wikidata += "the original developer.\n\n"
 238             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 239
 240         wikidata += '\n[[Category:' + wikicat + ']]\n'
 241         if len(app.NoSourceSince) > 0:
 242             wikidata += '\n[[Category:Apps missing source code]]\n'
 243         if validapks == 0 and not app.Disabled:
 244             wikidata += '\n[[Category:Apps with no packages]]\n'
 245         if cantupdate and not app.Disabled:
 246             wikidata += "\n[[Category:Apps we cannot update]]\n"
 247         if buildfails and not app.Disabled:
 248             wikidata += "\n[[Category:Apps with failing builds]]\n"
 249         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 250             wikidata += '\n[[Category:Apps to Update]]\n'
 251         if app.Disabled:
 252             wikidata += '\n[[Category:Apps that are disabled]]\n'
 253         if app.UpdateCheckMode == 'None' and not app.Disabled:
 254             wikidata += '\n[[Category:Apps with no update check]]\n'
 255         for appcat in app.Categories:
 256             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 257
 258         # We can't have underscores in the page name, even if they're in
 259         # the package ID, because MediaWiki messes with them...
 260         pagename = appid.replace('_', ' ')
 261
 262         # Drop a trailing newline, because mediawiki is going to drop it anyway
 263         # and it we don't we'll think the page has changed when it hasn't...
 264         if wikidata.endswith('\n'):
 265             wikidata = wikidata[:-1]
 266
 267         generated_pages[pagename] = wikidata
 268
 269         # Make a redirect from the name to the ID too, unless there's
 270         # already an existing page with the name and it isn't a redirect.
 271         noclobber = False
 272         apppagename = app.Name.replace('_', ' ')
 273         apppagename = apppagename.replace('{', '')
 274         apppagename = apppagename.replace('}', ' ')
 275         apppagename = apppagename.replace(':', ' ')
 276         apppagename = apppagename.replace('[', ' ')
 277         apppagename = apppagename.replace(']', ' ')
 278         # Drop double spaces caused mostly by replacing ':' above
 279         apppagename = apppagename.replace('  ', ' ')
 280         for expagename in site.allpages(prefix=apppagename,
 281                                         filterredir='nonredirects',
 282                                         generator=False):
 283             if expagename == apppagename:
 284                 noclobber = True
 285         # Another reason not to make the redirect page is if the app name
 286         # is the same as it's ID, because that will overwrite the real page
 287         # with an redirect to itself! (Although it seems like an odd
 288         # scenario this happens a lot, e.g. where there is metadata but no
 289         # builds or binaries to extract a name from.
 290         if apppagename == pagename:
 291             noclobber = True
 292         if not noclobber:
 293             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 294
 295     for tcat, genp in [(wikicat, generated_pages),
 296                        (wikiredircat, generated_redirects)]:
 297         catpages = site.Pages['Category:' + tcat]
 298         existingpages = []
 299         for page in catpages:
 300             existingpages.append(page.name)
 301             if page.name in genp:
 302                 pagetxt = page.edit()
 303                 if pagetxt != genp[page.name]:
 304                     logging.debug("Updating modified page " + page.name)
 305                     page.save(genp[page.name], summary='Auto-updated')
 306                 else:
 307                     logging.debug("Page " + page.name + " is unchanged")
 308             else:
 309                 logging.warn("Deleting page " + page.name)
 310                 page.delete('No longer published')
 311         for pagename, text in genp.items():
 312             logging.debug("Checking " + pagename)
 313             if pagename not in existingpages:
 314                 logging.debug("Creating page " + pagename)
 315                 try:
 316                     newpage = site.Pages[pagename]
 317                     newpage.save(text, summary='Auto-created')
 318                 except Exception as e:
 319                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 320
 321     # Purge server cache to ensure counts are up to date
 322     site.pages['Repository Maintenance'].purge()
 323
 324
 325 def delete_disabled_builds(apps, apkcache, repodirs):
 326     """Delete disabled build outputs.
 327
 328     :param apps: list of all applications, as per metadata.read_metadata
 329     :param apkcache: current apk cache information
 330     :param repodirs: the repo directories to process
 331     """
 332     for appid, app in apps.items():
 333         for build in app['builds']:
 334             if not build.disable:
 335                 continue
 336             apkfilename = common.get_release_filename(app, build)
 337             iconfilename = "%s.%s.png" % (
 338                 appid,
 339                 build.versionCode)
 340             for repodir in repodirs:
 341                 files = [
 342                     os.path.join(repodir, apkfilename),
 343                     os.path.join(repodir, apkfilename + '.asc'),
 344                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 345                 ]
 346                 for density in all_screen_densities:
 347                     repo_dir = get_icon_dir(repodir, density)
 348                     files.append(os.path.join(repo_dir, iconfilename))
 349
 350                 for f in files:
 351                     if os.path.exists(f):
 352                         logging.info("Deleting disabled build output " + f)
 353                         os.remove(f)
 354             if apkfilename in apkcache:
 355                 del apkcache[apkfilename]
 356
 357
 358 def resize_icon(iconpath, density):
 359
 360     if not os.path.isfile(iconpath):
 361         return
 362
 363     fp = None
 364     try:
 365         fp = open(iconpath, 'rb')
 366         im = Image.open(fp)
 367         size = dpi_to_px(density)
 368
 369         if any(length > size for length in im.size):
 370             oldsize = im.size
 371             im.thumbnail((size, size), Image.ANTIALIAS)
 372             logging.debug("%s was too large at %s - new size is %s" % (
 373                 iconpath, oldsize, im.size))
 374             im.save(iconpath, "PNG")
 375
 376     except Exception as e:
 377         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 378
 379     finally:
 380         if fp:
 381             fp.close()
 382
 383
 384 def resize_all_icons(repodirs):
 385     """Resize all icons that exceed the max size
 386
 387     :param repodirs: the repo directories to process
 388     """
 389     for repodir in repodirs:
 390         for density in screen_densities:
 391             icon_dir = get_icon_dir(repodir, density)
 392             icon_glob = os.path.join(icon_dir, '*.png')
 393             for iconpath in glob.glob(icon_glob):
 394                 resize_icon(iconpath, density)
 395
 396
 397 def getsig(apkpath):
 398     """ Get the signing certificate of an apk. To get the same md5 has that
 399     Android gets, we encode the .RSA certificate in a specific format and pass
 400     it hex-encoded to the md5 digest algorithm.
 401
 402     :param apkpath: path to the apk
 403     :returns: A string containing the md5 of the signature of the apk or None
 404               if an error occurred.
 405     """
 406
 407     with zipfile.ZipFile(apkpath, 'r') as apk:
 408         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 409
 410         if len(certs) < 1:
 411             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 412             return None
 413         if len(certs) > 1:
 414             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 415             return None
 416
 417         cert = apk.read(certs[0])
 418
 419     cert_encoded = common.get_certificate(cert)
 420
 421     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 422
 423
 424 def get_cache_file():
 425     return os.path.join('tmp', 'apkcache')
 426
 427
 428 def get_cache():
 429     """Get the cached dict of the APK index
 430
 431     Gather information about all the apk files in the repo directory,
 432     using cached data if possible. Some of the index operations take a
 433     long time, like calculating the SHA-256 and verifying the APK
 434     signature.
 435
 436     The cache is invalidated if the metadata version is different, or
 437     the 'allow_disabled_algorithms' config/option is different.  In
 438     those cases, there is no easy way to know what has changed from
 439     the cache, so just rerun the whole thing.
 440
 441     :return: apkcache
 442
 443     """
 444     apkcachefile = get_cache_file()
 445     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 446     if not options.clean and os.path.exists(apkcachefile):
 447         with open(apkcachefile, 'rb') as cf:
 448             apkcache = pickle.load(cf, encoding='utf-8')
 449         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 450            or apkcache.get('allow_disabled_algorithms') != ada:
 451             apkcache = {}
 452     else:
 453         apkcache = {}
 454
 455     apkcache["METADATA_VERSION"] = METADATA_VERSION
 456     apkcache['allow_disabled_algorithms'] = ada
 457
 458     return apkcache
 459
 460
 461 def write_cache(apkcache):
 462     apkcachefile = get_cache_file()
 463     cache_path = os.path.dirname(apkcachefile)
 464     if not os.path.exists(cache_path):
 465         os.makedirs(cache_path)
 466     with open(apkcachefile, 'wb') as cf:
 467         pickle.dump(apkcache, cf)
 468
 469
 470 def get_icon_bytes(apkzip, iconsrc):
 471     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 472     try:
 473         return apkzip.read(iconsrc)
 474     except KeyError:
 475         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 476
 477
 478 def sha256sum(filename):
 479     '''Calculate the sha256 of the given file'''
 480     sha = hashlib.sha256()
 481     with open(filename, 'rb') as f:
 482         while True:
 483             t = f.read(16384)
 484             if len(t) == 0:
 485                 break
 486             sha.update(t)
 487     return sha.hexdigest()
 488
 489
 490 def has_known_vulnerability(filename):
 491     """checks for known vulnerabilities in the APK
 492
 493     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 494     version.  Google also enforces this:
 495     https://support.google.com/faqs/answer/6376725?hl=en
 496
 497     Checks whether there are more than one classes.dex or AndroidManifest.xml
 498     files, which is invalid and an essential part of the "Master Key" attack.
 499
 500     http://www.saurik.com/id/17
 501     """
 502
 503     found_vuln = False
 504
 505     # statically load this pattern
 506     if not hasattr(has_known_vulnerability, "pattern"):
 507         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 508
 509     files_in_apk = set()
 510     with zipfile.ZipFile(filename) as zf:
 511         for name in zf.namelist():
 512             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 513                 lib = zf.open(name)
 514                 while True:
 515                     chunk = lib.read(4096)
 516                     if chunk == b'':
 517                         break
 518                     m = has_known_vulnerability.pattern.search(chunk)
 519                     if m:
 520                         version = m.group(1).decode('ascii')
 521                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 522                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 523                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 524                             logging.debug(_('"{path}" contains recent {name} ({version})')
 525                                           .format(path=filename, name=name, version=version))
 526                         else:
 527                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 528                                             .format(path=filename, name=name, version=version))
 529                             found_vuln = True
 530                         break
 531             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 532                 if name in files_in_apk:
 533                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 534                                     .format(apkfilename=filename, name=name))
 535                     found_vuln = True
 536                 files_in_apk.add(name)
 537     return found_vuln
 538
 539
 540 def insert_obbs(repodir, apps, apks):
 541     """Scans the .obb files in a given repo directory and adds them to the
 542     relevant APK instances.  OBB files have versionCodes like APK
 543     files, and they are loosely associated.  If there is an OBB file
 544     present, then any APK with the same or higher versionCode will use
 545     that OBB file.  There are two OBB types: main and patch, each APK
 546     can only have only have one of each.
 547
 548     https://developer.android.com/google/play/expansion-files.html
 549
 550     :param repodir: repo directory to scan
 551     :param apps: list of current, valid apps
 552     :param apks: current information on all APKs
 553
 554     """
 555
 556     def obbWarnDelete(f, msg):
 557         logging.warning(msg + ' ' + f)
 558         if options.delete_unknown:
 559             logging.error(_("Deleting unknown file: {path}").format(path=f))
 560             os.remove(f)
 561
 562     obbs = []
 563     java_Integer_MIN_VALUE = -pow(2, 31)
 564     currentPackageNames = apps.keys()
 565     for f in glob.glob(os.path.join(repodir, '*.obb')):
 566         obbfile = os.path.basename(f)
 567         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 568         chunks = obbfile.split('.')
 569         if chunks[0] != 'main' and chunks[0] != 'patch':
 570             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 571             continue
 572         if not re.match(r'^-?[0-9]+$', chunks[1]):
 573             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 574                           .format(name=chunks[0]))
 575             continue
 576         versionCode = int(chunks[1])
 577         packagename = ".".join(chunks[2:-1])
 578
 579         highestVersionCode = java_Integer_MIN_VALUE
 580         if packagename not in currentPackageNames:
 581             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 582             continue
 583         for apk in apks:
 584             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 585                 highestVersionCode = apk['versionCode']
 586         if versionCode > highestVersionCode:
 587             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 588                           .format(integer=str(versionCode)))
 589             continue
 590         obbsha256 = sha256sum(f)
 591         obbs.append((packagename, versionCode, obbfile, obbsha256))
 592
 593     for apk in apks:
 594         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 595             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 596                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 597                     apk['obbMainFile'] = obbfile
 598                     apk['obbMainFileSha256'] = obbsha256
 599                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 600                     apk['obbPatchFile'] = obbfile
 601                     apk['obbPatchFileSha256'] = obbsha256
 602             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 603                 break
 604
 605
 606 def translate_per_build_anti_features(apps, apks):
 607     """Grab the anti-features list from the build metadata
 608
 609     For most Anti-Features, they are really most applicable per-APK,
 610     not for an app.  An app can fix a vulnerability, add/remove
 611     tracking, etc.  This reads the 'antifeatures' list from the Build
 612     entries in the fdroiddata metadata file, then transforms it into
 613     the 'antiFeatures' list of unique items for the index.
 614
 615     The field key is all lower case in the metadata file to match the
 616     rest of the Build fields.  It is 'antiFeatures' camel case in the
 617     implementation, index, and fdroidclient since it is translated
 618     from the build 'antifeatures' field, not directly included.
 619
 620     """
 621
 622     antiFeatures = dict()
 623     for packageName, app in apps.items():
 624         d = dict()
 625         for build in app['builds']:
 626             afl = build.get('antifeatures')
 627             if afl:
 628                 d[int(build.versionCode)] = afl
 629         if len(d) > 0:
 630             antiFeatures[packageName] = d
 631
 632     for apk in apks:
 633         d = antiFeatures.get(apk['packageName'])
 634         if d:
 635             afl = d.get(apk['versionCode'])
 636             if afl:
 637                 apk['antiFeatures'].update(afl)
 638
 639
 640 def _get_localized_dict(app, locale):
 641     '''get the dict to add localized store metadata to'''
 642     if 'localized' not in app:
 643         app['localized'] = collections.OrderedDict()
 644     if locale not in app['localized']:
 645         app['localized'][locale] = collections.OrderedDict()
 646     return app['localized'][locale]
 647
 648
 649 def _set_localized_text_entry(app, locale, key, f):
 650     limit = config['char_limits'][key]
 651     localized = _get_localized_dict(app, locale)
 652     with open(f) as fp:
 653         text = fp.read()[:limit]
 654         if len(text) > 0:
 655             localized[key] = text
 656
 657
 658 def _set_author_entry(app, key, f):
 659     limit = config['char_limits']['author']
 660     with open(f) as fp:
 661         text = fp.read()[:limit]
 662         if len(text) > 0:
 663             app[key] = text
 664
 665
 666 def copy_triple_t_store_metadata(apps):
 667     """Include store metadata from the app's source repo
 668
 669     The Triple-T Gradle Play Publisher is a plugin that has a standard
 670     file layout for all of the metadata and graphics that the Google
 671     Play Store accepts.  Since F-Droid has the git repo, it can just
 672     pluck those files directly.  This method reads any text files into
 673     the app dict, then copies any graphics into the fdroid repo
 674     directory structure.
 675
 676     This needs to be run before insert_localized_app_metadata() so that
 677     the graphics files that are copied into the fdroid repo get
 678     properly indexed.
 679
 680     https://github.com/Triple-T/gradle-play-publisher#upload-images
 681     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 682
 683     """
 684
 685     if not os.path.isdir('build'):
 686         return  # nothing to do
 687
 688     for packageName, app in apps.items():
 689         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 690             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 691             for root, dirs, files in os.walk(d):
 692                 segments = root.split('/')
 693                 locale = segments[-2]
 694                 for f in files:
 695                     if f == 'fulldescription':
 696                         _set_localized_text_entry(app, locale, 'description',
 697                                                   os.path.join(root, f))
 698                         continue
 699                     elif f == 'shortdescription':
 700                         _set_localized_text_entry(app, locale, 'summary',
 701                                                   os.path.join(root, f))
 702                         continue
 703                     elif f == 'title':
 704                         _set_localized_text_entry(app, locale, 'name',
 705                                                   os.path.join(root, f))
 706                         continue
 707                     elif f == 'video':
 708                         _set_localized_text_entry(app, locale, 'video',
 709                                                   os.path.join(root, f))
 710                         continue
 711                     elif f == 'whatsnew':
 712                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 713                                                   os.path.join(root, f))
 714                         continue
 715                     elif f == 'contactEmail':
 716                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 717                         continue
 718                     elif f == 'contactPhone':
 719                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 720                         continue
 721                     elif f == 'contactWebsite':
 722                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 723                         continue
 724
 725                     base, extension = common.get_extension(f)
 726                     dirname = os.path.basename(root)
 727                     if extension in ALLOWED_EXTENSIONS \
 728                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 729                         if segments[-2] == 'listing':
 730                             locale = segments[-3]
 731                         else:
 732                             locale = segments[-2]
 733                         destdir = os.path.join('repo', packageName, locale, dirname)
 734                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 735                         sourcefile = os.path.join(root, f)
 736                         destfile = os.path.join(destdir, os.path.basename(f))
 737                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 738                         shutil.copy(sourcefile, destfile)
 739
 740
 741 def insert_localized_app_metadata(apps):
 742     """scans standard locations for graphics and localized text
 743
 744     Scans for localized description files, store graphics, and
 745     screenshot PNG files in statically defined screenshots directory
 746     and adds them to the app metadata.  The screenshots and graphic
 747     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 748     and must be in the following layout:
 749     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 750
 751     repo/packageName/locale/featureGraphic.png
 752     repo/packageName/locale/phoneScreenshots/1.png
 753     repo/packageName/locale/phoneScreenshots/2.png
 754
 755     The changelog files must be text files named with the versionCode
 756     ending with ".txt" and must be in the following layout:
 757     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 758
 759     repo/packageName/locale/changelogs/12345.txt
 760
 761     This will scan the each app's source repo then the metadata/ dir
 762     for these standard locations of changelog files.  If it finds
 763     them, they will be added to the dict of all packages, with the
 764     versions in the metadata/ folder taking precendence over the what
 765     is in the app's source repo.
 766
 767     Where "packageName" is the app's packageName and "locale" is the locale
 768     of the graphics, e.g. what language they are in, using the IETF RFC5646
 769     format (en-US, fr-CA, es-MX, etc).
 770
 771     This will also scan the app's git for a fastlane folder, and the
 772     metadata/ folder and the apps' source repos for standard locations
 773     of graphic and screenshot files.  If it finds them, it will copy
 774     them into the repo.  The fastlane files follow this pattern:
 775     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 776
 777     """
 778
 779     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 780     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 781     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 782     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 783
 784     for srcd in sorted(sourcedirs):
 785         if not os.path.isdir(srcd):
 786             continue
 787         for root, dirs, files in os.walk(srcd):
 788             segments = root.split('/')
 789             packageName = segments[1]
 790             if packageName not in apps:
 791                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 792                 continue
 793             locale = segments[-1]
 794             destdir = os.path.join('repo', packageName, locale)
 795
 796             # flavours specified in build receipt
 797             build_flavours = ""
 798             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 799                     and 'gradle' in apps[packageName].builds[-1]:
 800                 build_flavours = apps[packageName].builds[-1].gradle
 801
 802             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 803                 logging.debug("ignoring due to wrong flavour")
 804                 continue
 805
 806             for f in files:
 807                 if f in ('description.txt', 'full_description.txt'):
 808                     _set_localized_text_entry(apps[packageName], locale, 'description',
 809                                               os.path.join(root, f))
 810                     continue
 811                 elif f in ('summary.txt', 'short_description.txt'):
 812                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 813                                               os.path.join(root, f))
 814                     continue
 815                 elif f in ('name.txt', 'title.txt'):
 816                     _set_localized_text_entry(apps[packageName], locale, 'name',
 817                                               os.path.join(root, f))
 818                     continue
 819                 elif f == 'video.txt':
 820                     _set_localized_text_entry(apps[packageName], locale, 'video',
 821                                               os.path.join(root, f))
 822                     continue
 823                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 824                     locale = segments[-2]
 825                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 826                                               os.path.join(root, f))
 827                     continue
 828
 829                 base, extension = common.get_extension(f)
 830                 if locale == 'images':
 831                     locale = segments[-2]
 832                     destdir = os.path.join('repo', packageName, locale)
 833                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 834                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 835                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 836                     shutil.copy(os.path.join(root, f), destdir)
 837             for d in dirs:
 838                 if d in SCREENSHOT_DIRS:
 839                     if locale == 'images':
 840                         locale = segments[-2]
 841                         destdir = os.path.join('repo', packageName, locale)
 842                     for f in glob.glob(os.path.join(root, d, '*.*')):
 843                         _ignored, extension = common.get_extension(f)
 844                         if extension in ALLOWED_EXTENSIONS:
 845                             screenshotdestdir = os.path.join(destdir, d)
 846                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 847                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 848                             shutil.copy(f, screenshotdestdir)
 849
 850     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 851     for d in repofiles:
 852         if not os.path.isdir(d):
 853             continue
 854         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 855             if not os.path.isfile(f):
 856                 continue
 857             segments = f.split('/')
 858             packageName = segments[1]
 859             locale = segments[2]
 860             screenshotdir = segments[3]
 861             filename = os.path.basename(f)
 862             base, extension = common.get_extension(filename)
 863
 864             if packageName not in apps:
 865                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 866                                 .format(path=filename, name=packageName))
 867                 continue
 868             graphics = _get_localized_dict(apps[packageName], locale)
 869
 870             if extension not in ALLOWED_EXTENSIONS:
 871                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 872             elif base in GRAPHIC_NAMES:
 873                 # there can only be zero or one of these per locale
 874                 graphics[base] = filename
 875             elif screenshotdir in SCREENSHOT_DIRS:
 876                 # there can any number of these per locale
 877                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 878                 if screenshotdir not in graphics:
 879                     graphics[screenshotdir] = []
 880                 graphics[screenshotdir].append(filename)
 881             else:
 882                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 883
 884
 885 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 886     """Scan a repo for all files with an extension except APK/OBB
 887
 888     :param apkcache: current cached info about all repo files
 889     :param repodir: repo directory to scan
 890     :param knownapks: list of all known files, as per metadata.read_metadata
 891     :param use_date_from_file: use date from file (instead of current date)
 892                                for newly added files
 893     """
 894
 895     cachechanged = False
 896     repo_files = []
 897     repodir = repodir.encode('utf-8')
 898     for name in os.listdir(repodir):
 899         file_extension = common.get_file_extension(name)
 900         if file_extension == 'apk' or file_extension == 'obb':
 901             continue
 902         filename = os.path.join(repodir, name)
 903         name_utf8 = name.decode('utf-8')
 904         if filename.endswith(b'_src.tar.gz'):
 905             logging.debug(_('skipping source tarball: {path}')
 906                           .format(path=filename.decode('utf-8')))
 907             continue
 908         if not common.is_repo_file(filename):
 909             continue
 910         stat = os.stat(filename)
 911         if stat.st_size == 0:
 912             raise FDroidException(_('{path} is zero size!')
 913                                   .format(path=filename))
 914
 915         shasum = sha256sum(filename)
 916         usecache = False
 917         if name in apkcache:
 918             repo_file = apkcache[name]
 919             # added time is cached as tuple but used here as datetime instance
 920             if 'added' in repo_file:
 921                 a = repo_file['added']
 922                 if isinstance(a, datetime):
 923                     repo_file['added'] = a
 924                 else:
 925                     repo_file['added'] = datetime(*a[:6])
 926             if repo_file.get('hash') == shasum:
 927                 logging.debug(_("Reading {apkfilename} from cache")
 928                               .format(apkfilename=name_utf8))
 929                 usecache = True
 930             else:
 931                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 932                               .format(apkfilename=name_utf8))
 933
 934         if not usecache:
 935             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 936             repo_file = collections.OrderedDict()
 937             repo_file['name'] = os.path.splitext(name_utf8)[0]
 938             # TODO rename apkname globally to something more generic
 939             repo_file['apkName'] = name_utf8
 940             repo_file['hash'] = shasum
 941             repo_file['hashType'] = 'sha256'
 942             repo_file['versionCode'] = 0
 943             repo_file['versionName'] = shasum
 944             # the static ID is the SHA256 unless it is set in the metadata
 945             repo_file['packageName'] = shasum
 946
 947             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 948             if m:
 949                 repo_file['packageName'] = m.group(1)
 950                 repo_file['versionCode'] = int(m.group(2))
 951             srcfilename = name + b'_src.tar.gz'
 952             if os.path.exists(os.path.join(repodir, srcfilename)):
 953                 repo_file['srcname'] = srcfilename.decode('utf-8')
 954             repo_file['size'] = stat.st_size
 955
 956             apkcache[name] = repo_file
 957             cachechanged = True
 958
 959         if use_date_from_file:
 960             timestamp = stat.st_ctime
 961             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 962         else:
 963             default_date_param = None
 964
 965         # Record in knownapks, getting the added date at the same time..
 966         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 967                                     default_date=default_date_param)
 968         if added:
 969             repo_file['added'] = added
 970
 971         repo_files.append(repo_file)
 972
 973     return repo_files, cachechanged
 974
 975
 976 def scan_apk(apk_file):
 977     """
 978     Scans an APK file and returns dictionary with metadata of the APK.
 979
 980     Attention: This does *not* verify that the APK signature is correct.
 981
 982     :param apk_file: The (ideally absolute) path to the APK file
 983     :raises BuildException
 984     :return A dict containing APK metadata
 985     """
 986     apk = {
 987         'hash': sha256sum(apk_file),
 988         'hashType': 'sha256',
 989         'uses-permission': [],
 990         'uses-permission-sdk-23': [],
 991         'features': [],
 992         'icons_src': {},
 993         'icons': {},
 994         'antiFeatures': set(),
 995     }
 996
 997     if SdkToolsPopen(['aapt', 'version'], output=False):
 998         scan_apk_aapt(apk, apk_file)
 999     else:
1000         scan_apk_androguard(apk, apk_file)
1001
1002     # Get the signature, or rather the signing key fingerprints
1003     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1004     apk['sig'] = getsig(apk_file)
1005     if not apk['sig']:
1006         raise BuildException("Failed to get apk signature")
1007     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1008                                                                apk_file))
1009     if not apk.get('signer'):
1010         raise BuildException("Failed to get apk signing key fingerprint")
1011
1012     # Get size of the APK
1013     apk['size'] = os.path.getsize(apk_file)
1014
1015     if 'minSdkVersion' not in apk:
1016         logging.warning("No SDK version information found in {0}".format(apk_file))
1017         apk['minSdkVersion'] = 1
1018
1019     # Check for known vulnerabilities
1020     if has_known_vulnerability(apk_file):
1021         apk['antiFeatures'].add('KnownVuln')
1022
1023     return apk
1024
1025
1026 def scan_apk_aapt(apk, apkfile):
1027     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1028     if p.returncode != 0:
1029         if options.delete_unknown:
1030             if os.path.exists(apkfile):
1031                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1032                 os.remove(apkfile)
1033             else:
1034                 logging.error("Could not find {0} to remove it".format(apkfile))
1035         else:
1036             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1037         raise BuildException(_("Invalid APK"))
1038     for line in p.output.splitlines():
1039         if line.startswith("package:"):
1040             try:
1041                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1042                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1043                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1044             except Exception as e:
1045                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1046         elif line.startswith("application:"):
1047             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1048             # Keep path to non-dpi icon in case we need it
1049             match = re.match(APK_ICON_PAT_NODPI, line)
1050             if match:
1051                 apk['icons_src']['-1'] = match.group(1)
1052         elif line.startswith("launchable-activity:"):
1053             # Only use launchable-activity as fallback to application
1054             if not apk['name']:
1055                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1056             if '-1' not in apk['icons_src']:
1057                 match = re.match(APK_ICON_PAT_NODPI, line)
1058                 if match:
1059                     apk['icons_src']['-1'] = match.group(1)
1060         elif line.startswith("application-icon-"):
1061             match = re.match(APK_ICON_PAT, line)
1062             if match:
1063                 density = match.group(1)
1064                 path = match.group(2)
1065                 apk['icons_src'][density] = path
1066         elif line.startswith("sdkVersion:"):
1067             m = re.match(APK_SDK_VERSION_PAT, line)
1068             if m is None:
1069                 logging.error(line.replace('sdkVersion:', '')
1070                               + ' is not a valid minSdkVersion!')
1071             else:
1072                 apk['minSdkVersion'] = m.group(1)
1073                 # if target not set, default to min
1074                 if 'targetSdkVersion' not in apk:
1075                     apk['targetSdkVersion'] = m.group(1)
1076         elif line.startswith("targetSdkVersion:"):
1077             m = re.match(APK_SDK_VERSION_PAT, line)
1078             if m is None:
1079                 logging.error(line.replace('targetSdkVersion:', '')
1080                               + ' is not a valid targetSdkVersion!')
1081             else:
1082                 apk['targetSdkVersion'] = m.group(1)
1083         elif line.startswith("maxSdkVersion:"):
1084             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1085         elif line.startswith("native-code:"):
1086             apk['nativecode'] = []
1087             for arch in line[13:].split(' '):
1088                 apk['nativecode'].append(arch[1:-1])
1089         elif line.startswith('uses-permission:'):
1090             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1091             if perm_match['maxSdkVersion']:
1092                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1093             permission = UsesPermission(
1094                 perm_match['name'],
1095                 perm_match['maxSdkVersion']
1096             )
1097
1098             apk['uses-permission'].append(permission)
1099         elif line.startswith('uses-permission-sdk-23:'):
1100             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1101             if perm_match['maxSdkVersion']:
1102                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1103             permission_sdk_23 = UsesPermissionSdk23(
1104                 perm_match['name'],
1105                 perm_match['maxSdkVersion']
1106             )
1107
1108             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1109
1110         elif line.startswith('uses-feature:'):
1111             feature = re.match(APK_FEATURE_PAT, line).group(1)
1112             # Filter out this, it's only added with the latest SDK tools and
1113             # causes problems for lots of apps.
1114             if feature != "android.hardware.screen.portrait" \
1115                     and feature != "android.hardware.screen.landscape":
1116                 if feature.startswith("android.feature."):
1117                     feature = feature[16:]
1118                 apk['features'].add(feature)
1119
1120
1121 def scan_apk_androguard(apk, apkfile):
1122     try:
1123         from androguard.core.bytecodes.apk import APK
1124         apkobject = APK(apkfile)
1125         if apkobject.is_valid_APK():
1126             arsc = apkobject.get_android_resources()
1127         else:
1128             if options.delete_unknown:
1129                 if os.path.exists(apkfile):
1130                     logging.error(_("Failed to get apk information, deleting {path}")
1131                                   .format(path=apkfile))
1132                     os.remove(apkfile)
1133                 else:
1134                     logging.error(_("Could not find {path} to remove it")
1135                                   .format(path=apkfile))
1136             else:
1137                 logging.error(_("Failed to get apk information, skipping {path}")
1138                               .format(path=apkfile))
1139             raise BuildException(_("Invalid APK"))
1140     except ImportError:
1141         raise FDroidException("androguard library is not installed and aapt not present")
1142     except FileNotFoundError:
1143         logging.error(_("Could not open apk file for analysis"))
1144         raise BuildException(_("Invalid APK"))
1145
1146     apk['packageName'] = apkobject.get_package()
1147     apk['versionCode'] = int(apkobject.get_androidversion_code())
1148     apk['versionName'] = apkobject.get_androidversion_name()
1149     if apk['versionName'][0] == "@":
1150         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1151         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1152         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1153     apk['name'] = apkobject.get_app_name()
1154
1155     if apkobject.get_max_sdk_version() is not None:
1156         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1157     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1158     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1159
1160     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1161     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1162
1163     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1164
1165     for file in apkobject.get_files():
1166         d_re = density_re.match(file)
1167         if d_re:
1168             folder = d_re.group(1).split('-')
1169             if len(folder) > 1:
1170                 resolution = folder[1]
1171             else:
1172                 resolution = 'mdpi'
1173             density = screen_resolutions[resolution]
1174             apk['icons_src'][density] = d_re.group(0)
1175
1176     if apk['icons_src'].get('-1') is None:
1177         apk['icons_src']['-1'] = apk['icons_src']['160']
1178
1179     arch_re = re.compile("^lib/(.*)/.*$")
1180     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1181     if len(arch) >= 1:
1182         apk['nativecode'] = []
1183         apk['nativecode'].extend(sorted(list(arch)))
1184
1185     xml = apkobject.get_android_manifest_xml()
1186
1187     for item in xml.getElementsByTagName('uses-permission'):
1188         name = str(item.getAttribute("android:name"))
1189         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1190         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1191         permission = UsesPermission(
1192             name,
1193             maxSdkVersion
1194         )
1195         apk['uses-permission'].append(permission)
1196
1197     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1198         name = str(item.getAttribute("android:name"))
1199         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1200         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1201         permission_sdk_23 = UsesPermissionSdk23(
1202             name,
1203             maxSdkVersion
1204         )
1205         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1206
1207     for item in xml.getElementsByTagName('uses-feature'):
1208         feature = str(item.getAttribute("android:name"))
1209         if feature != "android.hardware.screen.portrait" \
1210                 and feature != "android.hardware.screen.landscape":
1211             if feature.startswith("android.feature."):
1212                 feature = feature[16:]
1213         apk['features'].append(feature)
1214
1215
1216 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1217                 allow_disabled_algorithms=False, archive_bad_sig=False):
1218     """Processes the apk with the given filename in the given repo directory.
1219
1220     This also extracts the icons.
1221
1222     :param apkcache: current apk cache information
1223     :param apkfilename: the filename of the apk to scan
1224     :param repodir: repo directory to scan
1225     :param knownapks: known apks info
1226     :param use_date_from_apk: use date from APK (instead of current date)
1227                               for newly added APKs
1228     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1229                                       disabled algorithms in the signature (e.g. MD5)
1230     :param archive_bad_sig: move APKs with a bad signature to the archive
1231     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1232      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1233     """
1234
1235     apk = {}
1236     apkfile = os.path.join(repodir, apkfilename)
1237
1238     cachechanged = False
1239     usecache = False
1240     if apkfilename in apkcache:
1241         apk = apkcache[apkfilename]
1242         if apk.get('hash') == sha256sum(apkfile):
1243             logging.debug(_("Reading {apkfilename} from cache")
1244                           .format(apkfilename=apkfilename))
1245             usecache = True
1246         else:
1247             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1248                           .format(apkfilename=apkfilename))
1249
1250     if not usecache:
1251         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1252
1253         try:
1254             apk = scan_apk(apkfile)
1255         except BuildException:
1256             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1257                             .format(apkfilename=apkfilename))
1258             return True, None, False
1259
1260         # Check for debuggable apks...
1261         if common.isApkAndDebuggable(apkfile):
1262             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1263
1264         if options.rename_apks:
1265             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1266             std_short_name = os.path.join(repodir, n)
1267             if apkfile != std_short_name:
1268                 if os.path.exists(std_short_name):
1269                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1270                     if apkfile != std_long_name:
1271                         if os.path.exists(std_long_name):
1272                             dupdir = os.path.join('duplicates', repodir)
1273                             if not os.path.isdir(dupdir):
1274                                 os.makedirs(dupdir, exist_ok=True)
1275                             dupfile = os.path.join('duplicates', std_long_name)
1276                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1277                             os.rename(apkfile, dupfile)
1278                             return True, None, False
1279                         else:
1280                             os.rename(apkfile, std_long_name)
1281                     apkfile = std_long_name
1282                 else:
1283                     os.rename(apkfile, std_short_name)
1284                     apkfile = std_short_name
1285                 apkfilename = apkfile[len(repodir) + 1:]
1286
1287         apk['apkName'] = apkfilename
1288         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1289         if os.path.exists(os.path.join(repodir, srcfilename)):
1290             apk['srcname'] = srcfilename
1291
1292         # verify the jar signature is correct, allow deprecated
1293         # algorithms only if the APK is in the archive.
1294         skipapk = False
1295         if not common.verify_apk_signature(apkfile):
1296             if repodir == 'archive' or allow_disabled_algorithms:
1297                 if common.verify_old_apk_signature(apkfile):
1298                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1299                 else:
1300                     skipapk = True
1301             else:
1302                 skipapk = True
1303
1304         if skipapk:
1305             if archive_bad_sig:
1306                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1307                                 .format(apkfilename=apkfilename))
1308                 move_apk_between_sections(repodir, 'archive', apk)
1309             else:
1310                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1311                                 .format(apkfilename=apkfilename))
1312             return True, None, False
1313
1314         apkzip = zipfile.ZipFile(apkfile, 'r')
1315
1316         manifest = apkzip.getinfo('AndroidManifest.xml')
1317         if manifest.date_time[1] == 0:  # month can't be zero
1318             logging.debug(_('AndroidManifest.xml has no date'))
1319         else:
1320             common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1321
1322         # extract icons from APK zip file
1323         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1324         try:
1325             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1326         finally:
1327             apkzip.close()  # ensure that APK zip file gets closed
1328
1329         # resize existing icons for densities missing in the APK
1330         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1331
1332         if use_date_from_apk and manifest.date_time[1] != 0:
1333             default_date_param = datetime(*manifest.date_time)
1334         else:
1335             default_date_param = None
1336
1337         # Record in known apks, getting the added date at the same time..
1338         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1339                                     default_date=default_date_param)
1340         if added:
1341             apk['added'] = added
1342
1343         apkcache[apkfilename] = apk
1344         cachechanged = True
1345
1346     return False, apk, cachechanged
1347
1348
1349 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1350     """Processes the apks in the given repo directory.
1351
1352     This also extracts the icons.
1353
1354     :param apkcache: current apk cache information
1355     :param repodir: repo directory to scan
1356     :param knownapks: known apks info
1357     :param use_date_from_apk: use date from APK (instead of current date)
1358                               for newly added APKs
1359     :returns: (apks, cachechanged) where apks is a list of apk information,
1360               and cachechanged is True if the apkcache got changed.
1361     """
1362
1363     cachechanged = False
1364
1365     for icon_dir in get_all_icon_dirs(repodir):
1366         if os.path.exists(icon_dir):
1367             if options.clean:
1368                 shutil.rmtree(icon_dir)
1369                 os.makedirs(icon_dir)
1370         else:
1371             os.makedirs(icon_dir)
1372
1373     apks = []
1374     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1375         apkfilename = apkfile[len(repodir) + 1:]
1376         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1377         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1378                                              use_date_from_apk, ada, True)
1379         if skip:
1380             continue
1381         apks.append(apk)
1382         cachechanged = cachechanged or cachethis
1383
1384     return apks, cachechanged
1385
1386
1387 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1388     """
1389     Extracts icons from the given APK zip in various densities,
1390     saves them into given repo directory
1391     and stores their names in the APK metadata dictionary.
1392
1393     :param icon_filename: A string representing the icon's file name
1394     :param apk: A populated dictionary containing APK metadata.
1395                 Needs to have 'icons_src' key
1396     :param apkzip: An opened zipfile.ZipFile of the APK file
1397     :param repo_dir: The directory of the APK's repository
1398     :return: A list of icon densities that are missing
1399     """
1400     empty_densities = []
1401     for density in screen_densities:
1402         if density not in apk['icons_src']:
1403             empty_densities.append(density)
1404             continue
1405         icon_src = apk['icons_src'][density]
1406         icon_dir = get_icon_dir(repo_dir, density)
1407         icon_dest = os.path.join(icon_dir, icon_filename)
1408
1409         # Extract the icon files per density
1410         if icon_src.endswith('.xml'):
1411             png = os.path.basename(icon_src)[:-4] + '.png'
1412             for f in apkzip.namelist():
1413                 if f.endswith(png):
1414                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1415                     if m and screen_resolutions[m.group(2)] == density:
1416                         icon_src = f
1417             if icon_src.endswith('.xml'):
1418                 empty_densities.append(density)
1419                 continue
1420         try:
1421             with open(icon_dest, 'wb') as f:
1422                 f.write(get_icon_bytes(apkzip, icon_src))
1423             apk['icons'][density] = icon_filename
1424         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1425             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1426             del apk['icons_src'][density]
1427             empty_densities.append(density)
1428
1429     if '-1' in apk['icons_src']:
1430         icon_src = apk['icons_src']['-1']
1431         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1432         with open(icon_path, 'wb') as f:
1433             f.write(get_icon_bytes(apkzip, icon_src))
1434         try:
1435             im = Image.open(icon_path)
1436             dpi = px_to_dpi(im.size[0])
1437             for density in screen_densities:
1438                 if density in apk['icons']:
1439                     break
1440                 if density == screen_densities[-1] or dpi >= int(density):
1441                     apk['icons'][density] = icon_filename
1442                     shutil.move(icon_path,
1443                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1444                     empty_densities.remove(density)
1445                     break
1446         except Exception as e:
1447             logging.warning(_("Failed reading {path}: {error}")
1448                             .format(path=icon_path, error=e))
1449
1450     if apk['icons']:
1451         apk['icon'] = icon_filename
1452
1453     return empty_densities
1454
1455
1456 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1457     """
1458     Resize existing icons for densities missing in the APK to ensure all densities are available
1459
1460     :param empty_densities: A list of icon densities that are missing
1461     :param icon_filename: A string representing the icon's file name
1462     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1463     :param repo_dir: The directory of the APK's repository
1464     """
1465     # First try resizing down to not lose quality
1466     last_density = None
1467     for density in screen_densities:
1468         if density not in empty_densities:
1469             last_density = density
1470             continue
1471         if last_density is None:
1472             continue
1473         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1474
1475         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1476         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1477         fp = None
1478         try:
1479             fp = open(last_icon_path, 'rb')
1480             im = Image.open(fp)
1481
1482             size = dpi_to_px(density)
1483
1484             im.thumbnail((size, size), Image.ANTIALIAS)
1485             im.save(icon_path, "PNG")
1486             empty_densities.remove(density)
1487         except Exception as e:
1488             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1489         finally:
1490             if fp:
1491                 fp.close()
1492
1493     # Then just copy from the highest resolution available
1494     last_density = None
1495     for density in reversed(screen_densities):
1496         if density not in empty_densities:
1497             last_density = density
1498             continue
1499
1500         if last_density is None:
1501             continue
1502
1503         shutil.copyfile(
1504             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1505             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1506         )
1507         empty_densities.remove(density)
1508
1509     for density in screen_densities:
1510         icon_dir = get_icon_dir(repo_dir, density)
1511         icon_dest = os.path.join(icon_dir, icon_filename)
1512         resize_icon(icon_dest, density)
1513
1514     # Copy from icons-mdpi to icons since mdpi is the baseline density
1515     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1516     if os.path.isfile(baseline):
1517         apk['icons']['0'] = icon_filename
1518         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1519
1520
1521 def apply_info_from_latest_apk(apps, apks):
1522     """
1523     Some information from the apks needs to be applied up to the application level.
1524     When doing this, we use the info from the most recent version's apk.
1525     We deal with figuring out when the app was added and last updated at the same time.
1526     """
1527     for appid, app in apps.items():
1528         bestver = UNSET_VERSION_CODE
1529         for apk in apks:
1530             if apk['packageName'] == appid:
1531                 if apk['versionCode'] > bestver:
1532                     bestver = apk['versionCode']
1533                     bestapk = apk
1534
1535                 if 'added' in apk:
1536                     if not app.added or apk['added'] < app.added:
1537                         app.added = apk['added']
1538                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1539                         app.lastUpdated = apk['added']
1540
1541         if not app.added:
1542             logging.debug("Don't know when " + appid + " was added")
1543         if not app.lastUpdated:
1544             logging.debug("Don't know when " + appid + " was last updated")
1545
1546         if bestver == UNSET_VERSION_CODE:
1547
1548             if app.Name is None:
1549                 app.Name = app.AutoName or appid
1550             app.icon = None
1551             logging.debug("Application " + appid + " has no packages")
1552         else:
1553             if app.Name is None:
1554                 app.Name = bestapk['name']
1555             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1556             if app.CurrentVersionCode is None:
1557                 app.CurrentVersionCode = str(bestver)
1558
1559
1560 def make_categories_txt(repodir, categories):
1561     '''Write a category list in the repo to allow quick access'''
1562     catdata = ''
1563     for cat in sorted(categories):
1564         catdata += cat + '\n'
1565     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1566         f.write(catdata)
1567
1568
1569 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1570
1571     def filter_apk_list_sorted(apk_list):
1572         res = []
1573         for apk in apk_list:
1574             if apk['packageName'] == appid:
1575                 res.append(apk)
1576
1577         # Sort the apk list by version code. First is highest/newest.
1578         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1579
1580     for appid, app in apps.items():
1581
1582         if app.ArchivePolicy:
1583             keepversions = int(app.ArchivePolicy[:-9])
1584         else:
1585             keepversions = defaultkeepversions
1586
1587         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1588                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1589
1590         current_app_apks = filter_apk_list_sorted(apks)
1591         if len(current_app_apks) > keepversions:
1592             # Move back the ones we don't want.
1593             for apk in current_app_apks[keepversions:]:
1594                 move_apk_between_sections(repodir, archivedir, apk)
1595                 archapks.append(apk)
1596                 apks.remove(apk)
1597
1598         current_app_archapks = filter_apk_list_sorted(archapks)
1599         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1600             kept = 0
1601             # Move forward the ones we want again, except DisableAlgorithm
1602             for apk in current_app_archapks:
1603                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1604                     move_apk_between_sections(archivedir, repodir, apk)
1605                     archapks.remove(apk)
1606                     apks.append(apk)
1607                     kept += 1
1608                 if kept == keepversions:
1609                     break
1610
1611
1612 def move_apk_between_sections(from_dir, to_dir, apk):
1613     """move an APK from repo to archive or vice versa"""
1614
1615     def _move_file(from_dir, to_dir, filename, ignore_missing):
1616         from_path = os.path.join(from_dir, filename)
1617         if ignore_missing and not os.path.exists(from_path):
1618             return
1619         to_path = os.path.join(to_dir, filename)
1620         if not os.path.exists(to_dir):
1621             os.mkdir(to_dir)
1622         shutil.move(from_path, to_path)
1623
1624     if from_dir == to_dir:
1625         return
1626
1627     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1628     _move_file(from_dir, to_dir, apk['apkName'], False)
1629     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1630     for density in all_screen_densities:
1631         from_icon_dir = get_icon_dir(from_dir, density)
1632         to_icon_dir = get_icon_dir(to_dir, density)
1633         if density not in apk.get('icons', []):
1634             continue
1635         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1636     if 'srcname' in apk:
1637         _move_file(from_dir, to_dir, apk['srcname'], False)
1638
1639
1640 def add_apks_to_per_app_repos(repodir, apks):
1641     apks_per_app = dict()
1642     for apk in apks:
1643         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1644         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1645         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1646         apks_per_app[apk['packageName']] = apk
1647
1648         if not os.path.exists(apk['per_app_icons']):
1649             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1650             os.makedirs(apk['per_app_icons'])
1651
1652         apkpath = os.path.join(repodir, apk['apkName'])
1653         shutil.copy(apkpath, apk['per_app_repo'])
1654         apksigpath = apkpath + '.sig'
1655         if os.path.exists(apksigpath):
1656             shutil.copy(apksigpath, apk['per_app_repo'])
1657         apkascpath = apkpath + '.asc'
1658         if os.path.exists(apkascpath):
1659             shutil.copy(apkascpath, apk['per_app_repo'])
1660
1661
1662 def create_metadata_from_template(apk):
1663     '''create a new metadata file using internal or external template
1664
1665     Generate warnings for apk's with no metadata (or create skeleton
1666     metadata files, if requested on the command line).  Though the
1667     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1668     since those impose things on the metadata file made from the
1669     template: field sort order, empty field value, formatting, etc.
1670     '''
1671
1672     import yaml
1673     if os.path.exists('template.yml'):
1674         with open('template.yml') as f:
1675             metatxt = f.read()
1676         if 'name' in apk and apk['name'] != '':
1677             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1678                              r'\1 ' + apk['name'],
1679                              metatxt,
1680                              flags=re.IGNORECASE | re.MULTILINE)
1681         else:
1682             logging.warning(_('{appid} does not have a name! Using package name instead.')
1683                             .format(appid=apk['packageName']))
1684             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1685                              r'\1 ' + apk['packageName'],
1686                              metatxt,
1687                              flags=re.IGNORECASE | re.MULTILINE)
1688         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1689             f.write(metatxt)
1690     else:
1691         app = dict()
1692         app['Categories'] = [os.path.basename(os.getcwd())]
1693         # include some blanks as part of the template
1694         app['AuthorName'] = ''
1695         app['Summary'] = ''
1696         app['WebSite'] = ''
1697         app['IssueTracker'] = ''
1698         app['SourceCode'] = ''
1699         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1700         if 'name' in apk and apk['name'] != '':
1701             app['Name'] = apk['name']
1702         else:
1703             logging.warning(_('{appid} does not have a name! Using package name instead.')
1704                             .format(appid=apk['packageName']))
1705             app['Name'] = apk['packageName']
1706         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1707             yaml.dump(app, f, default_flow_style=False)
1708     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1709
1710
1711 config = None
1712 options = None
1713
1714
1715 def main():
1716
1717     global config, options
1718
1719     # Parse command line...
1720     parser = ArgumentParser()
1721     common.setup_global_opts(parser)
1722     parser.add_argument("--create-key", action="store_true", default=False,
1723                         help=_("Add a repo signing key to an unsigned repo"))
1724     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1725                         help=_("Add skeleton metadata files for APKs that are missing them"))
1726     parser.add_argument("--delete-unknown", action="store_true", default=False,
1727                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1728     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1729                         help=_("Report on build data status"))
1730     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1731                         help=_("Interactively ask about things that need updating."))
1732     parser.add_argument("-I", "--icons", action="store_true", default=False,
1733                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1734     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1735                         help=_("Specify editor to use in interactive mode. Default " +
1736                                "is {path}").format(path='/etc/alternatives/editor'))
1737     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1738                         help=_("Update the wiki"))
1739     parser.add_argument("--pretty", action="store_true", default=False,
1740                         help=_("Produce human-readable XML/JSON for index files"))
1741     parser.add_argument("--clean", action="store_true", default=False,
1742                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1743     parser.add_argument("--nosign", action="store_true", default=False,
1744                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1745     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1746                         help=_("Use date from APK instead of current time for newly added APKs"))
1747     parser.add_argument("--rename-apks", action="store_true", default=False,
1748                         help=_("Rename APK files that do not match package.name_123.apk"))
1749     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1750                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1751     metadata.add_metadata_arguments(parser)
1752     options = parser.parse_args()
1753     metadata.warnings_action = options.W
1754
1755     config = common.read_config(options)
1756
1757     if not ('jarsigner' in config and 'keytool' in config):
1758         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1759
1760     repodirs = ['repo']
1761     if config['archive_older'] != 0:
1762         repodirs.append('archive')
1763         if not os.path.exists('archive'):
1764             os.mkdir('archive')
1765
1766     if options.icons:
1767         resize_all_icons(repodirs)
1768         sys.exit(0)
1769
1770     if options.rename_apks:
1771         options.clean = True
1772
1773     # check that icons exist now, rather than fail at the end of `fdroid update`
1774     for k in ['repo_icon', 'archive_icon']:
1775         if k in config:
1776             if not os.path.exists(config[k]):
1777                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1778                                  .format(name=k, path=config[k]))
1779                 sys.exit(1)
1780
1781     # if the user asks to create a keystore, do it now, reusing whatever it can
1782     if options.create_key:
1783         if os.path.exists(config['keystore']):
1784             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1785             logging.critical("\t'" + config['keystore'] + "'")
1786             sys.exit(1)
1787
1788         if 'repo_keyalias' not in config:
1789             config['repo_keyalias'] = socket.getfqdn()
1790             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1791         if 'keydname' not in config:
1792             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1793             common.write_to_config(config, 'keydname', config['keydname'])
1794         if 'keystore' not in config:
1795             config['keystore'] = common.default_config['keystore']
1796             common.write_to_config(config, 'keystore', config['keystore'])
1797
1798         password = common.genpassword()
1799         if 'keystorepass' not in config:
1800             config['keystorepass'] = password
1801             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1802         if 'keypass' not in config:
1803             config['keypass'] = password
1804             common.write_to_config(config, 'keypass', config['keypass'])
1805         common.genkeystore(config)
1806
1807     # Get all apps...
1808     apps = metadata.read_metadata()
1809
1810     # Generate a list of categories...
1811     categories = set()
1812     for app in apps.values():
1813         categories.update(app.Categories)
1814
1815     # Read known apks data (will be updated and written back when we've finished)
1816     knownapks = common.KnownApks()
1817
1818     # Get APK cache
1819     apkcache = get_cache()
1820
1821     # Delete builds for disabled apps
1822     delete_disabled_builds(apps, apkcache, repodirs)
1823
1824     # Scan all apks in the main repo
1825     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1826
1827     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1828                                            options.use_date_from_apk)
1829     cachechanged = cachechanged or fcachechanged
1830     apks += files
1831     for apk in apks:
1832         if apk['packageName'] not in apps:
1833             if options.create_metadata:
1834                 create_metadata_from_template(apk)
1835                 apps = metadata.read_metadata()
1836             else:
1837                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1838                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1839                 if options.delete_unknown:
1840                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1841                                  .format(apkfilename=apk['apkName']))
1842                     rmf = os.path.join(repodirs[0], apk['apkName'])
1843                     if not os.path.exists(rmf):
1844                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1845                     else:
1846                         os.remove(rmf)
1847                 else:
1848                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1849
1850     copy_triple_t_store_metadata(apps)
1851     insert_obbs(repodirs[0], apps, apks)
1852     insert_localized_app_metadata(apps)
1853     translate_per_build_anti_features(apps, apks)
1854
1855     # Scan the archive repo for apks as well
1856     if len(repodirs) > 1:
1857         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1858         if cc:
1859             cachechanged = True
1860     else:
1861         archapks = []
1862
1863     # Apply information from latest apks to the application and update dates
1864     apply_info_from_latest_apk(apps, apks + archapks)
1865
1866     # Sort the app list by name, then the web site doesn't have to by default.
1867     # (we had to wait until we'd scanned the apks to do this, because mostly the
1868     # name comes from there!)
1869     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1870
1871     # APKs are placed into multiple repos based on the app package, providing
1872     # per-app subscription feeds for nightly builds and things like it
1873     if config['per_app_repos']:
1874         add_apks_to_per_app_repos(repodirs[0], apks)
1875         for appid, app in apps.items():
1876             repodir = os.path.join(appid, 'fdroid', 'repo')
1877             appdict = dict()
1878             appdict[appid] = app
1879             if os.path.isdir(repodir):
1880                 index.make(appdict, [appid], apks, repodir, False)
1881             else:
1882                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1883         return
1884
1885     if len(repodirs) > 1:
1886         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1887
1888     # Make the index for the main repo...
1889     index.make(apps, sortedids, apks, repodirs[0], False)
1890     make_categories_txt(repodirs[0], categories)
1891
1892     # If there's an archive repo,  make the index for it. We already scanned it
1893     # earlier on.
1894     if len(repodirs) > 1:
1895         index.make(apps, sortedids, archapks, repodirs[1], True)
1896
1897     git_remote = config.get('binary_transparency_remote')
1898     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1899         from . import btlog
1900         btlog.make_binary_transparency_log(repodirs)
1901
1902     if config['update_stats']:
1903         # Update known apks info...
1904         knownapks.writeifchanged()
1905
1906         # Generate latest apps data for widget
1907         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1908             data = ''
1909             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1910                 for line in f:
1911                     appid = line.rstrip()
1912                     data += appid + "\t"
1913                     app = apps[appid]
1914                     data += app.Name + "\t"
1915                     if app.icon is not None:
1916                         data += app.icon + "\t"
1917                     data += app.License + "\n"
1918             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1919                 f.write(data)
1920
1921     if cachechanged:
1922         write_cache(apkcache)
1923
1924     # Update the wiki...
1925     if options.wiki:
1926         update_wiki(apps, sortedids, apks + archapks)
1927
1928     logging.info(_("Finished"))
1929
1930
1931 if __name__ == "__main__":
1932     main()