fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime, timedelta
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in app.AntiFeatures:
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.Bitcoin,
 153             app.Litecoin,
 154             app.License,
 155             requiresroot,
 156             app.AuthorName,
 157             app.AuthorEmail)
 158
 159         if app.Provides:
 160             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 161
 162         wikidata += app.Summary
 163         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 164
 165         wikidata += "=Description=\n"
 166         wikidata += metadata.description_wiki(app.Description) + "\n"
 167
 168         wikidata += "=Maintainer Notes=\n"
 169         if app.MaintainerNotes:
 170             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 171         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 172
 173         # Get a list of all packages for this application...
 174         apklist = []
 175         gotcurrentver = False
 176         cantupdate = False
 177         buildfails = False
 178         for apk in apks:
 179             if apk['packageName'] == appid:
 180                 if str(apk['versionCode']) == app.CurrentVersionCode:
 181                     gotcurrentver = True
 182                 apklist.append(apk)
 183         # Include ones we can't build, as a special case...
 184         for build in app.builds:
 185             if build.disable:
 186                 if build.versionCode == app.CurrentVersionCode:
 187                     cantupdate = True
 188                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 189                 apklist.append({'versionCode': int(build.versionCode),
 190                                 'versionName': build.versionName,
 191                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 192                                 })
 193             else:
 194                 builtit = False
 195                 for apk in apklist:
 196                     if apk['versionCode'] == int(build.versionCode):
 197                         builtit = True
 198                         break
 199                 if not builtit:
 200                     buildfails = True
 201                     apklist.append({'versionCode': int(build.versionCode),
 202                                     'versionName': build.versionName,
 203                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 204                                     })
 205         if app.CurrentVersionCode == '0':
 206             cantupdate = True
 207         # Sort with most recent first...
 208         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 209
 210         wikidata += "=Versions=\n"
 211         if len(apklist) == 0:
 212             wikidata += "We currently have no versions of this app available."
 213         elif not gotcurrentver:
 214             wikidata += "We don't have the current version of this app."
 215         else:
 216             wikidata += "We have the current version of this app."
 217         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 218         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 219         if len(app.NoSourceSince) > 0:
 220             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 221         if len(app.CurrentVersion) > 0:
 222             wikidata += "The current (recommended) version is " + app.CurrentVersion
 223             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 224         validapks = 0
 225         for apk in apklist:
 226             wikidata += "==" + apk['versionName'] + "==\n"
 227
 228             if 'buildproblem' in apk:
 229                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 230             else:
 231                 validapks += 1
 232                 wikidata += "This version is built and signed by "
 233                 if 'srcname' in apk:
 234                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 235                 else:
 236                     wikidata += "the original developer.\n\n"
 237             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 238
 239         wikidata += '\n[[Category:' + wikicat + ']]\n'
 240         if len(app.NoSourceSince) > 0:
 241             wikidata += '\n[[Category:Apps missing source code]]\n'
 242         if validapks == 0 and not app.Disabled:
 243             wikidata += '\n[[Category:Apps with no packages]]\n'
 244         if cantupdate and not app.Disabled:
 245             wikidata += "\n[[Category:Apps we cannot update]]\n"
 246         if buildfails and not app.Disabled:
 247             wikidata += "\n[[Category:Apps with failing builds]]\n"
 248         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 249             wikidata += '\n[[Category:Apps to Update]]\n'
 250         if app.Disabled:
 251             wikidata += '\n[[Category:Apps that are disabled]]\n'
 252         if app.UpdateCheckMode == 'None' and not app.Disabled:
 253             wikidata += '\n[[Category:Apps with no update check]]\n'
 254         for appcat in app.Categories:
 255             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 256
 257         # We can't have underscores in the page name, even if they're in
 258         # the package ID, because MediaWiki messes with them...
 259         pagename = appid.replace('_', ' ')
 260
 261         # Drop a trailing newline, because mediawiki is going to drop it anyway
 262         # and it we don't we'll think the page has changed when it hasn't...
 263         if wikidata.endswith('\n'):
 264             wikidata = wikidata[:-1]
 265
 266         generated_pages[pagename] = wikidata
 267
 268         # Make a redirect from the name to the ID too, unless there's
 269         # already an existing page with the name and it isn't a redirect.
 270         noclobber = False
 271         apppagename = app.Name.replace('_', ' ')
 272         apppagename = apppagename.replace('{', '')
 273         apppagename = apppagename.replace('}', ' ')
 274         apppagename = apppagename.replace(':', ' ')
 275         apppagename = apppagename.replace('[', ' ')
 276         apppagename = apppagename.replace(']', ' ')
 277         # Drop double spaces caused mostly by replacing ':' above
 278         apppagename = apppagename.replace('  ', ' ')
 279         for expagename in site.allpages(prefix=apppagename,
 280                                         filterredir='nonredirects',
 281                                         generator=False):
 282             if expagename == apppagename:
 283                 noclobber = True
 284         # Another reason not to make the redirect page is if the app name
 285         # is the same as it's ID, because that will overwrite the real page
 286         # with an redirect to itself! (Although it seems like an odd
 287         # scenario this happens a lot, e.g. where there is metadata but no
 288         # builds or binaries to extract a name from.
 289         if apppagename == pagename:
 290             noclobber = True
 291         if not noclobber:
 292             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 293
 294     for tcat, genp in [(wikicat, generated_pages),
 295                        (wikiredircat, generated_redirects)]:
 296         catpages = site.Pages['Category:' + tcat]
 297         existingpages = []
 298         for page in catpages:
 299             existingpages.append(page.name)
 300             if page.name in genp:
 301                 pagetxt = page.edit()
 302                 if pagetxt != genp[page.name]:
 303                     logging.debug("Updating modified page " + page.name)
 304                     page.save(genp[page.name], summary='Auto-updated')
 305                 else:
 306                     logging.debug("Page " + page.name + " is unchanged")
 307             else:
 308                 logging.warn("Deleting page " + page.name)
 309                 page.delete('No longer published')
 310         for pagename, text in genp.items():
 311             logging.debug("Checking " + pagename)
 312             if pagename not in existingpages:
 313                 logging.debug("Creating page " + pagename)
 314                 try:
 315                     newpage = site.Pages[pagename]
 316                     newpage.save(text, summary='Auto-created')
 317                 except Exception as e:
 318                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 319
 320     # Purge server cache to ensure counts are up to date
 321     site.pages['Repository Maintenance'].purge()
 322
 323
 324 def delete_disabled_builds(apps, apkcache, repodirs):
 325     """Delete disabled build outputs.
 326
 327     :param apps: list of all applications, as per metadata.read_metadata
 328     :param apkcache: current apk cache information
 329     :param repodirs: the repo directories to process
 330     """
 331     for appid, app in apps.items():
 332         for build in app['builds']:
 333             if not build.disable:
 334                 continue
 335             apkfilename = common.get_release_filename(app, build)
 336             iconfilename = "%s.%s.png" % (
 337                 appid,
 338                 build.versionCode)
 339             for repodir in repodirs:
 340                 files = [
 341                     os.path.join(repodir, apkfilename),
 342                     os.path.join(repodir, apkfilename + '.asc'),
 343                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 344                 ]
 345                 for density in all_screen_densities:
 346                     repo_dir = get_icon_dir(repodir, density)
 347                     files.append(os.path.join(repo_dir, iconfilename))
 348
 349                 for f in files:
 350                     if os.path.exists(f):
 351                         logging.info("Deleting disabled build output " + f)
 352                         os.remove(f)
 353             if apkfilename in apkcache:
 354                 del apkcache[apkfilename]
 355
 356
 357 def resize_icon(iconpath, density):
 358
 359     if not os.path.isfile(iconpath):
 360         return
 361
 362     fp = None
 363     try:
 364         fp = open(iconpath, 'rb')
 365         im = Image.open(fp)
 366         size = dpi_to_px(density)
 367
 368         if any(length > size for length in im.size):
 369             oldsize = im.size
 370             im.thumbnail((size, size), Image.ANTIALIAS)
 371             logging.debug("%s was too large at %s - new size is %s" % (
 372                 iconpath, oldsize, im.size))
 373             im.save(iconpath, "PNG")
 374
 375     except Exception as e:
 376         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 377
 378     finally:
 379         if fp:
 380             fp.close()
 381
 382
 383 def resize_all_icons(repodirs):
 384     """Resize all icons that exceed the max size
 385
 386     :param repodirs: the repo directories to process
 387     """
 388     for repodir in repodirs:
 389         for density in screen_densities:
 390             icon_dir = get_icon_dir(repodir, density)
 391             icon_glob = os.path.join(icon_dir, '*.png')
 392             for iconpath in glob.glob(icon_glob):
 393                 resize_icon(iconpath, density)
 394
 395
 396 def getsig(apkpath):
 397     """ Get the signing certificate of an apk. To get the same md5 has that
 398     Android gets, we encode the .RSA certificate in a specific format and pass
 399     it hex-encoded to the md5 digest algorithm.
 400
 401     :param apkpath: path to the apk
 402     :returns: A string containing the md5 of the signature of the apk or None
 403               if an error occurred.
 404     """
 405
 406     with zipfile.ZipFile(apkpath, 'r') as apk:
 407         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 408
 409         if len(certs) < 1:
 410             logging.error("Found no signing certificates on %s" % apkpath)
 411             return None
 412         if len(certs) > 1:
 413             logging.error("Found multiple signing certificates on %s" % apkpath)
 414             return None
 415
 416         cert = apk.read(certs[0])
 417
 418     cert_encoded = common.get_certificate(cert)
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_cache_file():
 424     return os.path.join('tmp', 'apkcache')
 425
 426
 427 def get_cache():
 428     """Get the cached dict of the APK index
 429
 430     Gather information about all the apk files in the repo directory,
 431     using cached data if possible. Some of the index operations take a
 432     long time, like calculating the SHA-256 and verifying the APK
 433     signature.
 434
 435     The cache is invalidated if the metadata version is different, or
 436     the 'allow_disabled_algorithms' config/option is different.  In
 437     those cases, there is no easy way to know what has changed from
 438     the cache, so just rerun the whole thing.
 439
 440     :return: apkcache
 441
 442     """
 443     apkcachefile = get_cache_file()
 444     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 445     if not options.clean and os.path.exists(apkcachefile):
 446         with open(apkcachefile, 'rb') as cf:
 447             apkcache = pickle.load(cf, encoding='utf-8')
 448         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 449            or apkcache.get('allow_disabled_algorithms') != ada:
 450             apkcache = {}
 451     else:
 452         apkcache = {}
 453
 454     apkcache["METADATA_VERSION"] = METADATA_VERSION
 455     apkcache['allow_disabled_algorithms'] = ada
 456
 457     return apkcache
 458
 459
 460 def write_cache(apkcache):
 461     apkcachefile = get_cache_file()
 462     cache_path = os.path.dirname(apkcachefile)
 463     if not os.path.exists(cache_path):
 464         os.makedirs(cache_path)
 465     with open(apkcachefile, 'wb') as cf:
 466         pickle.dump(apkcache, cf)
 467
 468
 469 def get_icon_bytes(apkzip, iconsrc):
 470     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 471     try:
 472         return apkzip.read(iconsrc)
 473     except KeyError:
 474         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 475
 476
 477 def sha256sum(filename):
 478     '''Calculate the sha256 of the given file'''
 479     sha = hashlib.sha256()
 480     with open(filename, 'rb') as f:
 481         while True:
 482             t = f.read(16384)
 483             if len(t) == 0:
 484                 break
 485             sha.update(t)
 486     return sha.hexdigest()
 487
 488
 489 def has_known_vulnerability(filename):
 490     """checks for known vulnerabilities in the APK
 491
 492     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 493     version.  Google also enforces this:
 494     https://support.google.com/faqs/answer/6376725?hl=en
 495
 496     Checks whether there are more than one classes.dex or AndroidManifest.xml
 497     files, which is invalid and an essential part of the "Master Key" attack.
 498
 499     http://www.saurik.com/id/17
 500     """
 501
 502     # statically load this pattern
 503     if not hasattr(has_known_vulnerability, "pattern"):
 504         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 505
 506     files_in_apk = set()
 507     with zipfile.ZipFile(filename) as zf:
 508         for name in zf.namelist():
 509             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 510                 lib = zf.open(name)
 511                 while True:
 512                     chunk = lib.read(4096)
 513                     if chunk == b'':
 514                         break
 515                     m = has_known_vulnerability.pattern.search(chunk)
 516                     if m:
 517                         version = m.group(1).decode('ascii')
 518                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 519                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 520                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 521                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 522                         else:
 523                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 524                             return True
 525                         break
 526             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 527                 if name in files_in_apk:
 528                     return True
 529                 files_in_apk.add(name)
 530
 531     return False
 532
 533
 534 def insert_obbs(repodir, apps, apks):
 535     """Scans the .obb files in a given repo directory and adds them to the
 536     relevant APK instances.  OBB files have versionCodes like APK
 537     files, and they are loosely associated.  If there is an OBB file
 538     present, then any APK with the same or higher versionCode will use
 539     that OBB file.  There are two OBB types: main and patch, each APK
 540     can only have only have one of each.
 541
 542     https://developer.android.com/google/play/expansion-files.html
 543
 544     :param repodir: repo directory to scan
 545     :param apps: list of current, valid apps
 546     :param apks: current information on all APKs
 547
 548     """
 549
 550     def obbWarnDelete(f, msg):
 551         logging.warning(msg + f)
 552         if options.delete_unknown:
 553             logging.error("Deleting unknown file: " + f)
 554             os.remove(f)
 555
 556     obbs = []
 557     java_Integer_MIN_VALUE = -pow(2, 31)
 558     currentPackageNames = apps.keys()
 559     for f in glob.glob(os.path.join(repodir, '*.obb')):
 560         obbfile = os.path.basename(f)
 561         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 562         chunks = obbfile.split('.')
 563         if chunks[0] != 'main' and chunks[0] != 'patch':
 564             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 565             continue
 566         if not re.match(r'^-?[0-9]+$', chunks[1]):
 567             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 568             continue
 569         versionCode = int(chunks[1])
 570         packagename = ".".join(chunks[2:-1])
 571
 572         highestVersionCode = java_Integer_MIN_VALUE
 573         if packagename not in currentPackageNames:
 574             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 575             continue
 576         for apk in apks:
 577             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 578                 highestVersionCode = apk['versionCode']
 579         if versionCode > highestVersionCode:
 580             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 581                           + ') than any APK: ')
 582             continue
 583         obbsha256 = sha256sum(f)
 584         obbs.append((packagename, versionCode, obbfile, obbsha256))
 585
 586     for apk in apks:
 587         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 588             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 589                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 590                     apk['obbMainFile'] = obbfile
 591                     apk['obbMainFileSha256'] = obbsha256
 592                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 593                     apk['obbPatchFile'] = obbfile
 594                     apk['obbPatchFileSha256'] = obbsha256
 595             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 596                 break
 597
 598
 599 def translate_per_build_anti_features(apps, apks):
 600     """Grab the anti-features list from the build metadata
 601
 602     For most Anti-Features, they are really most applicable per-APK,
 603     not for an app.  An app can fix a vulnerability, add/remove
 604     tracking, etc.  This reads the 'antifeatures' list from the Build
 605     entries in the fdroiddata metadata file, then transforms it into
 606     the 'antiFeatures' list of unique items for the index.
 607
 608     The field key is all lower case in the metadata file to match the
 609     rest of the Build fields.  It is 'antiFeatures' camel case in the
 610     implementation, index, and fdroidclient since it is translated
 611     from the build 'antifeatures' field, not directly included.
 612
 613     """
 614
 615     antiFeatures = dict()
 616     for packageName, app in apps.items():
 617         d = dict()
 618         for build in app['builds']:
 619             afl = build.get('antifeatures')
 620             if afl:
 621                 d[int(build.versionCode)] = afl
 622         if len(d) > 0:
 623             antiFeatures[packageName] = d
 624
 625     for apk in apks:
 626         d = antiFeatures.get(apk['packageName'])
 627         if d:
 628             afl = d.get(apk['versionCode'])
 629             if afl:
 630                 apk['antiFeatures'].update(afl)
 631
 632
 633 def _get_localized_dict(app, locale):
 634     '''get the dict to add localized store metadata to'''
 635     if 'localized' not in app:
 636         app['localized'] = collections.OrderedDict()
 637     if locale not in app['localized']:
 638         app['localized'][locale] = collections.OrderedDict()
 639     return app['localized'][locale]
 640
 641
 642 def _set_localized_text_entry(app, locale, key, f):
 643     limit = config['char_limits'][key]
 644     localized = _get_localized_dict(app, locale)
 645     with open(f) as fp:
 646         text = fp.read()[:limit]
 647         if len(text) > 0:
 648             localized[key] = text
 649
 650
 651 def _set_author_entry(app, key, f):
 652     limit = config['char_limits']['author']
 653     with open(f) as fp:
 654         text = fp.read()[:limit]
 655         if len(text) > 0:
 656             app[key] = text
 657
 658
 659 def copy_triple_t_store_metadata(apps):
 660     """Include store metadata from the app's source repo
 661
 662     The Triple-T Gradle Play Publisher is a plugin that has a standard
 663     file layout for all of the metadata and graphics that the Google
 664     Play Store accepts.  Since F-Droid has the git repo, it can just
 665     pluck those files directly.  This method reads any text files into
 666     the app dict, then copies any graphics into the fdroid repo
 667     directory structure.
 668
 669     This needs to be run before insert_localized_app_metadata() so that
 670     the graphics files that are copied into the fdroid repo get
 671     properly indexed.
 672
 673     https://github.com/Triple-T/gradle-play-publisher#upload-images
 674     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 675
 676     """
 677
 678     if not os.path.isdir('build'):
 679         return  # nothing to do
 680
 681     for packageName, app in apps.items():
 682         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 683             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 684             for root, dirs, files in os.walk(d):
 685                 segments = root.split('/')
 686                 locale = segments[-2]
 687                 for f in files:
 688                     if f == 'fulldescription':
 689                         _set_localized_text_entry(app, locale, 'description',
 690                                                   os.path.join(root, f))
 691                         continue
 692                     elif f == 'shortdescription':
 693                         _set_localized_text_entry(app, locale, 'summary',
 694                                                   os.path.join(root, f))
 695                         continue
 696                     elif f == 'title':
 697                         _set_localized_text_entry(app, locale, 'name',
 698                                                   os.path.join(root, f))
 699                         continue
 700                     elif f == 'video':
 701                         _set_localized_text_entry(app, locale, 'video',
 702                                                   os.path.join(root, f))
 703                         continue
 704                     elif f == 'whatsnew':
 705                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 706                                                   os.path.join(root, f))
 707                         continue
 708                     elif f == 'contactEmail':
 709                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 710                         continue
 711                     elif f == 'contactPhone':
 712                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 713                         continue
 714                     elif f == 'contactWebsite':
 715                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 716                         continue
 717
 718                     base, extension = common.get_extension(f)
 719                     dirname = os.path.basename(root)
 720                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 721                         if segments[-2] == 'listing':
 722                             locale = segments[-3]
 723                         else:
 724                             locale = segments[-2]
 725                         destdir = os.path.join('repo', packageName, locale)
 726                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 727                         sourcefile = os.path.join(root, f)
 728                         destfile = os.path.join(destdir, dirname + '.' + extension)
 729                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 730                         shutil.copy(sourcefile, destfile)
 731
 732
 733 def insert_localized_app_metadata(apps):
 734     """scans standard locations for graphics and localized text
 735
 736     Scans for localized description files, store graphics, and
 737     screenshot PNG files in statically defined screenshots directory
 738     and adds them to the app metadata.  The screenshots and graphic
 739     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 740     and must be in the following layout:
 741     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 742
 743     repo/packageName/locale/featureGraphic.png
 744     repo/packageName/locale/phoneScreenshots/1.png
 745     repo/packageName/locale/phoneScreenshots/2.png
 746
 747     The changelog files must be text files named with the versionCode
 748     ending with ".txt" and must be in the following layout:
 749     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 750
 751     repo/packageName/locale/changelogs/12345.txt
 752
 753     This will scan the each app's source repo then the metadata/ dir
 754     for these standard locations of changelog files.  If it finds
 755     them, they will be added to the dict of all packages, with the
 756     versions in the metadata/ folder taking precendence over the what
 757     is in the app's source repo.
 758
 759     Where "packageName" is the app's packageName and "locale" is the locale
 760     of the graphics, e.g. what language they are in, using the IETF RFC5646
 761     format (en-US, fr-CA, es-MX, etc).
 762
 763     This will also scan the app's git for a fastlane folder, and the
 764     metadata/ folder and the apps' source repos for standard locations
 765     of graphic and screenshot files.  If it finds them, it will copy
 766     them into the repo.  The fastlane files follow this pattern:
 767     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 768
 769     """
 770
 771     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 772     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 773     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 774
 775     for srcd in sorted(sourcedirs):
 776         if not os.path.isdir(srcd):
 777             continue
 778         for root, dirs, files in os.walk(srcd):
 779             segments = root.split('/')
 780             packageName = segments[1]
 781             if packageName not in apps:
 782                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 783                 continue
 784             locale = segments[-1]
 785             destdir = os.path.join('repo', packageName, locale)
 786             for f in files:
 787                 if f in ('description.txt', 'full_description.txt'):
 788                     _set_localized_text_entry(apps[packageName], locale, 'description',
 789                                               os.path.join(root, f))
 790                     continue
 791                 elif f in ('summary.txt', 'short_description.txt'):
 792                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 793                                               os.path.join(root, f))
 794                     continue
 795                 elif f in ('name.txt', 'title.txt'):
 796                     _set_localized_text_entry(apps[packageName], locale, 'name',
 797                                               os.path.join(root, f))
 798                     continue
 799                 elif f == 'video.txt':
 800                     _set_localized_text_entry(apps[packageName], locale, 'video',
 801                                               os.path.join(root, f))
 802                     continue
 803                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 804                     locale = segments[-2]
 805                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 806                                               os.path.join(root, f))
 807                     continue
 808
 809                 base, extension = common.get_extension(f)
 810                 if locale == 'images':
 811                     locale = segments[-2]
 812                     destdir = os.path.join('repo', packageName, locale)
 813                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 814                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 815                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 816                     shutil.copy(os.path.join(root, f), destdir)
 817             for d in dirs:
 818                 if d in SCREENSHOT_DIRS:
 819                     for f in glob.glob(os.path.join(root, d, '*.*')):
 820                         _, extension = common.get_extension(f)
 821                         if extension in ALLOWED_EXTENSIONS:
 822                             screenshotdestdir = os.path.join(destdir, d)
 823                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 824                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 825                             shutil.copy(f, screenshotdestdir)
 826
 827     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 828     for d in repofiles:
 829         if not os.path.isdir(d):
 830             continue
 831         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 832             if not os.path.isfile(f):
 833                 continue
 834             segments = f.split('/')
 835             packageName = segments[1]
 836             locale = segments[2]
 837             screenshotdir = segments[3]
 838             filename = os.path.basename(f)
 839             base, extension = common.get_extension(filename)
 840
 841             if packageName not in apps:
 842                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 843                                 % (filename, packageName))
 844                 continue
 845             graphics = _get_localized_dict(apps[packageName], locale)
 846
 847             if extension not in ALLOWED_EXTENSIONS:
 848                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 849             elif base in GRAPHIC_NAMES:
 850                 # there can only be zero or one of these per locale
 851                 graphics[base] = filename
 852             elif screenshotdir in SCREENSHOT_DIRS:
 853                 # there can any number of these per locale
 854                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 855                 if screenshotdir not in graphics:
 856                     graphics[screenshotdir] = []
 857                 graphics[screenshotdir].append(filename)
 858             else:
 859                 logging.warning('Unsupported graphics file found: ' + f)
 860
 861
 862 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 863     """Scan a repo for all files with an extension except APK/OBB
 864
 865     :param apkcache: current cached info about all repo files
 866     :param repodir: repo directory to scan
 867     :param knownapks: list of all known files, as per metadata.read_metadata
 868     :param use_date_from_file: use date from file (instead of current date)
 869                                for newly added files
 870     """
 871
 872     cachechanged = False
 873     repo_files = []
 874     repodir = repodir.encode('utf-8')
 875     for name in os.listdir(repodir):
 876         file_extension = common.get_file_extension(name)
 877         if file_extension == 'apk' or file_extension == 'obb':
 878             continue
 879         filename = os.path.join(repodir, name)
 880         name_utf8 = name.decode('utf-8')
 881         if filename.endswith(b'_src.tar.gz'):
 882             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 883             continue
 884         if not common.is_repo_file(filename):
 885             continue
 886         stat = os.stat(filename)
 887         if stat.st_size == 0:
 888             raise FDroidException(filename + ' is zero size!')
 889
 890         shasum = sha256sum(filename)
 891         usecache = False
 892         if name in apkcache:
 893             repo_file = apkcache[name]
 894             # added time is cached as tuple but used here as datetime instance
 895             if 'added' in repo_file:
 896                 a = repo_file['added']
 897                 if isinstance(a, datetime):
 898                     repo_file['added'] = a
 899                 else:
 900                     repo_file['added'] = datetime(*a[:6])
 901             if repo_file.get('hash') == shasum:
 902                 logging.debug("Reading " + name_utf8 + " from cache")
 903                 usecache = True
 904             else:
 905                 logging.debug("Ignoring stale cache data for " + name_utf8)
 906
 907         if not usecache:
 908             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 909             repo_file = collections.OrderedDict()
 910             repo_file['name'] = os.path.splitext(name_utf8)[0]
 911             # TODO rename apkname globally to something more generic
 912             repo_file['apkName'] = name_utf8
 913             repo_file['hash'] = shasum
 914             repo_file['hashType'] = 'sha256'
 915             repo_file['versionCode'] = 0
 916             repo_file['versionName'] = shasum
 917             # the static ID is the SHA256 unless it is set in the metadata
 918             repo_file['packageName'] = shasum
 919
 920             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 921             if m:
 922                 repo_file['packageName'] = m.group(1)
 923                 repo_file['versionCode'] = int(m.group(2))
 924             srcfilename = name + b'_src.tar.gz'
 925             if os.path.exists(os.path.join(repodir, srcfilename)):
 926                 repo_file['srcname'] = srcfilename.decode('utf-8')
 927             repo_file['size'] = stat.st_size
 928
 929             apkcache[name] = repo_file
 930             cachechanged = True
 931
 932         if use_date_from_file:
 933             timestamp = stat.st_ctime
 934             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 935         else:
 936             default_date_param = None
 937
 938         # Record in knownapks, getting the added date at the same time..
 939         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 940                                     default_date=default_date_param)
 941         if added:
 942             repo_file['added'] = added
 943
 944         repo_files.append(repo_file)
 945
 946     return repo_files, cachechanged
 947
 948
 949 def scan_apk(apk_file):
 950     """
 951     Scans an APK file and returns dictionary with metadata of the APK.
 952
 953     Attention: This does *not* verify that the APK signature is correct.
 954
 955     :param apk_file: The (ideally absolute) path to the APK file
 956     :raises BuildException
 957     :return A dict containing APK metadata
 958     """
 959     apk = {
 960         'hash': sha256sum(apk_file),
 961         'hashType': 'sha256',
 962         'uses-permission': [],
 963         'uses-permission-sdk-23': [],
 964         'features': [],
 965         'icons_src': {},
 966         'icons': {},
 967         'antiFeatures': set(),
 968     }
 969
 970     if SdkToolsPopen(['aapt', 'version'], output=False):
 971         scan_apk_aapt(apk, apk_file)
 972     else:
 973         scan_apk_androguard(apk, apk_file)
 974
 975     # Get the signature, or rather the signing key fingerprints
 976     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 977     apk['sig'] = getsig(apk_file)
 978     if not apk['sig']:
 979         raise BuildException("Failed to get apk signature")
 980     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
 981                                                                apk_file))
 982     if not apk.get('signer'):
 983         raise BuildException("Failed to get apk signing key fingerprint")
 984
 985     # Get size of the APK
 986     apk['size'] = os.path.getsize(apk_file)
 987
 988     if 'minSdkVersion' not in apk:
 989         logging.warning("No SDK version information found in {0}".format(apk_file))
 990         apk['minSdkVersion'] = 1
 991
 992     # Check for known vulnerabilities
 993     if has_known_vulnerability(apk_file):
 994         apk['antiFeatures'].add('KnownVuln')
 995
 996     return apk
 997
 998
 999 def scan_apk_aapt(apk, apkfile):
1000     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1001     if p.returncode != 0:
1002         if options.delete_unknown:
1003             if os.path.exists(apkfile):
1004                 logging.error("Failed to get apk information, deleting " + apkfile)
1005                 os.remove(apkfile)
1006             else:
1007                 logging.error("Could not find {0} to remove it".format(apkfile))
1008         else:
1009             logging.error("Failed to get apk information, skipping " + apkfile)
1010         raise BuildException("Invalid APK")
1011     for line in p.output.splitlines():
1012         if line.startswith("package:"):
1013             try:
1014                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1015                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1016                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1017             except Exception as e:
1018                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1019         elif line.startswith("application:"):
1020             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1021             # Keep path to non-dpi icon in case we need it
1022             match = re.match(APK_ICON_PAT_NODPI, line)
1023             if match:
1024                 apk['icons_src']['-1'] = match.group(1)
1025         elif line.startswith("launchable-activity:"):
1026             # Only use launchable-activity as fallback to application
1027             if not apk['name']:
1028                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1029             if '-1' not in apk['icons_src']:
1030                 match = re.match(APK_ICON_PAT_NODPI, line)
1031                 if match:
1032                     apk['icons_src']['-1'] = match.group(1)
1033         elif line.startswith("application-icon-"):
1034             match = re.match(APK_ICON_PAT, line)
1035             if match:
1036                 density = match.group(1)
1037                 path = match.group(2)
1038                 apk['icons_src'][density] = path
1039         elif line.startswith("sdkVersion:"):
1040             m = re.match(APK_SDK_VERSION_PAT, line)
1041             if m is None:
1042                 logging.error(line.replace('sdkVersion:', '')
1043                               + ' is not a valid minSdkVersion!')
1044             else:
1045                 apk['minSdkVersion'] = m.group(1)
1046                 # if target not set, default to min
1047                 if 'targetSdkVersion' not in apk:
1048                     apk['targetSdkVersion'] = m.group(1)
1049         elif line.startswith("targetSdkVersion:"):
1050             m = re.match(APK_SDK_VERSION_PAT, line)
1051             if m is None:
1052                 logging.error(line.replace('targetSdkVersion:', '')
1053                               + ' is not a valid targetSdkVersion!')
1054             else:
1055                 apk['targetSdkVersion'] = m.group(1)
1056         elif line.startswith("maxSdkVersion:"):
1057             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1058         elif line.startswith("native-code:"):
1059             apk['nativecode'] = []
1060             for arch in line[13:].split(' '):
1061                 apk['nativecode'].append(arch[1:-1])
1062         elif line.startswith('uses-permission:'):
1063             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1064             if perm_match['maxSdkVersion']:
1065                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1066             permission = UsesPermission(
1067                 perm_match['name'],
1068                 perm_match['maxSdkVersion']
1069             )
1070
1071             apk['uses-permission'].append(permission)
1072         elif line.startswith('uses-permission-sdk-23:'):
1073             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1074             if perm_match['maxSdkVersion']:
1075                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1076             permission_sdk_23 = UsesPermissionSdk23(
1077                 perm_match['name'],
1078                 perm_match['maxSdkVersion']
1079             )
1080
1081             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1082
1083         elif line.startswith('uses-feature:'):
1084             feature = re.match(APK_FEATURE_PAT, line).group(1)
1085             # Filter out this, it's only added with the latest SDK tools and
1086             # causes problems for lots of apps.
1087             if feature != "android.hardware.screen.portrait" \
1088                     and feature != "android.hardware.screen.landscape":
1089                 if feature.startswith("android.feature."):
1090                     feature = feature[16:]
1091                 apk['features'].add(feature)
1092
1093
1094 def scan_apk_androguard(apk, apkfile):
1095     try:
1096         from androguard.core.bytecodes.apk import APK
1097         apkobject = APK(apkfile)
1098         if apkobject.is_valid_APK():
1099             arsc = apkobject.get_android_resources()
1100         else:
1101             if options.delete_unknown:
1102                 if os.path.exists(apkfile):
1103                     logging.error("Failed to get apk information, deleting " + apkfile)
1104                     os.remove(apkfile)
1105                 else:
1106                     logging.error("Could not find {0} to remove it".format(apkfile))
1107             else:
1108                 logging.error("Failed to get apk information, skipping " + apkfile)
1109             raise BuildException("Invaild APK")
1110     except ImportError:
1111         raise FDroidException("androguard library is not installed and aapt not present")
1112     except FileNotFoundError:
1113         logging.error("Could not open apk file for analysis")
1114         raise BuildException("Invalid APK")
1115
1116     apk['packageName'] = apkobject.get_package()
1117     apk['versionCode'] = int(apkobject.get_androidversion_code())
1118     apk['versionName'] = apkobject.get_androidversion_name()
1119     if apk['versionName'][0] == "@":
1120         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1121         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1122         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1123     apk['name'] = apkobject.get_app_name()
1124
1125     if apkobject.get_max_sdk_version() is not None:
1126         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1127     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1128     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1129
1130     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1131     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1132
1133     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1134
1135     for file in apkobject.get_files():
1136         d_re = density_re.match(file)
1137         if d_re:
1138             folder = d_re.group(1).split('-')
1139             if len(folder) > 1:
1140                 resolution = folder[1]
1141             else:
1142                 resolution = 'mdpi'
1143             density = screen_resolutions[resolution]
1144             apk['icons_src'][density] = d_re.group(0)
1145
1146     if apk['icons_src'].get('-1') is None:
1147         apk['icons_src']['-1'] = apk['icons_src']['160']
1148
1149     arch_re = re.compile("^lib/(.*)/.*$")
1150     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1151     if len(arch) >= 1:
1152         apk['nativecode'] = []
1153         apk['nativecode'].extend(sorted(list(arch)))
1154
1155     xml = apkobject.get_android_manifest_xml()
1156
1157     for item in xml.getElementsByTagName('uses-permission'):
1158         name = str(item.getAttribute("android:name"))
1159         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1160         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1161         permission = UsesPermission(
1162             name,
1163             maxSdkVersion
1164         )
1165         apk['uses-permission'].append(permission)
1166
1167     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1168         name = str(item.getAttribute("android:name"))
1169         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1170         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1171         permission_sdk_23 = UsesPermissionSdk23(
1172             name,
1173             maxSdkVersion
1174         )
1175         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1176
1177     for item in xml.getElementsByTagName('uses-feature'):
1178         feature = str(item.getAttribute("android:name"))
1179         if feature != "android.hardware.screen.portrait" \
1180                 and feature != "android.hardware.screen.landscape":
1181             if feature.startswith("android.feature."):
1182                 feature = feature[16:]
1183         apk['features'].append(feature)
1184
1185
1186 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1187                 allow_disabled_algorithms=False, archive_bad_sig=False):
1188     """Processes the apk with the given filename in the given repo directory.
1189
1190     This also extracts the icons.
1191
1192     :param apkcache: current apk cache information
1193     :param apkfilename: the filename of the apk to scan
1194     :param repodir: repo directory to scan
1195     :param knownapks: known apks info
1196     :param use_date_from_apk: use date from APK (instead of current date)
1197                               for newly added APKs
1198     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1199                                       disabled algorithms in the signature (e.g. MD5)
1200     :param archive_bad_sig: move APKs with a bad signature to the archive
1201     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1202      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1203     """
1204
1205     apk = {}
1206     apkfile = os.path.join(repodir, apkfilename)
1207
1208     cachechanged = False
1209     usecache = False
1210     if apkfilename in apkcache:
1211         apk = apkcache[apkfilename]
1212         if apk.get('hash') == sha256sum(apkfile):
1213             logging.debug("Reading " + apkfilename + " from cache")
1214             usecache = True
1215         else:
1216             logging.debug("Ignoring stale cache data for " + apkfilename)
1217
1218     if not usecache:
1219         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1220
1221         try:
1222             apk = scan_apk(apkfile)
1223         except BuildException:
1224             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1225                             .format(apkfilename=apkfilename))
1226             return True, None, False
1227
1228         # Check for debuggable apks...
1229         if common.isApkAndDebuggable(apkfile):
1230             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1231
1232         if options.rename_apks:
1233             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1234             std_short_name = os.path.join(repodir, n)
1235             if apkfile != std_short_name:
1236                 if os.path.exists(std_short_name):
1237                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1238                     if apkfile != std_long_name:
1239                         if os.path.exists(std_long_name):
1240                             dupdir = os.path.join('duplicates', repodir)
1241                             if not os.path.isdir(dupdir):
1242                                 os.makedirs(dupdir, exist_ok=True)
1243                             dupfile = os.path.join('duplicates', std_long_name)
1244                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1245                             os.rename(apkfile, dupfile)
1246                             return True, None, False
1247                         else:
1248                             os.rename(apkfile, std_long_name)
1249                     apkfile = std_long_name
1250                 else:
1251                     os.rename(apkfile, std_short_name)
1252                     apkfile = std_short_name
1253                 apkfilename = apkfile[len(repodir) + 1:]
1254
1255         apk['apkName'] = apkfilename
1256         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1257         if os.path.exists(os.path.join(repodir, srcfilename)):
1258             apk['srcname'] = srcfilename
1259
1260         # verify the jar signature is correct, allow deprecated
1261         # algorithms only if the APK is in the archive.
1262         skipapk = False
1263         if not common.verify_apk_signature(apkfile):
1264             if repodir == 'archive' or allow_disabled_algorithms:
1265                 if common.verify_old_apk_signature(apkfile):
1266                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1267                 else:
1268                     skipapk = True
1269             else:
1270                 skipapk = True
1271
1272         if skipapk:
1273             if archive_bad_sig:
1274                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1275                 move_apk_between_sections(repodir, 'archive', apk)
1276             else:
1277                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1278             return True, None, False
1279
1280         apkzip = zipfile.ZipFile(apkfile, 'r')
1281
1282         # if an APK has files newer than the system time, suggest updating
1283         # the system clock.  This is useful for offline systems, used for
1284         # signing, which do not have another source of clock sync info. It
1285         # has to be more than 24 hours newer because ZIP/APK files do not
1286         # store timezone info
1287         manifest = apkzip.getinfo('AndroidManifest.xml')
1288         if manifest.date_time[1] == 0:  # month can't be zero
1289             logging.debug('AndroidManifest.xml has no date')
1290         else:
1291             dt_obj = datetime(*manifest.date_time)
1292             checkdt = dt_obj - timedelta(1)
1293             if datetime.today() < checkdt:
1294                 logging.warning('System clock is older than manifest in: '
1295                                 + apkfilename
1296                                 + '\nSet clock to that time using:\n'
1297                                 + 'sudo date -s "' + str(dt_obj) + '"')
1298
1299         # extract icons from APK zip file
1300         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1301         try:
1302             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1303         finally:
1304             apkzip.close()  # ensure that APK zip file gets closed
1305
1306         # resize existing icons for densities missing in the APK
1307         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1308
1309         if use_date_from_apk and manifest.date_time[1] != 0:
1310             default_date_param = datetime(*manifest.date_time)
1311         else:
1312             default_date_param = None
1313
1314         # Record in known apks, getting the added date at the same time..
1315         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1316                                     default_date=default_date_param)
1317         if added:
1318             apk['added'] = added
1319
1320         apkcache[apkfilename] = apk
1321         cachechanged = True
1322
1323     return False, apk, cachechanged
1324
1325
1326 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1327     """Processes the apks in the given repo directory.
1328
1329     This also extracts the icons.
1330
1331     :param apkcache: current apk cache information
1332     :param repodir: repo directory to scan
1333     :param knownapks: known apks info
1334     :param use_date_from_apk: use date from APK (instead of current date)
1335                               for newly added APKs
1336     :returns: (apks, cachechanged) where apks is a list of apk information,
1337               and cachechanged is True if the apkcache got changed.
1338     """
1339
1340     cachechanged = False
1341
1342     for icon_dir in get_all_icon_dirs(repodir):
1343         if os.path.exists(icon_dir):
1344             if options.clean:
1345                 shutil.rmtree(icon_dir)
1346                 os.makedirs(icon_dir)
1347         else:
1348             os.makedirs(icon_dir)
1349
1350     apks = []
1351     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1352         apkfilename = apkfile[len(repodir) + 1:]
1353         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1354         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1355                                              use_date_from_apk, ada, True)
1356         if skip:
1357             continue
1358         apks.append(apk)
1359         cachechanged = cachechanged or cachethis
1360
1361     return apks, cachechanged
1362
1363
1364 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1365     """
1366     Extracts icons from the given APK zip in various densities,
1367     saves them into given repo directory
1368     and stores their names in the APK metadata dictionary.
1369
1370     :param icon_filename: A string representing the icon's file name
1371     :param apk: A populated dictionary containing APK metadata.
1372                 Needs to have 'icons_src' key
1373     :param apkzip: An opened zipfile.ZipFile of the APK file
1374     :param repo_dir: The directory of the APK's repository
1375     :return: A list of icon densities that are missing
1376     """
1377     empty_densities = []
1378     for density in screen_densities:
1379         if density not in apk['icons_src']:
1380             empty_densities.append(density)
1381             continue
1382         icon_src = apk['icons_src'][density]
1383         icon_dir = get_icon_dir(repo_dir, density)
1384         icon_dest = os.path.join(icon_dir, icon_filename)
1385
1386         # Extract the icon files per density
1387         if icon_src.endswith('.xml'):
1388             png = os.path.basename(icon_src)[:-4] + '.png'
1389             for f in apkzip.namelist():
1390                 if f.endswith(png):
1391                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1392                     if m and screen_resolutions[m.group(2)] == density:
1393                         icon_src = f
1394             if icon_src.endswith('.xml'):
1395                 empty_densities.append(density)
1396                 continue
1397         try:
1398             with open(icon_dest, 'wb') as f:
1399                 f.write(get_icon_bytes(apkzip, icon_src))
1400             apk['icons'][density] = icon_filename
1401         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1402             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1403             del apk['icons_src'][density]
1404             empty_densities.append(density)
1405
1406     if '-1' in apk['icons_src']:
1407         icon_src = apk['icons_src']['-1']
1408         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1409         with open(icon_path, 'wb') as f:
1410             f.write(get_icon_bytes(apkzip, icon_src))
1411         try:
1412             im = Image.open(icon_path)
1413             dpi = px_to_dpi(im.size[0])
1414             for density in screen_densities:
1415                 if density in apk['icons']:
1416                     break
1417                 if density == screen_densities[-1] or dpi >= int(density):
1418                     apk['icons'][density] = icon_filename
1419                     shutil.move(icon_path,
1420                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1421                     empty_densities.remove(density)
1422                     break
1423         except Exception as e:
1424             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1425
1426     if apk['icons']:
1427         apk['icon'] = icon_filename
1428
1429     return empty_densities
1430
1431
1432 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1433     """
1434     Resize existing icons for densities missing in the APK to ensure all densities are available
1435
1436     :param empty_densities: A list of icon densities that are missing
1437     :param icon_filename: A string representing the icon's file name
1438     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1439     :param repo_dir: The directory of the APK's repository
1440     """
1441     # First try resizing down to not lose quality
1442     last_density = None
1443     for density in screen_densities:
1444         if density not in empty_densities:
1445             last_density = density
1446             continue
1447         if last_density is None:
1448             continue
1449         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1450
1451         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1452         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1453         fp = None
1454         try:
1455             fp = open(last_icon_path, 'rb')
1456             im = Image.open(fp)
1457
1458             size = dpi_to_px(density)
1459
1460             im.thumbnail((size, size), Image.ANTIALIAS)
1461             im.save(icon_path, "PNG")
1462             empty_densities.remove(density)
1463         except Exception as e:
1464             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1465         finally:
1466             if fp:
1467                 fp.close()
1468
1469     # Then just copy from the highest resolution available
1470     last_density = None
1471     for density in reversed(screen_densities):
1472         if density not in empty_densities:
1473             last_density = density
1474             continue
1475
1476         if last_density is None:
1477             continue
1478
1479         shutil.copyfile(
1480             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1481             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1482         )
1483         empty_densities.remove(density)
1484
1485     for density in screen_densities:
1486         icon_dir = get_icon_dir(repo_dir, density)
1487         icon_dest = os.path.join(icon_dir, icon_filename)
1488         resize_icon(icon_dest, density)
1489
1490     # Copy from icons-mdpi to icons since mdpi is the baseline density
1491     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1492     if os.path.isfile(baseline):
1493         apk['icons']['0'] = icon_filename
1494         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1495
1496
1497 def apply_info_from_latest_apk(apps, apks):
1498     """
1499     Some information from the apks needs to be applied up to the application level.
1500     When doing this, we use the info from the most recent version's apk.
1501     We deal with figuring out when the app was added and last updated at the same time.
1502     """
1503     for appid, app in apps.items():
1504         bestver = UNSET_VERSION_CODE
1505         for apk in apks:
1506             if apk['packageName'] == appid:
1507                 if apk['versionCode'] > bestver:
1508                     bestver = apk['versionCode']
1509                     bestapk = apk
1510
1511                 if 'added' in apk:
1512                     if not app.added or apk['added'] < app.added:
1513                         app.added = apk['added']
1514                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1515                         app.lastUpdated = apk['added']
1516
1517         if not app.added:
1518             logging.debug("Don't know when " + appid + " was added")
1519         if not app.lastUpdated:
1520             logging.debug("Don't know when " + appid + " was last updated")
1521
1522         if bestver == UNSET_VERSION_CODE:
1523
1524             if app.Name is None:
1525                 app.Name = app.AutoName or appid
1526             app.icon = None
1527             logging.debug("Application " + appid + " has no packages")
1528         else:
1529             if app.Name is None:
1530                 app.Name = bestapk['name']
1531             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1532             if app.CurrentVersionCode is None:
1533                 app.CurrentVersionCode = str(bestver)
1534
1535
1536 def make_categories_txt(repodir, categories):
1537     '''Write a category list in the repo to allow quick access'''
1538     catdata = ''
1539     for cat in sorted(categories):
1540         catdata += cat + '\n'
1541     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1542         f.write(catdata)
1543
1544
1545 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1546
1547     def filter_apk_list_sorted(apk_list):
1548         res = []
1549         for apk in apk_list:
1550             if apk['packageName'] == appid:
1551                 res.append(apk)
1552
1553         # Sort the apk list by version code. First is highest/newest.
1554         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1555
1556     for appid, app in apps.items():
1557
1558         if app.ArchivePolicy:
1559             keepversions = int(app.ArchivePolicy[:-9])
1560         else:
1561             keepversions = defaultkeepversions
1562
1563         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1564                       .format(appid, len(apks), keepversions, len(archapks)))
1565
1566         current_app_apks = filter_apk_list_sorted(apks)
1567         if len(current_app_apks) > keepversions:
1568             # Move back the ones we don't want.
1569             for apk in current_app_apks[keepversions:]:
1570                 move_apk_between_sections(repodir, archivedir, apk)
1571                 archapks.append(apk)
1572                 apks.remove(apk)
1573
1574         current_app_archapks = filter_apk_list_sorted(archapks)
1575         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1576             kept = 0
1577             # Move forward the ones we want again, except DisableAlgorithm
1578             for apk in current_app_archapks:
1579                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1580                     move_apk_between_sections(archivedir, repodir, apk)
1581                     archapks.remove(apk)
1582                     apks.append(apk)
1583                     kept += 1
1584                 if kept == keepversions:
1585                     break
1586
1587
1588 def move_apk_between_sections(from_dir, to_dir, apk):
1589     """move an APK from repo to archive or vice versa"""
1590
1591     def _move_file(from_dir, to_dir, filename, ignore_missing):
1592         from_path = os.path.join(from_dir, filename)
1593         if ignore_missing and not os.path.exists(from_path):
1594             return
1595         to_path = os.path.join(to_dir, filename)
1596         if not os.path.exists(to_dir):
1597             os.mkdir(to_dir)
1598         shutil.move(from_path, to_path)
1599
1600     if from_dir == to_dir:
1601         return
1602
1603     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1604     _move_file(from_dir, to_dir, apk['apkName'], False)
1605     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1606     for density in all_screen_densities:
1607         from_icon_dir = get_icon_dir(from_dir, density)
1608         to_icon_dir = get_icon_dir(to_dir, density)
1609         if density not in apk['icons']:
1610             continue
1611         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1612     if 'srcname' in apk:
1613         _move_file(from_dir, to_dir, apk['srcname'], False)
1614
1615
1616 def add_apks_to_per_app_repos(repodir, apks):
1617     apks_per_app = dict()
1618     for apk in apks:
1619         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1620         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1621         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1622         apks_per_app[apk['packageName']] = apk
1623
1624         if not os.path.exists(apk['per_app_icons']):
1625             logging.info('Adding new repo for only ' + apk['packageName'])
1626             os.makedirs(apk['per_app_icons'])
1627
1628         apkpath = os.path.join(repodir, apk['apkName'])
1629         shutil.copy(apkpath, apk['per_app_repo'])
1630         apksigpath = apkpath + '.sig'
1631         if os.path.exists(apksigpath):
1632             shutil.copy(apksigpath, apk['per_app_repo'])
1633         apkascpath = apkpath + '.asc'
1634         if os.path.exists(apkascpath):
1635             shutil.copy(apkascpath, apk['per_app_repo'])
1636
1637
1638 def create_metadata_from_template(apk):
1639     '''create a new metadata file using internal or external template
1640
1641     Generate warnings for apk's with no metadata (or create skeleton
1642     metadata files, if requested on the command line).  Though the
1643     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1644     since those impose things on the metadata file made from the
1645     template: field sort order, empty field value, formatting, etc.
1646     '''
1647
1648     import yaml
1649     if os.path.exists('template.yml'):
1650         with open('template.yml') as f:
1651             metatxt = f.read()
1652         if 'name' in apk and apk['name'] != '':
1653             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1654                              r'\1 ' + apk['name'],
1655                              metatxt,
1656                              flags=re.IGNORECASE | re.MULTILINE)
1657         else:
1658             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1659             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1660                              r'\1 ' + apk['packageName'],
1661                              metatxt,
1662                              flags=re.IGNORECASE | re.MULTILINE)
1663         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1664             f.write(metatxt)
1665     else:
1666         app = dict()
1667         app['Categories'] = [os.path.basename(os.getcwd())]
1668         # include some blanks as part of the template
1669         app['AuthorName'] = ''
1670         app['Summary'] = ''
1671         app['WebSite'] = ''
1672         app['IssueTracker'] = ''
1673         app['SourceCode'] = ''
1674         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1675         if 'name' in apk and apk['name'] != '':
1676             app['Name'] = apk['name']
1677         else:
1678             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1679             app['Name'] = apk['packageName']
1680         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1681             yaml.dump(app, f, default_flow_style=False)
1682     logging.info("Generated skeleton metadata for " + apk['packageName'])
1683
1684
1685 config = None
1686 options = None
1687
1688
1689 def main():
1690
1691     global config, options
1692
1693     # Parse command line...
1694     parser = ArgumentParser()
1695     common.setup_global_opts(parser)
1696     parser.add_argument("--create-key", action="store_true", default=False,
1697                         help=_("Create a repo signing key in a keystore"))
1698     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1699                         help=_("Create skeleton metadata files that are missing"))
1700     parser.add_argument("--delete-unknown", action="store_true", default=False,
1701                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1702     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1703                         help=_("Report on build data status"))
1704     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1705                         help=_("Interactively ask about things that need updating."))
1706     parser.add_argument("-I", "--icons", action="store_true", default=False,
1707                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1708     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1709                         help=_("Specify editor to use in interactive mode. Default ") +
1710                         "is /etc/alternatives/editor")
1711     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1712                         help=_("Update the wiki"))
1713     parser.add_argument("--pretty", action="store_true", default=False,
1714                         help=_("Produce human-readable index.xml"))
1715     parser.add_argument("--clean", action="store_true", default=False,
1716                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1717     parser.add_argument("--nosign", action="store_true", default=False,
1718                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1719     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1720                         help=_("Use date from APK instead of current time for newly added APKs"))
1721     parser.add_argument("--rename-apks", action="store_true", default=False,
1722                         help=_("Rename APK files that do not match package.name_123.apk"))
1723     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1724                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1725     metadata.add_metadata_arguments(parser)
1726     options = parser.parse_args()
1727     metadata.warnings_action = options.W
1728
1729     config = common.read_config(options)
1730
1731     if not ('jarsigner' in config and 'keytool' in config):
1732         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1733
1734     repodirs = ['repo']
1735     if config['archive_older'] != 0:
1736         repodirs.append('archive')
1737         if not os.path.exists('archive'):
1738             os.mkdir('archive')
1739
1740     if options.icons:
1741         resize_all_icons(repodirs)
1742         sys.exit(0)
1743
1744     if options.rename_apks:
1745         options.clean = True
1746
1747     # check that icons exist now, rather than fail at the end of `fdroid update`
1748     for k in ['repo_icon', 'archive_icon']:
1749         if k in config:
1750             if not os.path.exists(config[k]):
1751                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1752                 sys.exit(1)
1753
1754     # if the user asks to create a keystore, do it now, reusing whatever it can
1755     if options.create_key:
1756         if os.path.exists(config['keystore']):
1757             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1758             logging.critical("\t'" + config['keystore'] + "'")
1759             sys.exit(1)
1760
1761         if 'repo_keyalias' not in config:
1762             config['repo_keyalias'] = socket.getfqdn()
1763             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1764         if 'keydname' not in config:
1765             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1766             common.write_to_config(config, 'keydname', config['keydname'])
1767         if 'keystore' not in config:
1768             config['keystore'] = common.default_config['keystore']
1769             common.write_to_config(config, 'keystore', config['keystore'])
1770
1771         password = common.genpassword()
1772         if 'keystorepass' not in config:
1773             config['keystorepass'] = password
1774             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1775         if 'keypass' not in config:
1776             config['keypass'] = password
1777             common.write_to_config(config, 'keypass', config['keypass'])
1778         common.genkeystore(config)
1779
1780     # Get all apps...
1781     apps = metadata.read_metadata()
1782
1783     # Generate a list of categories...
1784     categories = set()
1785     for app in apps.values():
1786         categories.update(app.Categories)
1787
1788     # Read known apks data (will be updated and written back when we've finished)
1789     knownapks = common.KnownApks()
1790
1791     # Get APK cache
1792     apkcache = get_cache()
1793
1794     # Delete builds for disabled apps
1795     delete_disabled_builds(apps, apkcache, repodirs)
1796
1797     # Scan all apks in the main repo
1798     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1799
1800     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1801                                            options.use_date_from_apk)
1802     cachechanged = cachechanged or fcachechanged
1803     apks += files
1804     for apk in apks:
1805         if apk['packageName'] not in apps:
1806             if options.create_metadata:
1807                 create_metadata_from_template(apk)
1808                 apps = metadata.read_metadata()
1809             else:
1810                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1811                 if options.delete_unknown:
1812                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1813                     rmf = os.path.join(repodirs[0], apk['apkName'])
1814                     if not os.path.exists(rmf):
1815                         logging.error("Could not find {0} to remove it".format(rmf))
1816                     else:
1817                         os.remove(rmf)
1818                 else:
1819                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1820
1821     copy_triple_t_store_metadata(apps)
1822     insert_obbs(repodirs[0], apps, apks)
1823     insert_localized_app_metadata(apps)
1824     translate_per_build_anti_features(apps, apks)
1825
1826     # Scan the archive repo for apks as well
1827     if len(repodirs) > 1:
1828         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1829         if cc:
1830             cachechanged = True
1831     else:
1832         archapks = []
1833
1834     # Apply information from latest apks to the application and update dates
1835     apply_info_from_latest_apk(apps, apks + archapks)
1836
1837     # Sort the app list by name, then the web site doesn't have to by default.
1838     # (we had to wait until we'd scanned the apks to do this, because mostly the
1839     # name comes from there!)
1840     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1841
1842     # APKs are placed into multiple repos based on the app package, providing
1843     # per-app subscription feeds for nightly builds and things like it
1844     if config['per_app_repos']:
1845         add_apks_to_per_app_repos(repodirs[0], apks)
1846         for appid, app in apps.items():
1847             repodir = os.path.join(appid, 'fdroid', 'repo')
1848             appdict = dict()
1849             appdict[appid] = app
1850             if os.path.isdir(repodir):
1851                 index.make(appdict, [appid], apks, repodir, False)
1852             else:
1853                 logging.info('Skipping index generation for ' + appid)
1854         return
1855
1856     if len(repodirs) > 1:
1857         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1858
1859     # Make the index for the main repo...
1860     index.make(apps, sortedids, apks, repodirs[0], False)
1861     make_categories_txt(repodirs[0], categories)
1862
1863     # If there's an archive repo,  make the index for it. We already scanned it
1864     # earlier on.
1865     if len(repodirs) > 1:
1866         index.make(apps, sortedids, archapks, repodirs[1], True)
1867
1868     git_remote = config.get('binary_transparency_remote')
1869     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1870         from . import btlog
1871         btlog.make_binary_transparency_log(repodirs)
1872
1873     if config['update_stats']:
1874         # Update known apks info...
1875         knownapks.writeifchanged()
1876
1877         # Generate latest apps data for widget
1878         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1879             data = ''
1880             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1881                 for line in f:
1882                     appid = line.rstrip()
1883                     data += appid + "\t"
1884                     app = apps[appid]
1885                     data += app.Name + "\t"
1886                     if app.icon is not None:
1887                         data += app.icon + "\t"
1888                     data += app.License + "\n"
1889             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1890                 f.write(data)
1891
1892     if cachechanged:
1893         write_cache(apkcache)
1894
1895     # Update the wiki...
1896     if options.wiki:
1897         update_wiki(apps, sortedids, apks + archapks)
1898
1899     logging.info(_("Finished"))
1900
1901
1902 if __name__ == "__main__":
1903     main()