fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import _
  41 from . import common
  42 from . import index
  43 from . import metadata
  44 from .common import SdkToolsPopen
  45 from .exception import BuildException, FDroidException
  46
  47 METADATA_VERSION = 18
  48
  49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  50 UNSET_VERSION_CODE = -0x100000000
  51
  52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  59 APK_PERMISSION_PAT = \
  60     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  62
  63 screen_densities = ['640', '480', '320', '240', '160', '120']
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "undefined": '-1',
  72     "anydpi": '65534',
  73     "nodpi": '65535'
  74 }
  75
  76 all_screen_densities = ['0'] + screen_densities
  77
  78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  80
  81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  84                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  85
  86
  87 def dpi_to_px(density):
  88     return (int(density) * 48) / 160
  89
  90
  91 def px_to_dpi(px):
  92     return (int(px) * 160) / 48
  93
  94
  95 def get_icon_dir(repodir, density):
  96     if density == '0':
  97         return os.path.join(repodir, "icons")
  98     return os.path.join(repodir, "icons-%s" % density)
  99
 100
 101 def get_icon_dirs(repodir):
 102     for density in screen_densities:
 103         yield get_icon_dir(repodir, density)
 104
 105
 106 def get_all_icon_dirs(repodir):
 107     for density in all_screen_densities:
 108         yield get_icon_dir(repodir, density)
 109
 110
 111 def update_wiki(apps, sortedids, apks):
 112     """Update the wiki
 113
 114     :param apps: fully populated list of all applications
 115     :param apks: all apks, except...
 116     """
 117     logging.info("Updating wiki")
 118     wikicat = 'Apps'
 119     wikiredircat = 'App Redirects'
 120     import mwclient
 121     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 122                          path=config['wiki_path'])
 123     site.login(config['wiki_user'], config['wiki_password'])
 124     generated_pages = {}
 125     generated_redirects = {}
 126
 127     for appid in sortedids:
 128         app = metadata.App(apps[appid])
 129
 130         wikidata = ''
 131         if app.Disabled:
 132             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 133         if app.AntiFeatures:
 134             for af in app.AntiFeatures:
 135                 wikidata += '{{AntiFeature|' + af + '}}\n'
 136         if app.RequiresRoot:
 137             requiresroot = 'Yes'
 138         else:
 139             requiresroot = 'No'
 140         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 141             appid,
 142             app.Name,
 143             app.added.strftime('%Y-%m-%d') if app.added else '',
 144             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 145             app.SourceCode,
 146             app.IssueTracker,
 147             app.WebSite,
 148             app.Changelog,
 149             app.Donate,
 150             app.FlattrID,
 151             app.Bitcoin,
 152             app.Litecoin,
 153             app.License,
 154             requiresroot,
 155             app.AuthorName,
 156             app.AuthorEmail)
 157
 158         if app.Provides:
 159             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 160
 161         wikidata += app.Summary
 162         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 163
 164         wikidata += "=Description=\n"
 165         wikidata += metadata.description_wiki(app.Description) + "\n"
 166
 167         wikidata += "=Maintainer Notes=\n"
 168         if app.MaintainerNotes:
 169             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 170         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 171
 172         # Get a list of all packages for this application...
 173         apklist = []
 174         gotcurrentver = False
 175         cantupdate = False
 176         buildfails = False
 177         for apk in apks:
 178             if apk['packageName'] == appid:
 179                 if str(apk['versionCode']) == app.CurrentVersionCode:
 180                     gotcurrentver = True
 181                 apklist.append(apk)
 182         # Include ones we can't build, as a special case...
 183         for build in app.builds:
 184             if build.disable:
 185                 if build.versionCode == app.CurrentVersionCode:
 186                     cantupdate = True
 187                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 188                 apklist.append({'versionCode': int(build.versionCode),
 189                                 'versionName': build.versionName,
 190                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 191                                 })
 192             else:
 193                 builtit = False
 194                 for apk in apklist:
 195                     if apk['versionCode'] == int(build.versionCode):
 196                         builtit = True
 197                         break
 198                 if not builtit:
 199                     buildfails = True
 200                     apklist.append({'versionCode': int(build.versionCode),
 201                                     'versionName': build.versionName,
 202                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 203                                     })
 204         if app.CurrentVersionCode == '0':
 205             cantupdate = True
 206         # Sort with most recent first...
 207         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 208
 209         wikidata += "=Versions=\n"
 210         if len(apklist) == 0:
 211             wikidata += "We currently have no versions of this app available."
 212         elif not gotcurrentver:
 213             wikidata += "We don't have the current version of this app."
 214         else:
 215             wikidata += "We have the current version of this app."
 216         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 217         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 218         if len(app.NoSourceSince) > 0:
 219             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 220         if len(app.CurrentVersion) > 0:
 221             wikidata += "The current (recommended) version is " + app.CurrentVersion
 222             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 223         validapks = 0
 224         for apk in apklist:
 225             wikidata += "==" + apk['versionName'] + "==\n"
 226
 227             if 'buildproblem' in apk:
 228                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 229             else:
 230                 validapks += 1
 231                 wikidata += "This version is built and signed by "
 232                 if 'srcname' in apk:
 233                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 234                 else:
 235                     wikidata += "the original developer.\n\n"
 236             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 237
 238         wikidata += '\n[[Category:' + wikicat + ']]\n'
 239         if len(app.NoSourceSince) > 0:
 240             wikidata += '\n[[Category:Apps missing source code]]\n'
 241         if validapks == 0 and not app.Disabled:
 242             wikidata += '\n[[Category:Apps with no packages]]\n'
 243         if cantupdate and not app.Disabled:
 244             wikidata += "\n[[Category:Apps we cannot update]]\n"
 245         if buildfails and not app.Disabled:
 246             wikidata += "\n[[Category:Apps with failing builds]]\n"
 247         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 248             wikidata += '\n[[Category:Apps to Update]]\n'
 249         if app.Disabled:
 250             wikidata += '\n[[Category:Apps that are disabled]]\n'
 251         if app.UpdateCheckMode == 'None' and not app.Disabled:
 252             wikidata += '\n[[Category:Apps with no update check]]\n'
 253         for appcat in app.Categories:
 254             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 255
 256         # We can't have underscores in the page name, even if they're in
 257         # the package ID, because MediaWiki messes with them...
 258         pagename = appid.replace('_', ' ')
 259
 260         # Drop a trailing newline, because mediawiki is going to drop it anyway
 261         # and it we don't we'll think the page has changed when it hasn't...
 262         if wikidata.endswith('\n'):
 263             wikidata = wikidata[:-1]
 264
 265         generated_pages[pagename] = wikidata
 266
 267         # Make a redirect from the name to the ID too, unless there's
 268         # already an existing page with the name and it isn't a redirect.
 269         noclobber = False
 270         apppagename = app.Name.replace('_', ' ')
 271         apppagename = apppagename.replace('{', '')
 272         apppagename = apppagename.replace('}', ' ')
 273         apppagename = apppagename.replace(':', ' ')
 274         apppagename = apppagename.replace('[', ' ')
 275         apppagename = apppagename.replace(']', ' ')
 276         # Drop double spaces caused mostly by replacing ':' above
 277         apppagename = apppagename.replace('  ', ' ')
 278         for expagename in site.allpages(prefix=apppagename,
 279                                         filterredir='nonredirects',
 280                                         generator=False):
 281             if expagename == apppagename:
 282                 noclobber = True
 283         # Another reason not to make the redirect page is if the app name
 284         # is the same as it's ID, because that will overwrite the real page
 285         # with an redirect to itself! (Although it seems like an odd
 286         # scenario this happens a lot, e.g. where there is metadata but no
 287         # builds or binaries to extract a name from.
 288         if apppagename == pagename:
 289             noclobber = True
 290         if not noclobber:
 291             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 292
 293     for tcat, genp in [(wikicat, generated_pages),
 294                        (wikiredircat, generated_redirects)]:
 295         catpages = site.Pages['Category:' + tcat]
 296         existingpages = []
 297         for page in catpages:
 298             existingpages.append(page.name)
 299             if page.name in genp:
 300                 pagetxt = page.edit()
 301                 if pagetxt != genp[page.name]:
 302                     logging.debug("Updating modified page " + page.name)
 303                     page.save(genp[page.name], summary='Auto-updated')
 304                 else:
 305                     logging.debug("Page " + page.name + " is unchanged")
 306             else:
 307                 logging.warn("Deleting page " + page.name)
 308                 page.delete('No longer published')
 309         for pagename, text in genp.items():
 310             logging.debug("Checking " + pagename)
 311             if pagename not in existingpages:
 312                 logging.debug("Creating page " + pagename)
 313                 try:
 314                     newpage = site.Pages[pagename]
 315                     newpage.save(text, summary='Auto-created')
 316                 except Exception as e:
 317                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 318
 319     # Purge server cache to ensure counts are up to date
 320     site.pages['Repository Maintenance'].purge()
 321
 322
 323 def delete_disabled_builds(apps, apkcache, repodirs):
 324     """Delete disabled build outputs.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodirs: the repo directories to process
 329     """
 330     for appid, app in apps.items():
 331         for build in app['builds']:
 332             if not build.disable:
 333                 continue
 334             apkfilename = common.get_release_filename(app, build)
 335             iconfilename = "%s.%s.png" % (
 336                 appid,
 337                 build.versionCode)
 338             for repodir in repodirs:
 339                 files = [
 340                     os.path.join(repodir, apkfilename),
 341                     os.path.join(repodir, apkfilename + '.asc'),
 342                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 343                 ]
 344                 for density in all_screen_densities:
 345                     repo_dir = get_icon_dir(repodir, density)
 346                     files.append(os.path.join(repo_dir, iconfilename))
 347
 348                 for f in files:
 349                     if os.path.exists(f):
 350                         logging.info("Deleting disabled build output " + f)
 351                         os.remove(f)
 352             if apkfilename in apkcache:
 353                 del apkcache[apkfilename]
 354
 355
 356 def resize_icon(iconpath, density):
 357
 358     if not os.path.isfile(iconpath):
 359         return
 360
 361     fp = None
 362     try:
 363         fp = open(iconpath, 'rb')
 364         im = Image.open(fp)
 365         size = dpi_to_px(density)
 366
 367         if any(length > size for length in im.size):
 368             oldsize = im.size
 369             im.thumbnail((size, size), Image.ANTIALIAS)
 370             logging.debug("%s was too large at %s - new size is %s" % (
 371                 iconpath, oldsize, im.size))
 372             im.save(iconpath, "PNG")
 373
 374     except Exception as e:
 375         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 376
 377     finally:
 378         if fp:
 379             fp.close()
 380
 381
 382 def resize_all_icons(repodirs):
 383     """Resize all icons that exceed the max size
 384
 385     :param repodirs: the repo directories to process
 386     """
 387     for repodir in repodirs:
 388         for density in screen_densities:
 389             icon_dir = get_icon_dir(repodir, density)
 390             icon_glob = os.path.join(icon_dir, '*.png')
 391             for iconpath in glob.glob(icon_glob):
 392                 resize_icon(iconpath, density)
 393
 394
 395 def getsig(apkpath):
 396     """ Get the signing certificate of an apk. To get the same md5 has that
 397     Android gets, we encode the .RSA certificate in a specific format and pass
 398     it hex-encoded to the md5 digest algorithm.
 399
 400     :param apkpath: path to the apk
 401     :returns: A string containing the md5 of the signature of the apk or None
 402               if an error occurred.
 403     """
 404
 405     with zipfile.ZipFile(apkpath, 'r') as apk:
 406         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 407
 408         if len(certs) < 1:
 409             logging.error("Found no signing certificates on %s" % apkpath)
 410             return None
 411         if len(certs) > 1:
 412             logging.error("Found multiple signing certificates on %s" % apkpath)
 413             return None
 414
 415         cert = apk.read(certs[0])
 416
 417     cert_encoded = common.get_certificate(cert)
 418
 419     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 420
 421
 422 def get_cache_file():
 423     return os.path.join('tmp', 'apkcache')
 424
 425
 426 def get_cache():
 427     """Get the cached dict of the APK index
 428
 429     Gather information about all the apk files in the repo directory,
 430     using cached data if possible. Some of the index operations take a
 431     long time, like calculating the SHA-256 and verifying the APK
 432     signature.
 433
 434     The cache is invalidated if the metadata version is different, or
 435     the 'allow_disabled_algorithms' config/option is different.  In
 436     those cases, there is no easy way to know what has changed from
 437     the cache, so just rerun the whole thing.
 438
 439     :return: apkcache
 440
 441     """
 442     apkcachefile = get_cache_file()
 443     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 444     if not options.clean and os.path.exists(apkcachefile):
 445         with open(apkcachefile, 'rb') as cf:
 446             apkcache = pickle.load(cf, encoding='utf-8')
 447         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 448            or apkcache.get('allow_disabled_algorithms') != ada:
 449             apkcache = {}
 450     else:
 451         apkcache = {}
 452
 453     apkcache["METADATA_VERSION"] = METADATA_VERSION
 454     apkcache['allow_disabled_algorithms'] = ada
 455
 456     return apkcache
 457
 458
 459 def write_cache(apkcache):
 460     apkcachefile = get_cache_file()
 461     cache_path = os.path.dirname(apkcachefile)
 462     if not os.path.exists(cache_path):
 463         os.makedirs(cache_path)
 464     with open(apkcachefile, 'wb') as cf:
 465         pickle.dump(apkcache, cf)
 466
 467
 468 def get_icon_bytes(apkzip, iconsrc):
 469     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 470     try:
 471         return apkzip.read(iconsrc)
 472     except KeyError:
 473         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 474
 475
 476 def sha256sum(filename):
 477     '''Calculate the sha256 of the given file'''
 478     sha = hashlib.sha256()
 479     with open(filename, 'rb') as f:
 480         while True:
 481             t = f.read(16384)
 482             if len(t) == 0:
 483                 break
 484             sha.update(t)
 485     return sha.hexdigest()
 486
 487
 488 def has_known_vulnerability(filename):
 489     """checks for known vulnerabilities in the APK
 490
 491     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 492     version.  Google also enforces this:
 493     https://support.google.com/faqs/answer/6376725?hl=en
 494
 495     Checks whether there are more than one classes.dex or AndroidManifest.xml
 496     files, which is invalid and an essential part of the "Master Key" attack.
 497
 498     http://www.saurik.com/id/17
 499     """
 500
 501     # statically load this pattern
 502     if not hasattr(has_known_vulnerability, "pattern"):
 503         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 504
 505     files_in_apk = set()
 506     with zipfile.ZipFile(filename) as zf:
 507         for name in zf.namelist():
 508             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 509                 lib = zf.open(name)
 510                 while True:
 511                     chunk = lib.read(4096)
 512                     if chunk == b'':
 513                         break
 514                     m = has_known_vulnerability.pattern.search(chunk)
 515                     if m:
 516                         version = m.group(1).decode('ascii')
 517                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 518                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 519                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 520                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 521                         else:
 522                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 523                             return True
 524                         break
 525             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 526                 if name in files_in_apk:
 527                     return True
 528                 files_in_apk.add(name)
 529
 530     return False
 531
 532
 533 def insert_obbs(repodir, apps, apks):
 534     """Scans the .obb files in a given repo directory and adds them to the
 535     relevant APK instances.  OBB files have versionCodes like APK
 536     files, and they are loosely associated.  If there is an OBB file
 537     present, then any APK with the same or higher versionCode will use
 538     that OBB file.  There are two OBB types: main and patch, each APK
 539     can only have only have one of each.
 540
 541     https://developer.android.com/google/play/expansion-files.html
 542
 543     :param repodir: repo directory to scan
 544     :param apps: list of current, valid apps
 545     :param apks: current information on all APKs
 546
 547     """
 548
 549     def obbWarnDelete(f, msg):
 550         logging.warning(msg + f)
 551         if options.delete_unknown:
 552             logging.error("Deleting unknown file: " + f)
 553             os.remove(f)
 554
 555     obbs = []
 556     java_Integer_MIN_VALUE = -pow(2, 31)
 557     currentPackageNames = apps.keys()
 558     for f in glob.glob(os.path.join(repodir, '*.obb')):
 559         obbfile = os.path.basename(f)
 560         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 561         chunks = obbfile.split('.')
 562         if chunks[0] != 'main' and chunks[0] != 'patch':
 563             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 564             continue
 565         if not re.match(r'^-?[0-9]+$', chunks[1]):
 566             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 567             continue
 568         versionCode = int(chunks[1])
 569         packagename = ".".join(chunks[2:-1])
 570
 571         highestVersionCode = java_Integer_MIN_VALUE
 572         if packagename not in currentPackageNames:
 573             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 574             continue
 575         for apk in apks:
 576             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 577                 highestVersionCode = apk['versionCode']
 578         if versionCode > highestVersionCode:
 579             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 580                           + ') than any APK: ')
 581             continue
 582         obbsha256 = sha256sum(f)
 583         obbs.append((packagename, versionCode, obbfile, obbsha256))
 584
 585     for apk in apks:
 586         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 587             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 588                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 589                     apk['obbMainFile'] = obbfile
 590                     apk['obbMainFileSha256'] = obbsha256
 591                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 592                     apk['obbPatchFile'] = obbfile
 593                     apk['obbPatchFileSha256'] = obbsha256
 594             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 595                 break
 596
 597
 598 def translate_per_build_anti_features(apps, apks):
 599     """Grab the anti-features list from the build metadata
 600
 601     For most Anti-Features, they are really most applicable per-APK,
 602     not for an app.  An app can fix a vulnerability, add/remove
 603     tracking, etc.  This reads the 'antifeatures' list from the Build
 604     entries in the fdroiddata metadata file, then transforms it into
 605     the 'antiFeatures' list of unique items for the index.
 606
 607     The field key is all lower case in the metadata file to match the
 608     rest of the Build fields.  It is 'antiFeatures' camel case in the
 609     implementation, index, and fdroidclient since it is translated
 610     from the build 'antifeatures' field, not directly included.
 611
 612     """
 613
 614     antiFeatures = dict()
 615     for packageName, app in apps.items():
 616         d = dict()
 617         for build in app['builds']:
 618             afl = build.get('antifeatures')
 619             if afl:
 620                 d[int(build.versionCode)] = afl
 621         if len(d) > 0:
 622             antiFeatures[packageName] = d
 623
 624     for apk in apks:
 625         d = antiFeatures.get(apk['packageName'])
 626         if d:
 627             afl = d.get(apk['versionCode'])
 628             if afl:
 629                 apk['antiFeatures'].update(afl)
 630
 631
 632 def _get_localized_dict(app, locale):
 633     '''get the dict to add localized store metadata to'''
 634     if 'localized' not in app:
 635         app['localized'] = collections.OrderedDict()
 636     if locale not in app['localized']:
 637         app['localized'][locale] = collections.OrderedDict()
 638     return app['localized'][locale]
 639
 640
 641 def _set_localized_text_entry(app, locale, key, f):
 642     limit = config['char_limits'][key]
 643     localized = _get_localized_dict(app, locale)
 644     with open(f) as fp:
 645         text = fp.read()[:limit]
 646         if len(text) > 0:
 647             localized[key] = text
 648
 649
 650 def _set_author_entry(app, key, f):
 651     limit = config['char_limits']['author']
 652     with open(f) as fp:
 653         text = fp.read()[:limit]
 654         if len(text) > 0:
 655             app[key] = text
 656
 657
 658 def copy_triple_t_store_metadata(apps):
 659     """Include store metadata from the app's source repo
 660
 661     The Triple-T Gradle Play Publisher is a plugin that has a standard
 662     file layout for all of the metadata and graphics that the Google
 663     Play Store accepts.  Since F-Droid has the git repo, it can just
 664     pluck those files directly.  This method reads any text files into
 665     the app dict, then copies any graphics into the fdroid repo
 666     directory structure.
 667
 668     This needs to be run before insert_localized_app_metadata() so that
 669     the graphics files that are copied into the fdroid repo get
 670     properly indexed.
 671
 672     https://github.com/Triple-T/gradle-play-publisher#upload-images
 673     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 674
 675     """
 676
 677     if not os.path.isdir('build'):
 678         return  # nothing to do
 679
 680     for packageName, app in apps.items():
 681         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 682             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 683             for root, dirs, files in os.walk(d):
 684                 segments = root.split('/')
 685                 locale = segments[-2]
 686                 for f in files:
 687                     if f == 'fulldescription':
 688                         _set_localized_text_entry(app, locale, 'description',
 689                                                   os.path.join(root, f))
 690                         continue
 691                     elif f == 'shortdescription':
 692                         _set_localized_text_entry(app, locale, 'summary',
 693                                                   os.path.join(root, f))
 694                         continue
 695                     elif f == 'title':
 696                         _set_localized_text_entry(app, locale, 'name',
 697                                                   os.path.join(root, f))
 698                         continue
 699                     elif f == 'video':
 700                         _set_localized_text_entry(app, locale, 'video',
 701                                                   os.path.join(root, f))
 702                         continue
 703                     elif f == 'whatsnew':
 704                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 705                                                   os.path.join(root, f))
 706                         continue
 707                     elif f == 'contactEmail':
 708                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 709                         continue
 710                     elif f == 'contactPhone':
 711                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 712                         continue
 713                     elif f == 'contactWebsite':
 714                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 715                         continue
 716
 717                     base, extension = common.get_extension(f)
 718                     dirname = os.path.basename(root)
 719                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 720                         if segments[-2] == 'listing':
 721                             locale = segments[-3]
 722                         else:
 723                             locale = segments[-2]
 724                         destdir = os.path.join('repo', packageName, locale)
 725                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 726                         sourcefile = os.path.join(root, f)
 727                         destfile = os.path.join(destdir, dirname + '.' + extension)
 728                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 729                         shutil.copy(sourcefile, destfile)
 730
 731
 732 def insert_localized_app_metadata(apps):
 733     """scans standard locations for graphics and localized text
 734
 735     Scans for localized description files, store graphics, and
 736     screenshot PNG files in statically defined screenshots directory
 737     and adds them to the app metadata.  The screenshots and graphic
 738     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 739     and must be in the following layout:
 740     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 741
 742     repo/packageName/locale/featureGraphic.png
 743     repo/packageName/locale/phoneScreenshots/1.png
 744     repo/packageName/locale/phoneScreenshots/2.png
 745
 746     The changelog files must be text files named with the versionCode
 747     ending with ".txt" and must be in the following layout:
 748     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 749
 750     repo/packageName/locale/changelogs/12345.txt
 751
 752     This will scan the each app's source repo then the metadata/ dir
 753     for these standard locations of changelog files.  If it finds
 754     them, they will be added to the dict of all packages, with the
 755     versions in the metadata/ folder taking precendence over the what
 756     is in the app's source repo.
 757
 758     Where "packageName" is the app's packageName and "locale" is the locale
 759     of the graphics, e.g. what language they are in, using the IETF RFC5646
 760     format (en-US, fr-CA, es-MX, etc).
 761
 762     This will also scan the app's git for a fastlane folder, and the
 763     metadata/ folder and the apps' source repos for standard locations
 764     of graphic and screenshot files.  If it finds them, it will copy
 765     them into the repo.  The fastlane files follow this pattern:
 766     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 767
 768     """
 769
 770     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 771     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 772     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 773
 774     for srcd in sorted(sourcedirs):
 775         if not os.path.isdir(srcd):
 776             continue
 777         for root, dirs, files in os.walk(srcd):
 778             segments = root.split('/')
 779             packageName = segments[1]
 780             if packageName not in apps:
 781                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 782                 continue
 783             locale = segments[-1]
 784             destdir = os.path.join('repo', packageName, locale)
 785             for f in files:
 786                 if f in ('description.txt', 'full_description.txt'):
 787                     _set_localized_text_entry(apps[packageName], locale, 'description',
 788                                               os.path.join(root, f))
 789                     continue
 790                 elif f in ('summary.txt', 'short_description.txt'):
 791                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 792                                               os.path.join(root, f))
 793                     continue
 794                 elif f in ('name.txt', 'title.txt'):
 795                     _set_localized_text_entry(apps[packageName], locale, 'name',
 796                                               os.path.join(root, f))
 797                     continue
 798                 elif f == 'video.txt':
 799                     _set_localized_text_entry(apps[packageName], locale, 'video',
 800                                               os.path.join(root, f))
 801                     continue
 802                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 803                     locale = segments[-2]
 804                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 805                                               os.path.join(root, f))
 806                     continue
 807
 808                 base, extension = common.get_extension(f)
 809                 if locale == 'images':
 810                     locale = segments[-2]
 811                     destdir = os.path.join('repo', packageName, locale)
 812                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 813                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 814                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 815                     shutil.copy(os.path.join(root, f), destdir)
 816             for d in dirs:
 817                 if d in SCREENSHOT_DIRS:
 818                     for f in glob.glob(os.path.join(root, d, '*.*')):
 819                         _, extension = common.get_extension(f)
 820                         if extension in ALLOWED_EXTENSIONS:
 821                             screenshotdestdir = os.path.join(destdir, d)
 822                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 823                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 824                             shutil.copy(f, screenshotdestdir)
 825
 826     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 827     for d in repofiles:
 828         if not os.path.isdir(d):
 829             continue
 830         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 831             if not os.path.isfile(f):
 832                 continue
 833             segments = f.split('/')
 834             packageName = segments[1]
 835             locale = segments[2]
 836             screenshotdir = segments[3]
 837             filename = os.path.basename(f)
 838             base, extension = common.get_extension(filename)
 839
 840             if packageName not in apps:
 841                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 842                                 % (filename, packageName))
 843                 continue
 844             graphics = _get_localized_dict(apps[packageName], locale)
 845
 846             if extension not in ALLOWED_EXTENSIONS:
 847                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 848             elif base in GRAPHIC_NAMES:
 849                 # there can only be zero or one of these per locale
 850                 graphics[base] = filename
 851             elif screenshotdir in SCREENSHOT_DIRS:
 852                 # there can any number of these per locale
 853                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 854                 if screenshotdir not in graphics:
 855                     graphics[screenshotdir] = []
 856                 graphics[screenshotdir].append(filename)
 857             else:
 858                 logging.warning('Unsupported graphics file found: ' + f)
 859
 860
 861 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 862     """Scan a repo for all files with an extension except APK/OBB
 863
 864     :param apkcache: current cached info about all repo files
 865     :param repodir: repo directory to scan
 866     :param knownapks: list of all known files, as per metadata.read_metadata
 867     :param use_date_from_file: use date from file (instead of current date)
 868                                for newly added files
 869     """
 870
 871     cachechanged = False
 872     repo_files = []
 873     repodir = repodir.encode('utf-8')
 874     for name in os.listdir(repodir):
 875         file_extension = common.get_file_extension(name)
 876         if file_extension == 'apk' or file_extension == 'obb':
 877             continue
 878         filename = os.path.join(repodir, name)
 879         name_utf8 = name.decode('utf-8')
 880         if filename.endswith(b'_src.tar.gz'):
 881             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 882             continue
 883         if not common.is_repo_file(filename):
 884             continue
 885         stat = os.stat(filename)
 886         if stat.st_size == 0:
 887             raise FDroidException(filename + ' is zero size!')
 888
 889         shasum = sha256sum(filename)
 890         usecache = False
 891         if name in apkcache:
 892             repo_file = apkcache[name]
 893             # added time is cached as tuple but used here as datetime instance
 894             if 'added' in repo_file:
 895                 a = repo_file['added']
 896                 if isinstance(a, datetime):
 897                     repo_file['added'] = a
 898                 else:
 899                     repo_file['added'] = datetime(*a[:6])
 900             if repo_file.get('hash') == shasum:
 901                 logging.debug("Reading " + name_utf8 + " from cache")
 902                 usecache = True
 903             else:
 904                 logging.debug("Ignoring stale cache data for " + name)
 905
 906         if not usecache:
 907             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 908             repo_file = collections.OrderedDict()
 909             repo_file['name'] = os.path.splitext(name_utf8)[0]
 910             # TODO rename apkname globally to something more generic
 911             repo_file['apkName'] = name_utf8
 912             repo_file['hash'] = shasum
 913             repo_file['hashType'] = 'sha256'
 914             repo_file['versionCode'] = 0
 915             repo_file['versionName'] = shasum
 916             # the static ID is the SHA256 unless it is set in the metadata
 917             repo_file['packageName'] = shasum
 918
 919             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 920             if m:
 921                 repo_file['packageName'] = m.group(1)
 922                 repo_file['versionCode'] = int(m.group(2))
 923             srcfilename = name + b'_src.tar.gz'
 924             if os.path.exists(os.path.join(repodir, srcfilename)):
 925                 repo_file['srcname'] = srcfilename.decode('utf-8')
 926             repo_file['size'] = stat.st_size
 927
 928             apkcache[name] = repo_file
 929             cachechanged = True
 930
 931         if use_date_from_file:
 932             timestamp = stat.st_ctime
 933             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 934         else:
 935             default_date_param = None
 936
 937         # Record in knownapks, getting the added date at the same time..
 938         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 939                                     default_date=default_date_param)
 940         if added:
 941             repo_file['added'] = added
 942
 943         repo_files.append(repo_file)
 944
 945     return repo_files, cachechanged
 946
 947
 948 def scan_apk(apk_file):
 949     """
 950     Scans an APK file and returns dictionary with metadata of the APK.
 951
 952     Attention: This does *not* verify that the APK signature is correct.
 953
 954     :param apk_file: The (ideally absolute) path to the APK file
 955     :raises BuildException
 956     :return A dict containing APK metadata
 957     """
 958     apk = {
 959         'hash': sha256sum(apk_file),
 960         'hashType': 'sha256',
 961         'uses-permission': [],
 962         'uses-permission-sdk-23': [],
 963         'features': [],
 964         'icons_src': {},
 965         'icons': {},
 966         'antiFeatures': set(),
 967     }
 968
 969     if SdkToolsPopen(['aapt', 'version'], output=False):
 970         scan_apk_aapt(apk, apk_file)
 971     else:
 972         scan_apk_androguard(apk, apk_file)
 973
 974     # Get the signature
 975     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 976     apk['sig'] = getsig(apk_file)
 977     if not apk['sig']:
 978         raise BuildException("Failed to get apk signature")
 979
 980     # Get size of the APK
 981     apk['size'] = os.path.getsize(apk_file)
 982
 983     if 'minSdkVersion' not in apk:
 984         logging.warning("No SDK version information found in {0}".format(apk_file))
 985         apk['minSdkVersion'] = 1
 986
 987     # Check for known vulnerabilities
 988     if has_known_vulnerability(apk_file):
 989         apk['antiFeatures'].add('KnownVuln')
 990
 991     return apk
 992
 993
 994 def scan_apk_aapt(apk, apkfile):
 995     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 996     if p.returncode != 0:
 997         if options.delete_unknown:
 998             if os.path.exists(apkfile):
 999                 logging.error("Failed to get apk information, deleting " + apkfile)
1000                 os.remove(apkfile)
1001             else:
1002                 logging.error("Could not find {0} to remove it".format(apkfile))
1003         else:
1004             logging.error("Failed to get apk information, skipping " + apkfile)
1005         raise BuildException("Invalid APK")
1006     for line in p.output.splitlines():
1007         if line.startswith("package:"):
1008             try:
1009                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1010                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1011                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1012             except Exception as e:
1013                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1014         elif line.startswith("application:"):
1015             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1016             # Keep path to non-dpi icon in case we need it
1017             match = re.match(APK_ICON_PAT_NODPI, line)
1018             if match:
1019                 apk['icons_src']['-1'] = match.group(1)
1020         elif line.startswith("launchable-activity:"):
1021             # Only use launchable-activity as fallback to application
1022             if not apk['name']:
1023                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1024             if '-1' not in apk['icons_src']:
1025                 match = re.match(APK_ICON_PAT_NODPI, line)
1026                 if match:
1027                     apk['icons_src']['-1'] = match.group(1)
1028         elif line.startswith("application-icon-"):
1029             match = re.match(APK_ICON_PAT, line)
1030             if match:
1031                 density = match.group(1)
1032                 path = match.group(2)
1033                 apk['icons_src'][density] = path
1034         elif line.startswith("sdkVersion:"):
1035             m = re.match(APK_SDK_VERSION_PAT, line)
1036             if m is None:
1037                 logging.error(line.replace('sdkVersion:', '')
1038                               + ' is not a valid minSdkVersion!')
1039             else:
1040                 apk['minSdkVersion'] = m.group(1)
1041                 # if target not set, default to min
1042                 if 'targetSdkVersion' not in apk:
1043                     apk['targetSdkVersion'] = m.group(1)
1044         elif line.startswith("targetSdkVersion:"):
1045             m = re.match(APK_SDK_VERSION_PAT, line)
1046             if m is None:
1047                 logging.error(line.replace('targetSdkVersion:', '')
1048                               + ' is not a valid targetSdkVersion!')
1049             else:
1050                 apk['targetSdkVersion'] = m.group(1)
1051         elif line.startswith("maxSdkVersion:"):
1052             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1053         elif line.startswith("native-code:"):
1054             apk['nativecode'] = []
1055             for arch in line[13:].split(' '):
1056                 apk['nativecode'].append(arch[1:-1])
1057         elif line.startswith('uses-permission:'):
1058             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1059             if perm_match['maxSdkVersion']:
1060                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1061             permission = UsesPermission(
1062                 perm_match['name'],
1063                 perm_match['maxSdkVersion']
1064             )
1065
1066             apk['uses-permission'].append(permission)
1067         elif line.startswith('uses-permission-sdk-23:'):
1068             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1069             if perm_match['maxSdkVersion']:
1070                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1071             permission_sdk_23 = UsesPermissionSdk23(
1072                 perm_match['name'],
1073                 perm_match['maxSdkVersion']
1074             )
1075
1076             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1077
1078         elif line.startswith('uses-feature:'):
1079             feature = re.match(APK_FEATURE_PAT, line).group(1)
1080             # Filter out this, it's only added with the latest SDK tools and
1081             # causes problems for lots of apps.
1082             if feature != "android.hardware.screen.portrait" \
1083                     and feature != "android.hardware.screen.landscape":
1084                 if feature.startswith("android.feature."):
1085                     feature = feature[16:]
1086                 apk['features'].add(feature)
1087
1088
1089 def scan_apk_androguard(apk, apkfile):
1090     try:
1091         from androguard.core.bytecodes.apk import APK
1092         apkobject = APK(apkfile)
1093         if apkobject.is_valid_APK():
1094             arsc = apkobject.get_android_resources()
1095         else:
1096             if options.delete_unknown:
1097                 if os.path.exists(apkfile):
1098                     logging.error("Failed to get apk information, deleting " + apkfile)
1099                     os.remove(apkfile)
1100                 else:
1101                     logging.error("Could not find {0} to remove it".format(apkfile))
1102             else:
1103                 logging.error("Failed to get apk information, skipping " + apkfile)
1104             raise BuildException("Invaild APK")
1105     except ImportError:
1106         raise FDroidException("androguard library is not installed and aapt not present")
1107     except FileNotFoundError:
1108         logging.error("Could not open apk file for analysis")
1109         raise BuildException("Invalid APK")
1110
1111     apk['packageName'] = apkobject.get_package()
1112     apk['versionCode'] = int(apkobject.get_androidversion_code())
1113     apk['versionName'] = apkobject.get_androidversion_name()
1114     if apk['versionName'][0] == "@":
1115         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1116         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1117         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1118     apk['name'] = apkobject.get_app_name()
1119
1120     if apkobject.get_max_sdk_version() is not None:
1121         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1122     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1123     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1124
1125     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1126     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1127
1128     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1129
1130     for file in apkobject.get_files():
1131         d_re = density_re.match(file)
1132         if d_re:
1133             folder = d_re.group(1).split('-')
1134             if len(folder) > 1:
1135                 resolution = folder[1]
1136             else:
1137                 resolution = 'mdpi'
1138             density = screen_resolutions[resolution]
1139             apk['icons_src'][density] = d_re.group(0)
1140
1141     if apk['icons_src'].get('-1') is None:
1142         apk['icons_src']['-1'] = apk['icons_src']['160']
1143
1144     arch_re = re.compile("^lib/(.*)/.*$")
1145     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1146     if len(arch) >= 1:
1147         apk['nativecode'] = []
1148         apk['nativecode'].extend(sorted(list(arch)))
1149
1150     xml = apkobject.get_android_manifest_xml()
1151
1152     for item in xml.getElementsByTagName('uses-permission'):
1153         name = str(item.getAttribute("android:name"))
1154         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1155         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1156         permission = UsesPermission(
1157             name,
1158             maxSdkVersion
1159         )
1160         apk['uses-permission'].append(permission)
1161
1162     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1163         name = str(item.getAttribute("android:name"))
1164         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1165         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1166         permission_sdk_23 = UsesPermissionSdk23(
1167             name,
1168             maxSdkVersion
1169         )
1170         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1171
1172     for item in xml.getElementsByTagName('uses-feature'):
1173         feature = str(item.getAttribute("android:name"))
1174         if feature != "android.hardware.screen.portrait" \
1175                 and feature != "android.hardware.screen.landscape":
1176             if feature.startswith("android.feature."):
1177                 feature = feature[16:]
1178         apk['features'].append(feature)
1179
1180
1181 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1182                 allow_disabled_algorithms=False, archive_bad_sig=False):
1183     """Processes the apk with the given filename in the given repo directory.
1184
1185     This also extracts the icons.
1186
1187     :param apkcache: current apk cache information
1188     :param apkfilename: the filename of the apk to scan
1189     :param repodir: repo directory to scan
1190     :param knownapks: known apks info
1191     :param use_date_from_apk: use date from APK (instead of current date)
1192                               for newly added APKs
1193     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1194                                       disabled algorithms in the signature (e.g. MD5)
1195     :param archive_bad_sig: move APKs with a bad signature to the archive
1196     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1197      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1198     """
1199
1200     if ' ' in apkfilename:
1201         if options.rename_apks:
1202             newfilename = apkfilename.replace(' ', '_')
1203             os.rename(os.path.join(repodir, apkfilename),
1204                       os.path.join(repodir, newfilename))
1205             apkfilename = newfilename
1206         else:
1207             logging.critical("Spaces in filenames are not allowed.")
1208             return True, None, False
1209
1210     apk = {}
1211     apkfile = os.path.join(repodir, apkfilename)
1212
1213     cachechanged = False
1214     usecache = False
1215     if apkfilename in apkcache:
1216         apk = apkcache[apkfilename]
1217         if apk.get('hash') == sha256sum(apkfile):
1218             logging.debug("Reading " + apkfilename + " from cache")
1219             usecache = True
1220         else:
1221             logging.debug("Ignoring stale cache data for " + apkfilename)
1222
1223     if not usecache:
1224         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1225
1226         try:
1227             apk = scan_apk(apkfile)
1228         except BuildException:
1229             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1230                             .format(apkfilename=apkfilename))
1231             return True, None, False
1232
1233         # Check for debuggable apks...
1234         if common.isApkAndDebuggable(apkfile):
1235             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1236
1237         if options.rename_apks:
1238             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1239             std_short_name = os.path.join(repodir, n)
1240             if apkfile != std_short_name:
1241                 if os.path.exists(std_short_name):
1242                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1243                     if apkfile != std_long_name:
1244                         if os.path.exists(std_long_name):
1245                             dupdir = os.path.join('duplicates', repodir)
1246                             if not os.path.isdir(dupdir):
1247                                 os.makedirs(dupdir, exist_ok=True)
1248                             dupfile = os.path.join('duplicates', std_long_name)
1249                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1250                             os.rename(apkfile, dupfile)
1251                             return True, None, False
1252                         else:
1253                             os.rename(apkfile, std_long_name)
1254                     apkfile = std_long_name
1255                 else:
1256                     os.rename(apkfile, std_short_name)
1257                     apkfile = std_short_name
1258                 apkfilename = apkfile[len(repodir) + 1:]
1259
1260         apk['apkName'] = apkfilename
1261         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1262         if os.path.exists(os.path.join(repodir, srcfilename)):
1263             apk['srcname'] = srcfilename
1264
1265         # verify the jar signature is correct, allow deprecated
1266         # algorithms only if the APK is in the archive.
1267         skipapk = False
1268         if not common.verify_apk_signature(apkfile):
1269             if repodir == 'archive' or allow_disabled_algorithms:
1270                 if common.verify_old_apk_signature(apkfile):
1271                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1272                 else:
1273                     skipapk = True
1274             else:
1275                 skipapk = True
1276
1277         if skipapk:
1278             if archive_bad_sig:
1279                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1280                 move_apk_between_sections(repodir, 'archive', apk)
1281             else:
1282                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1283             return True, None, False
1284
1285         apkzip = zipfile.ZipFile(apkfile, 'r')
1286
1287         # if an APK has files newer than the system time, suggest updating
1288         # the system clock.  This is useful for offline systems, used for
1289         # signing, which do not have another source of clock sync info. It
1290         # has to be more than 24 hours newer because ZIP/APK files do not
1291         # store timezone info
1292         manifest = apkzip.getinfo('AndroidManifest.xml')
1293         if manifest.date_time[1] == 0:  # month can't be zero
1294             logging.debug('AndroidManifest.xml has no date')
1295         else:
1296             dt_obj = datetime(*manifest.date_time)
1297             checkdt = dt_obj - timedelta(1)
1298             if datetime.today() < checkdt:
1299                 logging.warning('System clock is older than manifest in: '
1300                                 + apkfilename
1301                                 + '\nSet clock to that time using:\n'
1302                                 + 'sudo date -s "' + str(dt_obj) + '"')
1303
1304         # extract icons from APK zip file
1305         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1306         try:
1307             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1308         finally:
1309             apkzip.close()  # ensure that APK zip file gets closed
1310
1311         # resize existing icons for densities missing in the APK
1312         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1313
1314         if use_date_from_apk and manifest.date_time[1] != 0:
1315             default_date_param = datetime(*manifest.date_time)
1316         else:
1317             default_date_param = None
1318
1319         # Record in known apks, getting the added date at the same time..
1320         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1321                                     default_date=default_date_param)
1322         if added:
1323             apk['added'] = added
1324
1325         apkcache[apkfilename] = apk
1326         cachechanged = True
1327
1328     return False, apk, cachechanged
1329
1330
1331 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1332     """Processes the apks in the given repo directory.
1333
1334     This also extracts the icons.
1335
1336     :param apkcache: current apk cache information
1337     :param repodir: repo directory to scan
1338     :param knownapks: known apks info
1339     :param use_date_from_apk: use date from APK (instead of current date)
1340                               for newly added APKs
1341     :returns: (apks, cachechanged) where apks is a list of apk information,
1342               and cachechanged is True if the apkcache got changed.
1343     """
1344
1345     cachechanged = False
1346
1347     for icon_dir in get_all_icon_dirs(repodir):
1348         if os.path.exists(icon_dir):
1349             if options.clean:
1350                 shutil.rmtree(icon_dir)
1351                 os.makedirs(icon_dir)
1352         else:
1353             os.makedirs(icon_dir)
1354
1355     apks = []
1356     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1357         apkfilename = apkfile[len(repodir) + 1:]
1358         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1359         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1360                                              use_date_from_apk, ada, True)
1361         if skip:
1362             continue
1363         apks.append(apk)
1364         cachechanged = cachechanged or cachethis
1365
1366     return apks, cachechanged
1367
1368
1369 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1370     """
1371     Extracts icons from the given APK zip in various densities,
1372     saves them into given repo directory
1373     and stores their names in the APK metadata dictionary.
1374
1375     :param icon_filename: A string representing the icon's file name
1376     :param apk: A populated dictionary containing APK metadata.
1377                 Needs to have 'icons_src' key
1378     :param apkzip: An opened zipfile.ZipFile of the APK file
1379     :param repo_dir: The directory of the APK's repository
1380     :return: A list of icon densities that are missing
1381     """
1382     empty_densities = []
1383     for density in screen_densities:
1384         if density not in apk['icons_src']:
1385             empty_densities.append(density)
1386             continue
1387         icon_src = apk['icons_src'][density]
1388         icon_dir = get_icon_dir(repo_dir, density)
1389         icon_dest = os.path.join(icon_dir, icon_filename)
1390
1391         # Extract the icon files per density
1392         if icon_src.endswith('.xml'):
1393             png = os.path.basename(icon_src)[:-4] + '.png'
1394             for f in apkzip.namelist():
1395                 if f.endswith(png):
1396                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1397                     if m and screen_resolutions[m.group(2)] == density:
1398                         icon_src = f
1399             if icon_src.endswith('.xml'):
1400                 empty_densities.append(density)
1401                 continue
1402         try:
1403             with open(icon_dest, 'wb') as f:
1404                 f.write(get_icon_bytes(apkzip, icon_src))
1405             apk['icons'][density] = icon_filename
1406         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1407             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1408             del apk['icons_src'][density]
1409             empty_densities.append(density)
1410
1411     if '-1' in apk['icons_src']:
1412         icon_src = apk['icons_src']['-1']
1413         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1414         with open(icon_path, 'wb') as f:
1415             f.write(get_icon_bytes(apkzip, icon_src))
1416         try:
1417             im = Image.open(icon_path)
1418             dpi = px_to_dpi(im.size[0])
1419             for density in screen_densities:
1420                 if density in apk['icons']:
1421                     break
1422                 if density == screen_densities[-1] or dpi >= int(density):
1423                     apk['icons'][density] = icon_filename
1424                     shutil.move(icon_path,
1425                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1426                     empty_densities.remove(density)
1427                     break
1428         except Exception as e:
1429             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1430
1431     if apk['icons']:
1432         apk['icon'] = icon_filename
1433
1434     return empty_densities
1435
1436
1437 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1438     """
1439     Resize existing icons for densities missing in the APK to ensure all densities are available
1440
1441     :param empty_densities: A list of icon densities that are missing
1442     :param icon_filename: A string representing the icon's file name
1443     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1444     :param repo_dir: The directory of the APK's repository
1445     """
1446     # First try resizing down to not lose quality
1447     last_density = None
1448     for density in screen_densities:
1449         if density not in empty_densities:
1450             last_density = density
1451             continue
1452         if last_density is None:
1453             continue
1454         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1455
1456         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1457         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1458         fp = None
1459         try:
1460             fp = open(last_icon_path, 'rb')
1461             im = Image.open(fp)
1462
1463             size = dpi_to_px(density)
1464
1465             im.thumbnail((size, size), Image.ANTIALIAS)
1466             im.save(icon_path, "PNG")
1467             empty_densities.remove(density)
1468         except Exception as e:
1469             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1470         finally:
1471             if fp:
1472                 fp.close()
1473
1474     # Then just copy from the highest resolution available
1475     last_density = None
1476     for density in reversed(screen_densities):
1477         if density not in empty_densities:
1478             last_density = density
1479             continue
1480
1481         if last_density is None:
1482             continue
1483
1484         shutil.copyfile(
1485             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1486             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1487         )
1488         empty_densities.remove(density)
1489
1490     for density in screen_densities:
1491         icon_dir = get_icon_dir(repo_dir, density)
1492         icon_dest = os.path.join(icon_dir, icon_filename)
1493         resize_icon(icon_dest, density)
1494
1495     # Copy from icons-mdpi to icons since mdpi is the baseline density
1496     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1497     if os.path.isfile(baseline):
1498         apk['icons']['0'] = icon_filename
1499         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1500
1501
1502 def apply_info_from_latest_apk(apps, apks):
1503     """
1504     Some information from the apks needs to be applied up to the application level.
1505     When doing this, we use the info from the most recent version's apk.
1506     We deal with figuring out when the app was added and last updated at the same time.
1507     """
1508     for appid, app in apps.items():
1509         bestver = UNSET_VERSION_CODE
1510         for apk in apks:
1511             if apk['packageName'] == appid:
1512                 if apk['versionCode'] > bestver:
1513                     bestver = apk['versionCode']
1514                     bestapk = apk
1515
1516                 if 'added' in apk:
1517                     if not app.added or apk['added'] < app.added:
1518                         app.added = apk['added']
1519                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1520                         app.lastUpdated = apk['added']
1521
1522         if not app.added:
1523             logging.debug("Don't know when " + appid + " was added")
1524         if not app.lastUpdated:
1525             logging.debug("Don't know when " + appid + " was last updated")
1526
1527         if bestver == UNSET_VERSION_CODE:
1528
1529             if app.Name is None:
1530                 app.Name = app.AutoName or appid
1531             app.icon = None
1532             logging.debug("Application " + appid + " has no packages")
1533         else:
1534             if app.Name is None:
1535                 app.Name = bestapk['name']
1536             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1537             if app.CurrentVersionCode is None:
1538                 app.CurrentVersionCode = str(bestver)
1539
1540
1541 def make_categories_txt(repodir, categories):
1542     '''Write a category list in the repo to allow quick access'''
1543     catdata = ''
1544     for cat in sorted(categories):
1545         catdata += cat + '\n'
1546     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1547         f.write(catdata)
1548
1549
1550 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1551
1552     def filter_apk_list_sorted(apk_list):
1553         res = []
1554         for apk in apk_list:
1555             if apk['packageName'] == appid:
1556                 res.append(apk)
1557
1558         # Sort the apk list by version code. First is highest/newest.
1559         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1560
1561     for appid, app in apps.items():
1562
1563         if app.ArchivePolicy:
1564             keepversions = int(app.ArchivePolicy[:-9])
1565         else:
1566             keepversions = defaultkeepversions
1567
1568         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1569                       .format(appid, len(apks), keepversions, len(archapks)))
1570
1571         current_app_apks = filter_apk_list_sorted(apks)
1572         if len(current_app_apks) > keepversions:
1573             # Move back the ones we don't want.
1574             for apk in current_app_apks[keepversions:]:
1575                 move_apk_between_sections(repodir, archivedir, apk)
1576                 archapks.append(apk)
1577                 apks.remove(apk)
1578
1579         current_app_archapks = filter_apk_list_sorted(archapks)
1580         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1581             kept = 0
1582             # Move forward the ones we want again, except DisableAlgorithm
1583             for apk in current_app_archapks:
1584                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1585                     move_apk_between_sections(archivedir, repodir, apk)
1586                     archapks.remove(apk)
1587                     apks.append(apk)
1588                     kept += 1
1589                 if kept == keepversions:
1590                     break
1591
1592
1593 def move_apk_between_sections(from_dir, to_dir, apk):
1594     """move an APK from repo to archive or vice versa"""
1595
1596     def _move_file(from_dir, to_dir, filename, ignore_missing):
1597         from_path = os.path.join(from_dir, filename)
1598         if ignore_missing and not os.path.exists(from_path):
1599             return
1600         to_path = os.path.join(to_dir, filename)
1601         if not os.path.exists(to_dir):
1602             os.mkdir(to_dir)
1603         shutil.move(from_path, to_path)
1604
1605     if from_dir == to_dir:
1606         return
1607
1608     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1609     _move_file(from_dir, to_dir, apk['apkName'], False)
1610     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1611     for density in all_screen_densities:
1612         from_icon_dir = get_icon_dir(from_dir, density)
1613         to_icon_dir = get_icon_dir(to_dir, density)
1614         if density not in apk['icons']:
1615             continue
1616         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1617     if 'srcname' in apk:
1618         _move_file(from_dir, to_dir, apk['srcname'], False)
1619
1620
1621 def add_apks_to_per_app_repos(repodir, apks):
1622     apks_per_app = dict()
1623     for apk in apks:
1624         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1625         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1626         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1627         apks_per_app[apk['packageName']] = apk
1628
1629         if not os.path.exists(apk['per_app_icons']):
1630             logging.info('Adding new repo for only ' + apk['packageName'])
1631             os.makedirs(apk['per_app_icons'])
1632
1633         apkpath = os.path.join(repodir, apk['apkName'])
1634         shutil.copy(apkpath, apk['per_app_repo'])
1635         apksigpath = apkpath + '.sig'
1636         if os.path.exists(apksigpath):
1637             shutil.copy(apksigpath, apk['per_app_repo'])
1638         apkascpath = apkpath + '.asc'
1639         if os.path.exists(apkascpath):
1640             shutil.copy(apkascpath, apk['per_app_repo'])
1641
1642
1643 def create_metadata_from_template(apk):
1644     '''create a new metadata file using internal or external template
1645
1646     Generate warnings for apk's with no metadata (or create skeleton
1647     metadata files, if requested on the command line).  Though the
1648     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1649     since those impose things on the metadata file made from the
1650     template: field sort order, empty field value, formatting, etc.
1651     '''
1652
1653     import yaml
1654     if os.path.exists('template.yml'):
1655         with open('template.yml') as f:
1656             metatxt = f.read()
1657         if 'name' in apk and apk['name'] != '':
1658             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1659                              r'\1 ' + apk['name'],
1660                              metatxt,
1661                              flags=re.IGNORECASE | re.MULTILINE)
1662         else:
1663             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1664             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1665                              r'\1 ' + apk['packageName'],
1666                              metatxt,
1667                              flags=re.IGNORECASE | re.MULTILINE)
1668         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1669             f.write(metatxt)
1670     else:
1671         app = dict()
1672         app['Categories'] = [os.path.basename(os.getcwd())]
1673         # include some blanks as part of the template
1674         app['AuthorName'] = ''
1675         app['Summary'] = ''
1676         app['WebSite'] = ''
1677         app['IssueTracker'] = ''
1678         app['SourceCode'] = ''
1679         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1680         if 'name' in apk and apk['name'] != '':
1681             app['Name'] = apk['name']
1682         else:
1683             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1684             app['Name'] = apk['packageName']
1685         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1686             yaml.dump(app, f, default_flow_style=False)
1687     logging.info("Generated skeleton metadata for " + apk['packageName'])
1688
1689
1690 config = None
1691 options = None
1692
1693
1694 def main():
1695
1696     global config, options
1697
1698     # Parse command line...
1699     parser = ArgumentParser()
1700     common.setup_global_opts(parser)
1701     parser.add_argument("--create-key", action="store_true", default=False,
1702                         help=_("Create a repo signing key in a keystore"))
1703     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1704                         help=_("Create skeleton metadata files that are missing"))
1705     parser.add_argument("--delete-unknown", action="store_true", default=False,
1706                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1707     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1708                         help=_("Report on build data status"))
1709     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1710                         help=_("Interactively ask about things that need updating."))
1711     parser.add_argument("-I", "--icons", action="store_true", default=False,
1712                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1713     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1714                         help=_("Specify editor to use in interactive mode. Default ") +
1715                         "is /etc/alternatives/editor")
1716     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1717                         help=_("Update the wiki"))
1718     parser.add_argument("--pretty", action="store_true", default=False,
1719                         help=_("Produce human-readable index.xml"))
1720     parser.add_argument("--clean", action="store_true", default=False,
1721                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1722     parser.add_argument("--nosign", action="store_true", default=False,
1723                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1724     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1725                         help=_("Use date from APK instead of current time for newly added APKs"))
1726     parser.add_argument("--rename-apks", action="store_true", default=False,
1727                         help=_("Rename APK files that do not match package.name_123.apk"))
1728     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1729                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1730     metadata.add_metadata_arguments(parser)
1731     options = parser.parse_args()
1732     metadata.warnings_action = options.W
1733
1734     config = common.read_config(options)
1735
1736     if not ('jarsigner' in config and 'keytool' in config):
1737         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1738
1739     repodirs = ['repo']
1740     if config['archive_older'] != 0:
1741         repodirs.append('archive')
1742         if not os.path.exists('archive'):
1743             os.mkdir('archive')
1744
1745     if options.icons:
1746         resize_all_icons(repodirs)
1747         sys.exit(0)
1748
1749     if options.rename_apks:
1750         options.clean = True
1751
1752     # check that icons exist now, rather than fail at the end of `fdroid update`
1753     for k in ['repo_icon', 'archive_icon']:
1754         if k in config:
1755             if not os.path.exists(config[k]):
1756                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1757                 sys.exit(1)
1758
1759     # if the user asks to create a keystore, do it now, reusing whatever it can
1760     if options.create_key:
1761         if os.path.exists(config['keystore']):
1762             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1763             logging.critical("\t'" + config['keystore'] + "'")
1764             sys.exit(1)
1765
1766         if 'repo_keyalias' not in config:
1767             config['repo_keyalias'] = socket.getfqdn()
1768             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1769         if 'keydname' not in config:
1770             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1771             common.write_to_config(config, 'keydname', config['keydname'])
1772         if 'keystore' not in config:
1773             config['keystore'] = common.default_config['keystore']
1774             common.write_to_config(config, 'keystore', config['keystore'])
1775
1776         password = common.genpassword()
1777         if 'keystorepass' not in config:
1778             config['keystorepass'] = password
1779             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1780         if 'keypass' not in config:
1781             config['keypass'] = password
1782             common.write_to_config(config, 'keypass', config['keypass'])
1783         common.genkeystore(config)
1784
1785     # Get all apps...
1786     apps = metadata.read_metadata()
1787
1788     # Generate a list of categories...
1789     categories = set()
1790     for app in apps.values():
1791         categories.update(app.Categories)
1792
1793     # Read known apks data (will be updated and written back when we've finished)
1794     knownapks = common.KnownApks()
1795
1796     # Get APK cache
1797     apkcache = get_cache()
1798
1799     # Delete builds for disabled apps
1800     delete_disabled_builds(apps, apkcache, repodirs)
1801
1802     # Scan all apks in the main repo
1803     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1804
1805     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1806                                            options.use_date_from_apk)
1807     cachechanged = cachechanged or fcachechanged
1808     apks += files
1809     for apk in apks:
1810         if apk['packageName'] not in apps:
1811             if options.create_metadata:
1812                 create_metadata_from_template(apk)
1813                 apps = metadata.read_metadata()
1814             else:
1815                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1816                 if options.delete_unknown:
1817                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1818                     rmf = os.path.join(repodirs[0], apk['apkName'])
1819                     if not os.path.exists(rmf):
1820                         logging.error("Could not find {0} to remove it".format(rmf))
1821                     else:
1822                         os.remove(rmf)
1823                 else:
1824                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1825
1826     copy_triple_t_store_metadata(apps)
1827     insert_obbs(repodirs[0], apps, apks)
1828     insert_localized_app_metadata(apps)
1829     translate_per_build_anti_features(apps, apks)
1830
1831     # Scan the archive repo for apks as well
1832     if len(repodirs) > 1:
1833         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1834         if cc:
1835             cachechanged = True
1836     else:
1837         archapks = []
1838
1839     # Apply information from latest apks to the application and update dates
1840     apply_info_from_latest_apk(apps, apks + archapks)
1841
1842     # Sort the app list by name, then the web site doesn't have to by default.
1843     # (we had to wait until we'd scanned the apks to do this, because mostly the
1844     # name comes from there!)
1845     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1846
1847     # APKs are placed into multiple repos based on the app package, providing
1848     # per-app subscription feeds for nightly builds and things like it
1849     if config['per_app_repos']:
1850         add_apks_to_per_app_repos(repodirs[0], apks)
1851         for appid, app in apps.items():
1852             repodir = os.path.join(appid, 'fdroid', 'repo')
1853             appdict = dict()
1854             appdict[appid] = app
1855             if os.path.isdir(repodir):
1856                 index.make(appdict, [appid], apks, repodir, False)
1857             else:
1858                 logging.info('Skipping index generation for ' + appid)
1859         return
1860
1861     if len(repodirs) > 1:
1862         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1863
1864     # Make the index for the main repo...
1865     index.make(apps, sortedids, apks, repodirs[0], False)
1866     make_categories_txt(repodirs[0], categories)
1867
1868     # If there's an archive repo,  make the index for it. We already scanned it
1869     # earlier on.
1870     if len(repodirs) > 1:
1871         index.make(apps, sortedids, archapks, repodirs[1], True)
1872
1873     git_remote = config.get('binary_transparency_remote')
1874     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1875         from . import btlog
1876         btlog.make_binary_transparency_log(repodirs)
1877
1878     if config['update_stats']:
1879         # Update known apks info...
1880         knownapks.writeifchanged()
1881
1882         # Generate latest apps data for widget
1883         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1884             data = ''
1885             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1886                 for line in f:
1887                     appid = line.rstrip()
1888                     data += appid + "\t"
1889                     app = apps[appid]
1890                     data += app.Name + "\t"
1891                     if app.icon is not None:
1892                         data += app.icon + "\t"
1893                     data += app.License + "\n"
1894             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1895                 f.write(data)
1896
1897     if cachechanged:
1898         write_cache(apkcache)
1899
1900     # Update the wiki...
1901     if options.wiki:
1902         update_wiki(apps, sortedids, apks + archapks)
1903
1904     logging.info(_("Finished"))
1905
1906
1907 if __name__ == "__main__":
1908     main()