fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime, timedelta
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0':
  98         return os.path.join(repodir, "icons")
  99     return os.path.join(repodir, "icons-%s" % density)
 100
 101
 102 def get_icon_dirs(repodir):
 103     for density in screen_densities:
 104         yield get_icon_dir(repodir, density)
 105
 106
 107 def get_all_icon_dirs(repodir):
 108     for density in all_screen_densities:
 109         yield get_icon_dir(repodir, density)
 110
 111
 112 def update_wiki(apps, sortedids, apks):
 113     """Update the wiki
 114
 115     :param apps: fully populated list of all applications
 116     :param apks: all apks, except...
 117     """
 118     logging.info("Updating wiki")
 119     wikicat = 'Apps'
 120     wikiredircat = 'App Redirects'
 121     import mwclient
 122     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 123                          path=config['wiki_path'])
 124     site.login(config['wiki_user'], config['wiki_password'])
 125     generated_pages = {}
 126     generated_redirects = {}
 127
 128     for appid in sortedids:
 129         app = metadata.App(apps[appid])
 130
 131         wikidata = ''
 132         if app.Disabled:
 133             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 134         if app.AntiFeatures:
 135             for af in app.AntiFeatures:
 136                 wikidata += '{{AntiFeature|' + af + '}}\n'
 137         if app.RequiresRoot:
 138             requiresroot = 'Yes'
 139         else:
 140             requiresroot = 'No'
 141         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 142             appid,
 143             app.Name,
 144             app.added.strftime('%Y-%m-%d') if app.added else '',
 145             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 146             app.SourceCode,
 147             app.IssueTracker,
 148             app.WebSite,
 149             app.Changelog,
 150             app.Donate,
 151             app.FlattrID,
 152             app.Bitcoin,
 153             app.Litecoin,
 154             app.License,
 155             requiresroot,
 156             app.AuthorName,
 157             app.AuthorEmail)
 158
 159         if app.Provides:
 160             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 161
 162         wikidata += app.Summary
 163         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 164
 165         wikidata += "=Description=\n"
 166         wikidata += metadata.description_wiki(app.Description) + "\n"
 167
 168         wikidata += "=Maintainer Notes=\n"
 169         if app.MaintainerNotes:
 170             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 171         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 172
 173         # Get a list of all packages for this application...
 174         apklist = []
 175         gotcurrentver = False
 176         cantupdate = False
 177         buildfails = False
 178         for apk in apks:
 179             if apk['packageName'] == appid:
 180                 if str(apk['versionCode']) == app.CurrentVersionCode:
 181                     gotcurrentver = True
 182                 apklist.append(apk)
 183         # Include ones we can't build, as a special case...
 184         for build in app.builds:
 185             if build.disable:
 186                 if build.versionCode == app.CurrentVersionCode:
 187                     cantupdate = True
 188                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 189                 apklist.append({'versionCode': int(build.versionCode),
 190                                 'versionName': build.versionName,
 191                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 192                                 })
 193             else:
 194                 builtit = False
 195                 for apk in apklist:
 196                     if apk['versionCode'] == int(build.versionCode):
 197                         builtit = True
 198                         break
 199                 if not builtit:
 200                     buildfails = True
 201                     apklist.append({'versionCode': int(build.versionCode),
 202                                     'versionName': build.versionName,
 203                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 204                                     })
 205         if app.CurrentVersionCode == '0':
 206             cantupdate = True
 207         # Sort with most recent first...
 208         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 209
 210         wikidata += "=Versions=\n"
 211         if len(apklist) == 0:
 212             wikidata += "We currently have no versions of this app available."
 213         elif not gotcurrentver:
 214             wikidata += "We don't have the current version of this app."
 215         else:
 216             wikidata += "We have the current version of this app."
 217         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 218         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 219         if len(app.NoSourceSince) > 0:
 220             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 221         if len(app.CurrentVersion) > 0:
 222             wikidata += "The current (recommended) version is " + app.CurrentVersion
 223             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 224         validapks = 0
 225         for apk in apklist:
 226             wikidata += "==" + apk['versionName'] + "==\n"
 227
 228             if 'buildproblem' in apk:
 229                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 230             else:
 231                 validapks += 1
 232                 wikidata += "This version is built and signed by "
 233                 if 'srcname' in apk:
 234                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 235                 else:
 236                     wikidata += "the original developer.\n\n"
 237             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 238
 239         wikidata += '\n[[Category:' + wikicat + ']]\n'
 240         if len(app.NoSourceSince) > 0:
 241             wikidata += '\n[[Category:Apps missing source code]]\n'
 242         if validapks == 0 and not app.Disabled:
 243             wikidata += '\n[[Category:Apps with no packages]]\n'
 244         if cantupdate and not app.Disabled:
 245             wikidata += "\n[[Category:Apps we cannot update]]\n"
 246         if buildfails and not app.Disabled:
 247             wikidata += "\n[[Category:Apps with failing builds]]\n"
 248         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 249             wikidata += '\n[[Category:Apps to Update]]\n'
 250         if app.Disabled:
 251             wikidata += '\n[[Category:Apps that are disabled]]\n'
 252         if app.UpdateCheckMode == 'None' and not app.Disabled:
 253             wikidata += '\n[[Category:Apps with no update check]]\n'
 254         for appcat in app.Categories:
 255             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 256
 257         # We can't have underscores in the page name, even if they're in
 258         # the package ID, because MediaWiki messes with them...
 259         pagename = appid.replace('_', ' ')
 260
 261         # Drop a trailing newline, because mediawiki is going to drop it anyway
 262         # and it we don't we'll think the page has changed when it hasn't...
 263         if wikidata.endswith('\n'):
 264             wikidata = wikidata[:-1]
 265
 266         generated_pages[pagename] = wikidata
 267
 268         # Make a redirect from the name to the ID too, unless there's
 269         # already an existing page with the name and it isn't a redirect.
 270         noclobber = False
 271         apppagename = app.Name.replace('_', ' ')
 272         apppagename = apppagename.replace('{', '')
 273         apppagename = apppagename.replace('}', ' ')
 274         apppagename = apppagename.replace(':', ' ')
 275         apppagename = apppagename.replace('[', ' ')
 276         apppagename = apppagename.replace(']', ' ')
 277         # Drop double spaces caused mostly by replacing ':' above
 278         apppagename = apppagename.replace('  ', ' ')
 279         for expagename in site.allpages(prefix=apppagename,
 280                                         filterredir='nonredirects',
 281                                         generator=False):
 282             if expagename == apppagename:
 283                 noclobber = True
 284         # Another reason not to make the redirect page is if the app name
 285         # is the same as it's ID, because that will overwrite the real page
 286         # with an redirect to itself! (Although it seems like an odd
 287         # scenario this happens a lot, e.g. where there is metadata but no
 288         # builds or binaries to extract a name from.
 289         if apppagename == pagename:
 290             noclobber = True
 291         if not noclobber:
 292             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 293
 294     for tcat, genp in [(wikicat, generated_pages),
 295                        (wikiredircat, generated_redirects)]:
 296         catpages = site.Pages['Category:' + tcat]
 297         existingpages = []
 298         for page in catpages:
 299             existingpages.append(page.name)
 300             if page.name in genp:
 301                 pagetxt = page.edit()
 302                 if pagetxt != genp[page.name]:
 303                     logging.debug("Updating modified page " + page.name)
 304                     page.save(genp[page.name], summary='Auto-updated')
 305                 else:
 306                     logging.debug("Page " + page.name + " is unchanged")
 307             else:
 308                 logging.warn("Deleting page " + page.name)
 309                 page.delete('No longer published')
 310         for pagename, text in genp.items():
 311             logging.debug("Checking " + pagename)
 312             if pagename not in existingpages:
 313                 logging.debug("Creating page " + pagename)
 314                 try:
 315                     newpage = site.Pages[pagename]
 316                     newpage.save(text, summary='Auto-created')
 317                 except Exception as e:
 318                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 319
 320     # Purge server cache to ensure counts are up to date
 321     site.pages['Repository Maintenance'].purge()
 322
 323
 324 def delete_disabled_builds(apps, apkcache, repodirs):
 325     """Delete disabled build outputs.
 326
 327     :param apps: list of all applications, as per metadata.read_metadata
 328     :param apkcache: current apk cache information
 329     :param repodirs: the repo directories to process
 330     """
 331     for appid, app in apps.items():
 332         for build in app['builds']:
 333             if not build.disable:
 334                 continue
 335             apkfilename = common.get_release_filename(app, build)
 336             iconfilename = "%s.%s.png" % (
 337                 appid,
 338                 build.versionCode)
 339             for repodir in repodirs:
 340                 files = [
 341                     os.path.join(repodir, apkfilename),
 342                     os.path.join(repodir, apkfilename + '.asc'),
 343                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 344                 ]
 345                 for density in all_screen_densities:
 346                     repo_dir = get_icon_dir(repodir, density)
 347                     files.append(os.path.join(repo_dir, iconfilename))
 348
 349                 for f in files:
 350                     if os.path.exists(f):
 351                         logging.info("Deleting disabled build output " + f)
 352                         os.remove(f)
 353             if apkfilename in apkcache:
 354                 del apkcache[apkfilename]
 355
 356
 357 def resize_icon(iconpath, density):
 358
 359     if not os.path.isfile(iconpath):
 360         return
 361
 362     fp = None
 363     try:
 364         fp = open(iconpath, 'rb')
 365         im = Image.open(fp)
 366         size = dpi_to_px(density)
 367
 368         if any(length > size for length in im.size):
 369             oldsize = im.size
 370             im.thumbnail((size, size), Image.ANTIALIAS)
 371             logging.debug("%s was too large at %s - new size is %s" % (
 372                 iconpath, oldsize, im.size))
 373             im.save(iconpath, "PNG")
 374
 375     except Exception as e:
 376         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 377
 378     finally:
 379         if fp:
 380             fp.close()
 381
 382
 383 def resize_all_icons(repodirs):
 384     """Resize all icons that exceed the max size
 385
 386     :param repodirs: the repo directories to process
 387     """
 388     for repodir in repodirs:
 389         for density in screen_densities:
 390             icon_dir = get_icon_dir(repodir, density)
 391             icon_glob = os.path.join(icon_dir, '*.png')
 392             for iconpath in glob.glob(icon_glob):
 393                 resize_icon(iconpath, density)
 394
 395
 396 def getsig(apkpath):
 397     """ Get the signing certificate of an apk. To get the same md5 has that
 398     Android gets, we encode the .RSA certificate in a specific format and pass
 399     it hex-encoded to the md5 digest algorithm.
 400
 401     :param apkpath: path to the apk
 402     :returns: A string containing the md5 of the signature of the apk or None
 403               if an error occurred.
 404     """
 405
 406     with zipfile.ZipFile(apkpath, 'r') as apk:
 407         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 408
 409         if len(certs) < 1:
 410             logging.error("Found no signing certificates on %s" % apkpath)
 411             return None
 412         if len(certs) > 1:
 413             logging.error("Found multiple signing certificates on %s" % apkpath)
 414             return None
 415
 416         cert = apk.read(certs[0])
 417
 418     cert_encoded = common.get_certificate(cert)
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_cache_file():
 424     return os.path.join('tmp', 'apkcache')
 425
 426
 427 def get_cache():
 428     """Get the cached dict of the APK index
 429
 430     Gather information about all the apk files in the repo directory,
 431     using cached data if possible. Some of the index operations take a
 432     long time, like calculating the SHA-256 and verifying the APK
 433     signature.
 434
 435     The cache is invalidated if the metadata version is different, or
 436     the 'allow_disabled_algorithms' config/option is different.  In
 437     those cases, there is no easy way to know what has changed from
 438     the cache, so just rerun the whole thing.
 439
 440     :return: apkcache
 441
 442     """
 443     apkcachefile = get_cache_file()
 444     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 445     if not options.clean and os.path.exists(apkcachefile):
 446         with open(apkcachefile, 'rb') as cf:
 447             apkcache = pickle.load(cf, encoding='utf-8')
 448         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 449            or apkcache.get('allow_disabled_algorithms') != ada:
 450             apkcache = {}
 451     else:
 452         apkcache = {}
 453
 454     apkcache["METADATA_VERSION"] = METADATA_VERSION
 455     apkcache['allow_disabled_algorithms'] = ada
 456
 457     return apkcache
 458
 459
 460 def write_cache(apkcache):
 461     apkcachefile = get_cache_file()
 462     cache_path = os.path.dirname(apkcachefile)
 463     if not os.path.exists(cache_path):
 464         os.makedirs(cache_path)
 465     with open(apkcachefile, 'wb') as cf:
 466         pickle.dump(apkcache, cf)
 467
 468
 469 def get_icon_bytes(apkzip, iconsrc):
 470     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 471     try:
 472         return apkzip.read(iconsrc)
 473     except KeyError:
 474         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 475
 476
 477 def sha256sum(filename):
 478     '''Calculate the sha256 of the given file'''
 479     sha = hashlib.sha256()
 480     with open(filename, 'rb') as f:
 481         while True:
 482             t = f.read(16384)
 483             if len(t) == 0:
 484                 break
 485             sha.update(t)
 486     return sha.hexdigest()
 487
 488
 489 def has_known_vulnerability(filename):
 490     """checks for known vulnerabilities in the APK
 491
 492     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 493     version.  Google also enforces this:
 494     https://support.google.com/faqs/answer/6376725?hl=en
 495
 496     Checks whether there are more than one classes.dex or AndroidManifest.xml
 497     files, which is invalid and an essential part of the "Master Key" attack.
 498
 499     http://www.saurik.com/id/17
 500     """
 501
 502     # statically load this pattern
 503     if not hasattr(has_known_vulnerability, "pattern"):
 504         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 505
 506     files_in_apk = set()
 507     with zipfile.ZipFile(filename) as zf:
 508         for name in zf.namelist():
 509             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 510                 lib = zf.open(name)
 511                 while True:
 512                     chunk = lib.read(4096)
 513                     if chunk == b'':
 514                         break
 515                     m = has_known_vulnerability.pattern.search(chunk)
 516                     if m:
 517                         version = m.group(1).decode('ascii')
 518                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 519                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 520                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 521                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 522                         else:
 523                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 524                             return True
 525                         break
 526             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 527                 if name in files_in_apk:
 528                     return True
 529                 files_in_apk.add(name)
 530
 531     return False
 532
 533
 534 def insert_obbs(repodir, apps, apks):
 535     """Scans the .obb files in a given repo directory and adds them to the
 536     relevant APK instances.  OBB files have versionCodes like APK
 537     files, and they are loosely associated.  If there is an OBB file
 538     present, then any APK with the same or higher versionCode will use
 539     that OBB file.  There are two OBB types: main and patch, each APK
 540     can only have only have one of each.
 541
 542     https://developer.android.com/google/play/expansion-files.html
 543
 544     :param repodir: repo directory to scan
 545     :param apps: list of current, valid apps
 546     :param apks: current information on all APKs
 547
 548     """
 549
 550     def obbWarnDelete(f, msg):
 551         logging.warning(msg + f)
 552         if options.delete_unknown:
 553             logging.error("Deleting unknown file: " + f)
 554             os.remove(f)
 555
 556     obbs = []
 557     java_Integer_MIN_VALUE = -pow(2, 31)
 558     currentPackageNames = apps.keys()
 559     for f in glob.glob(os.path.join(repodir, '*.obb')):
 560         obbfile = os.path.basename(f)
 561         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 562         chunks = obbfile.split('.')
 563         if chunks[0] != 'main' and chunks[0] != 'patch':
 564             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 565             continue
 566         if not re.match(r'^-?[0-9]+$', chunks[1]):
 567             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 568             continue
 569         versionCode = int(chunks[1])
 570         packagename = ".".join(chunks[2:-1])
 571
 572         highestVersionCode = java_Integer_MIN_VALUE
 573         if packagename not in currentPackageNames:
 574             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 575             continue
 576         for apk in apks:
 577             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 578                 highestVersionCode = apk['versionCode']
 579         if versionCode > highestVersionCode:
 580             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 581                           + ') than any APK: ')
 582             continue
 583         obbsha256 = sha256sum(f)
 584         obbs.append((packagename, versionCode, obbfile, obbsha256))
 585
 586     for apk in apks:
 587         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 588             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 589                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 590                     apk['obbMainFile'] = obbfile
 591                     apk['obbMainFileSha256'] = obbsha256
 592                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 593                     apk['obbPatchFile'] = obbfile
 594                     apk['obbPatchFileSha256'] = obbsha256
 595             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 596                 break
 597
 598
 599 def translate_per_build_anti_features(apps, apks):
 600     """Grab the anti-features list from the build metadata
 601
 602     For most Anti-Features, they are really most applicable per-APK,
 603     not for an app.  An app can fix a vulnerability, add/remove
 604     tracking, etc.  This reads the 'antifeatures' list from the Build
 605     entries in the fdroiddata metadata file, then transforms it into
 606     the 'antiFeatures' list of unique items for the index.
 607
 608     The field key is all lower case in the metadata file to match the
 609     rest of the Build fields.  It is 'antiFeatures' camel case in the
 610     implementation, index, and fdroidclient since it is translated
 611     from the build 'antifeatures' field, not directly included.
 612
 613     """
 614
 615     antiFeatures = dict()
 616     for packageName, app in apps.items():
 617         d = dict()
 618         for build in app['builds']:
 619             afl = build.get('antifeatures')
 620             if afl:
 621                 d[int(build.versionCode)] = afl
 622         if len(d) > 0:
 623             antiFeatures[packageName] = d
 624
 625     for apk in apks:
 626         d = antiFeatures.get(apk['packageName'])
 627         if d:
 628             afl = d.get(apk['versionCode'])
 629             if afl:
 630                 apk['antiFeatures'].update(afl)
 631
 632
 633 def _get_localized_dict(app, locale):
 634     '''get the dict to add localized store metadata to'''
 635     if 'localized' not in app:
 636         app['localized'] = collections.OrderedDict()
 637     if locale not in app['localized']:
 638         app['localized'][locale] = collections.OrderedDict()
 639     return app['localized'][locale]
 640
 641
 642 def _set_localized_text_entry(app, locale, key, f):
 643     limit = config['char_limits'][key]
 644     localized = _get_localized_dict(app, locale)
 645     with open(f) as fp:
 646         text = fp.read()[:limit]
 647         if len(text) > 0:
 648             localized[key] = text
 649
 650
 651 def _set_author_entry(app, key, f):
 652     limit = config['char_limits']['author']
 653     with open(f) as fp:
 654         text = fp.read()[:limit]
 655         if len(text) > 0:
 656             app[key] = text
 657
 658
 659 def copy_triple_t_store_metadata(apps):
 660     """Include store metadata from the app's source repo
 661
 662     The Triple-T Gradle Play Publisher is a plugin that has a standard
 663     file layout for all of the metadata and graphics that the Google
 664     Play Store accepts.  Since F-Droid has the git repo, it can just
 665     pluck those files directly.  This method reads any text files into
 666     the app dict, then copies any graphics into the fdroid repo
 667     directory structure.
 668
 669     This needs to be run before insert_localized_app_metadata() so that
 670     the graphics files that are copied into the fdroid repo get
 671     properly indexed.
 672
 673     https://github.com/Triple-T/gradle-play-publisher#upload-images
 674     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 675
 676     """
 677
 678     if not os.path.isdir('build'):
 679         return  # nothing to do
 680
 681     for packageName, app in apps.items():
 682         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 683             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 684             for root, dirs, files in os.walk(d):
 685                 segments = root.split('/')
 686                 locale = segments[-2]
 687                 for f in files:
 688                     if f == 'fulldescription':
 689                         _set_localized_text_entry(app, locale, 'description',
 690                                                   os.path.join(root, f))
 691                         continue
 692                     elif f == 'shortdescription':
 693                         _set_localized_text_entry(app, locale, 'summary',
 694                                                   os.path.join(root, f))
 695                         continue
 696                     elif f == 'title':
 697                         _set_localized_text_entry(app, locale, 'name',
 698                                                   os.path.join(root, f))
 699                         continue
 700                     elif f == 'video':
 701                         _set_localized_text_entry(app, locale, 'video',
 702                                                   os.path.join(root, f))
 703                         continue
 704                     elif f == 'whatsnew':
 705                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 706                                                   os.path.join(root, f))
 707                         continue
 708                     elif f == 'contactEmail':
 709                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 710                         continue
 711                     elif f == 'contactPhone':
 712                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 713                         continue
 714                     elif f == 'contactWebsite':
 715                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 716                         continue
 717
 718                     base, extension = common.get_extension(f)
 719                     dirname = os.path.basename(root)
 720                     if extension in ALLOWED_EXTENSIONS \
 721                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 722                         if segments[-2] == 'listing':
 723                             locale = segments[-3]
 724                         else:
 725                             locale = segments[-2]
 726                         destdir = os.path.join('repo', packageName, locale, dirname)
 727                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 728                         sourcefile = os.path.join(root, f)
 729                         destfile = os.path.join(destdir, os.path.basename(f))
 730                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 731                         shutil.copy(sourcefile, destfile)
 732
 733
 734 def insert_localized_app_metadata(apps):
 735     """scans standard locations for graphics and localized text
 736
 737     Scans for localized description files, store graphics, and
 738     screenshot PNG files in statically defined screenshots directory
 739     and adds them to the app metadata.  The screenshots and graphic
 740     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 741     and must be in the following layout:
 742     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 743
 744     repo/packageName/locale/featureGraphic.png
 745     repo/packageName/locale/phoneScreenshots/1.png
 746     repo/packageName/locale/phoneScreenshots/2.png
 747
 748     The changelog files must be text files named with the versionCode
 749     ending with ".txt" and must be in the following layout:
 750     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 751
 752     repo/packageName/locale/changelogs/12345.txt
 753
 754     This will scan the each app's source repo then the metadata/ dir
 755     for these standard locations of changelog files.  If it finds
 756     them, they will be added to the dict of all packages, with the
 757     versions in the metadata/ folder taking precendence over the what
 758     is in the app's source repo.
 759
 760     Where "packageName" is the app's packageName and "locale" is the locale
 761     of the graphics, e.g. what language they are in, using the IETF RFC5646
 762     format (en-US, fr-CA, es-MX, etc).
 763
 764     This will also scan the app's git for a fastlane folder, and the
 765     metadata/ folder and the apps' source repos for standard locations
 766     of graphic and screenshot files.  If it finds them, it will copy
 767     them into the repo.  The fastlane files follow this pattern:
 768     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 769
 770     """
 771
 772     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 773     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 774     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 775
 776     for srcd in sorted(sourcedirs):
 777         if not os.path.isdir(srcd):
 778             continue
 779         for root, dirs, files in os.walk(srcd):
 780             segments = root.split('/')
 781             packageName = segments[1]
 782             if packageName not in apps:
 783                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 784                 continue
 785             locale = segments[-1]
 786             destdir = os.path.join('repo', packageName, locale)
 787             for f in files:
 788                 if f in ('description.txt', 'full_description.txt'):
 789                     _set_localized_text_entry(apps[packageName], locale, 'description',
 790                                               os.path.join(root, f))
 791                     continue
 792                 elif f in ('summary.txt', 'short_description.txt'):
 793                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 794                                               os.path.join(root, f))
 795                     continue
 796                 elif f in ('name.txt', 'title.txt'):
 797                     _set_localized_text_entry(apps[packageName], locale, 'name',
 798                                               os.path.join(root, f))
 799                     continue
 800                 elif f == 'video.txt':
 801                     _set_localized_text_entry(apps[packageName], locale, 'video',
 802                                               os.path.join(root, f))
 803                     continue
 804                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 805                     locale = segments[-2]
 806                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 807                                               os.path.join(root, f))
 808                     continue
 809
 810                 base, extension = common.get_extension(f)
 811                 if locale == 'images':
 812                     locale = segments[-2]
 813                     destdir = os.path.join('repo', packageName, locale)
 814                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 815                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 816                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 817                     shutil.copy(os.path.join(root, f), destdir)
 818             for d in dirs:
 819                 if d in SCREENSHOT_DIRS:
 820                     if locale == 'images':
 821                         locale = segments[-2]
 822                         destdir = os.path.join('repo', packageName, locale)
 823                     for f in glob.glob(os.path.join(root, d, '*.*')):
 824                         _, extension = common.get_extension(f)
 825                         if extension in ALLOWED_EXTENSIONS:
 826                             screenshotdestdir = os.path.join(destdir, d)
 827                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 828                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 829                             shutil.copy(f, screenshotdestdir)
 830
 831     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 832     for d in repofiles:
 833         if not os.path.isdir(d):
 834             continue
 835         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 836             if not os.path.isfile(f):
 837                 continue
 838             segments = f.split('/')
 839             packageName = segments[1]
 840             locale = segments[2]
 841             screenshotdir = segments[3]
 842             filename = os.path.basename(f)
 843             base, extension = common.get_extension(filename)
 844
 845             if packageName not in apps:
 846                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 847                                 % (filename, packageName))
 848                 continue
 849             graphics = _get_localized_dict(apps[packageName], locale)
 850
 851             if extension not in ALLOWED_EXTENSIONS:
 852                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 853             elif base in GRAPHIC_NAMES:
 854                 # there can only be zero or one of these per locale
 855                 graphics[base] = filename
 856             elif screenshotdir in SCREENSHOT_DIRS:
 857                 # there can any number of these per locale
 858                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 859                 if screenshotdir not in graphics:
 860                     graphics[screenshotdir] = []
 861                 graphics[screenshotdir].append(filename)
 862             else:
 863                 logging.warning('Unsupported graphics file found: ' + f)
 864
 865
 866 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 867     """Scan a repo for all files with an extension except APK/OBB
 868
 869     :param apkcache: current cached info about all repo files
 870     :param repodir: repo directory to scan
 871     :param knownapks: list of all known files, as per metadata.read_metadata
 872     :param use_date_from_file: use date from file (instead of current date)
 873                                for newly added files
 874     """
 875
 876     cachechanged = False
 877     repo_files = []
 878     repodir = repodir.encode('utf-8')
 879     for name in os.listdir(repodir):
 880         file_extension = common.get_file_extension(name)
 881         if file_extension == 'apk' or file_extension == 'obb':
 882             continue
 883         filename = os.path.join(repodir, name)
 884         name_utf8 = name.decode('utf-8')
 885         if filename.endswith(b'_src.tar.gz'):
 886             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 887             continue
 888         if not common.is_repo_file(filename):
 889             continue
 890         stat = os.stat(filename)
 891         if stat.st_size == 0:
 892             raise FDroidException(filename + ' is zero size!')
 893
 894         shasum = sha256sum(filename)
 895         usecache = False
 896         if name in apkcache:
 897             repo_file = apkcache[name]
 898             # added time is cached as tuple but used here as datetime instance
 899             if 'added' in repo_file:
 900                 a = repo_file['added']
 901                 if isinstance(a, datetime):
 902                     repo_file['added'] = a
 903                 else:
 904                     repo_file['added'] = datetime(*a[:6])
 905             if repo_file.get('hash') == shasum:
 906                 logging.debug("Reading " + name_utf8 + " from cache")
 907                 usecache = True
 908             else:
 909                 logging.debug("Ignoring stale cache data for " + name_utf8)
 910
 911         if not usecache:
 912             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 913             repo_file = collections.OrderedDict()
 914             repo_file['name'] = os.path.splitext(name_utf8)[0]
 915             # TODO rename apkname globally to something more generic
 916             repo_file['apkName'] = name_utf8
 917             repo_file['hash'] = shasum
 918             repo_file['hashType'] = 'sha256'
 919             repo_file['versionCode'] = 0
 920             repo_file['versionName'] = shasum
 921             # the static ID is the SHA256 unless it is set in the metadata
 922             repo_file['packageName'] = shasum
 923
 924             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 925             if m:
 926                 repo_file['packageName'] = m.group(1)
 927                 repo_file['versionCode'] = int(m.group(2))
 928             srcfilename = name + b'_src.tar.gz'
 929             if os.path.exists(os.path.join(repodir, srcfilename)):
 930                 repo_file['srcname'] = srcfilename.decode('utf-8')
 931             repo_file['size'] = stat.st_size
 932
 933             apkcache[name] = repo_file
 934             cachechanged = True
 935
 936         if use_date_from_file:
 937             timestamp = stat.st_ctime
 938             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
 939         else:
 940             default_date_param = None
 941
 942         # Record in knownapks, getting the added date at the same time..
 943         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 944                                     default_date=default_date_param)
 945         if added:
 946             repo_file['added'] = added
 947
 948         repo_files.append(repo_file)
 949
 950     return repo_files, cachechanged
 951
 952
 953 def scan_apk(apk_file):
 954     """
 955     Scans an APK file and returns dictionary with metadata of the APK.
 956
 957     Attention: This does *not* verify that the APK signature is correct.
 958
 959     :param apk_file: The (ideally absolute) path to the APK file
 960     :raises BuildException
 961     :return A dict containing APK metadata
 962     """
 963     apk = {
 964         'hash': sha256sum(apk_file),
 965         'hashType': 'sha256',
 966         'uses-permission': [],
 967         'uses-permission-sdk-23': [],
 968         'features': [],
 969         'icons_src': {},
 970         'icons': {},
 971         'antiFeatures': set(),
 972     }
 973
 974     if SdkToolsPopen(['aapt', 'version'], output=False):
 975         scan_apk_aapt(apk, apk_file)
 976     else:
 977         scan_apk_androguard(apk, apk_file)
 978
 979     # Get the signature, or rather the signing key fingerprints
 980     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 981     apk['sig'] = getsig(apk_file)
 982     if not apk['sig']:
 983         raise BuildException("Failed to get apk signature")
 984     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
 985                                                                apk_file))
 986     if not apk.get('signer'):
 987         raise BuildException("Failed to get apk signing key fingerprint")
 988
 989     # Get size of the APK
 990     apk['size'] = os.path.getsize(apk_file)
 991
 992     if 'minSdkVersion' not in apk:
 993         logging.warning("No SDK version information found in {0}".format(apk_file))
 994         apk['minSdkVersion'] = 1
 995
 996     # Check for known vulnerabilities
 997     if has_known_vulnerability(apk_file):
 998         apk['antiFeatures'].add('KnownVuln')
 999
1000     return apk
1001
1002
1003 def scan_apk_aapt(apk, apkfile):
1004     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1005     if p.returncode != 0:
1006         if options.delete_unknown:
1007             if os.path.exists(apkfile):
1008                 logging.error("Failed to get apk information, deleting " + apkfile)
1009                 os.remove(apkfile)
1010             else:
1011                 logging.error("Could not find {0} to remove it".format(apkfile))
1012         else:
1013             logging.error("Failed to get apk information, skipping " + apkfile)
1014         raise BuildException("Invalid APK")
1015     for line in p.output.splitlines():
1016         if line.startswith("package:"):
1017             try:
1018                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1019                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1020                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1021             except Exception as e:
1022                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1023         elif line.startswith("application:"):
1024             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1025             # Keep path to non-dpi icon in case we need it
1026             match = re.match(APK_ICON_PAT_NODPI, line)
1027             if match:
1028                 apk['icons_src']['-1'] = match.group(1)
1029         elif line.startswith("launchable-activity:"):
1030             # Only use launchable-activity as fallback to application
1031             if not apk['name']:
1032                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1033             if '-1' not in apk['icons_src']:
1034                 match = re.match(APK_ICON_PAT_NODPI, line)
1035                 if match:
1036                     apk['icons_src']['-1'] = match.group(1)
1037         elif line.startswith("application-icon-"):
1038             match = re.match(APK_ICON_PAT, line)
1039             if match:
1040                 density = match.group(1)
1041                 path = match.group(2)
1042                 apk['icons_src'][density] = path
1043         elif line.startswith("sdkVersion:"):
1044             m = re.match(APK_SDK_VERSION_PAT, line)
1045             if m is None:
1046                 logging.error(line.replace('sdkVersion:', '')
1047                               + ' is not a valid minSdkVersion!')
1048             else:
1049                 apk['minSdkVersion'] = m.group(1)
1050                 # if target not set, default to min
1051                 if 'targetSdkVersion' not in apk:
1052                     apk['targetSdkVersion'] = m.group(1)
1053         elif line.startswith("targetSdkVersion:"):
1054             m = re.match(APK_SDK_VERSION_PAT, line)
1055             if m is None:
1056                 logging.error(line.replace('targetSdkVersion:', '')
1057                               + ' is not a valid targetSdkVersion!')
1058             else:
1059                 apk['targetSdkVersion'] = m.group(1)
1060         elif line.startswith("maxSdkVersion:"):
1061             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1062         elif line.startswith("native-code:"):
1063             apk['nativecode'] = []
1064             for arch in line[13:].split(' '):
1065                 apk['nativecode'].append(arch[1:-1])
1066         elif line.startswith('uses-permission:'):
1067             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1068             if perm_match['maxSdkVersion']:
1069                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1070             permission = UsesPermission(
1071                 perm_match['name'],
1072                 perm_match['maxSdkVersion']
1073             )
1074
1075             apk['uses-permission'].append(permission)
1076         elif line.startswith('uses-permission-sdk-23:'):
1077             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1078             if perm_match['maxSdkVersion']:
1079                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1080             permission_sdk_23 = UsesPermissionSdk23(
1081                 perm_match['name'],
1082                 perm_match['maxSdkVersion']
1083             )
1084
1085             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1086
1087         elif line.startswith('uses-feature:'):
1088             feature = re.match(APK_FEATURE_PAT, line).group(1)
1089             # Filter out this, it's only added with the latest SDK tools and
1090             # causes problems for lots of apps.
1091             if feature != "android.hardware.screen.portrait" \
1092                     and feature != "android.hardware.screen.landscape":
1093                 if feature.startswith("android.feature."):
1094                     feature = feature[16:]
1095                 apk['features'].add(feature)
1096
1097
1098 def scan_apk_androguard(apk, apkfile):
1099     try:
1100         from androguard.core.bytecodes.apk import APK
1101         apkobject = APK(apkfile)
1102         if apkobject.is_valid_APK():
1103             arsc = apkobject.get_android_resources()
1104         else:
1105             if options.delete_unknown:
1106                 if os.path.exists(apkfile):
1107                     logging.error("Failed to get apk information, deleting " + apkfile)
1108                     os.remove(apkfile)
1109                 else:
1110                     logging.error("Could not find {0} to remove it".format(apkfile))
1111             else:
1112                 logging.error("Failed to get apk information, skipping " + apkfile)
1113             raise BuildException("Invaild APK")
1114     except ImportError:
1115         raise FDroidException("androguard library is not installed and aapt not present")
1116     except FileNotFoundError:
1117         logging.error("Could not open apk file for analysis")
1118         raise BuildException("Invalid APK")
1119
1120     apk['packageName'] = apkobject.get_package()
1121     apk['versionCode'] = int(apkobject.get_androidversion_code())
1122     apk['versionName'] = apkobject.get_androidversion_name()
1123     if apk['versionName'][0] == "@":
1124         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1125         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1126         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1127     apk['name'] = apkobject.get_app_name()
1128
1129     if apkobject.get_max_sdk_version() is not None:
1130         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1131     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1132     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1133
1134     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1135     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1136
1137     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1138
1139     for file in apkobject.get_files():
1140         d_re = density_re.match(file)
1141         if d_re:
1142             folder = d_re.group(1).split('-')
1143             if len(folder) > 1:
1144                 resolution = folder[1]
1145             else:
1146                 resolution = 'mdpi'
1147             density = screen_resolutions[resolution]
1148             apk['icons_src'][density] = d_re.group(0)
1149
1150     if apk['icons_src'].get('-1') is None:
1151         apk['icons_src']['-1'] = apk['icons_src']['160']
1152
1153     arch_re = re.compile("^lib/(.*)/.*$")
1154     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1155     if len(arch) >= 1:
1156         apk['nativecode'] = []
1157         apk['nativecode'].extend(sorted(list(arch)))
1158
1159     xml = apkobject.get_android_manifest_xml()
1160
1161     for item in xml.getElementsByTagName('uses-permission'):
1162         name = str(item.getAttribute("android:name"))
1163         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1164         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1165         permission = UsesPermission(
1166             name,
1167             maxSdkVersion
1168         )
1169         apk['uses-permission'].append(permission)
1170
1171     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1172         name = str(item.getAttribute("android:name"))
1173         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1174         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1175         permission_sdk_23 = UsesPermissionSdk23(
1176             name,
1177             maxSdkVersion
1178         )
1179         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1180
1181     for item in xml.getElementsByTagName('uses-feature'):
1182         feature = str(item.getAttribute("android:name"))
1183         if feature != "android.hardware.screen.portrait" \
1184                 and feature != "android.hardware.screen.landscape":
1185             if feature.startswith("android.feature."):
1186                 feature = feature[16:]
1187         apk['features'].append(feature)
1188
1189
1190 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1191                 allow_disabled_algorithms=False, archive_bad_sig=False):
1192     """Processes the apk with the given filename in the given repo directory.
1193
1194     This also extracts the icons.
1195
1196     :param apkcache: current apk cache information
1197     :param apkfilename: the filename of the apk to scan
1198     :param repodir: repo directory to scan
1199     :param knownapks: known apks info
1200     :param use_date_from_apk: use date from APK (instead of current date)
1201                               for newly added APKs
1202     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1203                                       disabled algorithms in the signature (e.g. MD5)
1204     :param archive_bad_sig: move APKs with a bad signature to the archive
1205     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1206      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1207     """
1208
1209     apk = {}
1210     apkfile = os.path.join(repodir, apkfilename)
1211
1212     cachechanged = False
1213     usecache = False
1214     if apkfilename in apkcache:
1215         apk = apkcache[apkfilename]
1216         if apk.get('hash') == sha256sum(apkfile):
1217             logging.debug("Reading " + apkfilename + " from cache")
1218             usecache = True
1219         else:
1220             logging.debug("Ignoring stale cache data for " + apkfilename)
1221
1222     if not usecache:
1223         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1224
1225         try:
1226             apk = scan_apk(apkfile)
1227         except BuildException:
1228             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1229                             .format(apkfilename=apkfilename))
1230             return True, None, False
1231
1232         # Check for debuggable apks...
1233         if common.isApkAndDebuggable(apkfile):
1234             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1235
1236         if options.rename_apks:
1237             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1238             std_short_name = os.path.join(repodir, n)
1239             if apkfile != std_short_name:
1240                 if os.path.exists(std_short_name):
1241                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1242                     if apkfile != std_long_name:
1243                         if os.path.exists(std_long_name):
1244                             dupdir = os.path.join('duplicates', repodir)
1245                             if not os.path.isdir(dupdir):
1246                                 os.makedirs(dupdir, exist_ok=True)
1247                             dupfile = os.path.join('duplicates', std_long_name)
1248                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1249                             os.rename(apkfile, dupfile)
1250                             return True, None, False
1251                         else:
1252                             os.rename(apkfile, std_long_name)
1253                     apkfile = std_long_name
1254                 else:
1255                     os.rename(apkfile, std_short_name)
1256                     apkfile = std_short_name
1257                 apkfilename = apkfile[len(repodir) + 1:]
1258
1259         apk['apkName'] = apkfilename
1260         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1261         if os.path.exists(os.path.join(repodir, srcfilename)):
1262             apk['srcname'] = srcfilename
1263
1264         # verify the jar signature is correct, allow deprecated
1265         # algorithms only if the APK is in the archive.
1266         skipapk = False
1267         if not common.verify_apk_signature(apkfile):
1268             if repodir == 'archive' or allow_disabled_algorithms:
1269                 if common.verify_old_apk_signature(apkfile):
1270                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1271                 else:
1272                     skipapk = True
1273             else:
1274                 skipapk = True
1275
1276         if skipapk:
1277             if archive_bad_sig:
1278                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1279                 move_apk_between_sections(repodir, 'archive', apk)
1280             else:
1281                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1282             return True, None, False
1283
1284         apkzip = zipfile.ZipFile(apkfile, 'r')
1285
1286         # if an APK has files newer than the system time, suggest updating
1287         # the system clock.  This is useful for offline systems, used for
1288         # signing, which do not have another source of clock sync info. It
1289         # has to be more than 24 hours newer because ZIP/APK files do not
1290         # store timezone info
1291         manifest = apkzip.getinfo('AndroidManifest.xml')
1292         if manifest.date_time[1] == 0:  # month can't be zero
1293             logging.debug('AndroidManifest.xml has no date')
1294         else:
1295             dt_obj = datetime(*manifest.date_time)
1296             checkdt = dt_obj - timedelta(1)
1297             if datetime.today() < checkdt:
1298                 logging.warning('System clock is older than manifest in: '
1299                                 + apkfilename
1300                                 + '\nSet clock to that time using:\n'
1301                                 + 'sudo date -s "' + str(dt_obj) + '"')
1302
1303         # extract icons from APK zip file
1304         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1305         try:
1306             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1307         finally:
1308             apkzip.close()  # ensure that APK zip file gets closed
1309
1310         # resize existing icons for densities missing in the APK
1311         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1312
1313         if use_date_from_apk and manifest.date_time[1] != 0:
1314             default_date_param = datetime(*manifest.date_time)
1315         else:
1316             default_date_param = None
1317
1318         # Record in known apks, getting the added date at the same time..
1319         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1320                                     default_date=default_date_param)
1321         if added:
1322             apk['added'] = added
1323
1324         apkcache[apkfilename] = apk
1325         cachechanged = True
1326
1327     return False, apk, cachechanged
1328
1329
1330 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1331     """Processes the apks in the given repo directory.
1332
1333     This also extracts the icons.
1334
1335     :param apkcache: current apk cache information
1336     :param repodir: repo directory to scan
1337     :param knownapks: known apks info
1338     :param use_date_from_apk: use date from APK (instead of current date)
1339                               for newly added APKs
1340     :returns: (apks, cachechanged) where apks is a list of apk information,
1341               and cachechanged is True if the apkcache got changed.
1342     """
1343
1344     cachechanged = False
1345
1346     for icon_dir in get_all_icon_dirs(repodir):
1347         if os.path.exists(icon_dir):
1348             if options.clean:
1349                 shutil.rmtree(icon_dir)
1350                 os.makedirs(icon_dir)
1351         else:
1352             os.makedirs(icon_dir)
1353
1354     apks = []
1355     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1356         apkfilename = apkfile[len(repodir) + 1:]
1357         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1358         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1359                                              use_date_from_apk, ada, True)
1360         if skip:
1361             continue
1362         apks.append(apk)
1363         cachechanged = cachechanged or cachethis
1364
1365     return apks, cachechanged
1366
1367
1368 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1369     """
1370     Extracts icons from the given APK zip in various densities,
1371     saves them into given repo directory
1372     and stores their names in the APK metadata dictionary.
1373
1374     :param icon_filename: A string representing the icon's file name
1375     :param apk: A populated dictionary containing APK metadata.
1376                 Needs to have 'icons_src' key
1377     :param apkzip: An opened zipfile.ZipFile of the APK file
1378     :param repo_dir: The directory of the APK's repository
1379     :return: A list of icon densities that are missing
1380     """
1381     empty_densities = []
1382     for density in screen_densities:
1383         if density not in apk['icons_src']:
1384             empty_densities.append(density)
1385             continue
1386         icon_src = apk['icons_src'][density]
1387         icon_dir = get_icon_dir(repo_dir, density)
1388         icon_dest = os.path.join(icon_dir, icon_filename)
1389
1390         # Extract the icon files per density
1391         if icon_src.endswith('.xml'):
1392             png = os.path.basename(icon_src)[:-4] + '.png'
1393             for f in apkzip.namelist():
1394                 if f.endswith(png):
1395                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1396                     if m and screen_resolutions[m.group(2)] == density:
1397                         icon_src = f
1398             if icon_src.endswith('.xml'):
1399                 empty_densities.append(density)
1400                 continue
1401         try:
1402             with open(icon_dest, 'wb') as f:
1403                 f.write(get_icon_bytes(apkzip, icon_src))
1404             apk['icons'][density] = icon_filename
1405         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1406             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1407             del apk['icons_src'][density]
1408             empty_densities.append(density)
1409
1410     if '-1' in apk['icons_src']:
1411         icon_src = apk['icons_src']['-1']
1412         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1413         with open(icon_path, 'wb') as f:
1414             f.write(get_icon_bytes(apkzip, icon_src))
1415         try:
1416             im = Image.open(icon_path)
1417             dpi = px_to_dpi(im.size[0])
1418             for density in screen_densities:
1419                 if density in apk['icons']:
1420                     break
1421                 if density == screen_densities[-1] or dpi >= int(density):
1422                     apk['icons'][density] = icon_filename
1423                     shutil.move(icon_path,
1424                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1425                     empty_densities.remove(density)
1426                     break
1427         except Exception as e:
1428             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1429
1430     if apk['icons']:
1431         apk['icon'] = icon_filename
1432
1433     return empty_densities
1434
1435
1436 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1437     """
1438     Resize existing icons for densities missing in the APK to ensure all densities are available
1439
1440     :param empty_densities: A list of icon densities that are missing
1441     :param icon_filename: A string representing the icon's file name
1442     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1443     :param repo_dir: The directory of the APK's repository
1444     """
1445     # First try resizing down to not lose quality
1446     last_density = None
1447     for density in screen_densities:
1448         if density not in empty_densities:
1449             last_density = density
1450             continue
1451         if last_density is None:
1452             continue
1453         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1454
1455         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1456         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1457         fp = None
1458         try:
1459             fp = open(last_icon_path, 'rb')
1460             im = Image.open(fp)
1461
1462             size = dpi_to_px(density)
1463
1464             im.thumbnail((size, size), Image.ANTIALIAS)
1465             im.save(icon_path, "PNG")
1466             empty_densities.remove(density)
1467         except Exception as e:
1468             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1469         finally:
1470             if fp:
1471                 fp.close()
1472
1473     # Then just copy from the highest resolution available
1474     last_density = None
1475     for density in reversed(screen_densities):
1476         if density not in empty_densities:
1477             last_density = density
1478             continue
1479
1480         if last_density is None:
1481             continue
1482
1483         shutil.copyfile(
1484             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1485             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1486         )
1487         empty_densities.remove(density)
1488
1489     for density in screen_densities:
1490         icon_dir = get_icon_dir(repo_dir, density)
1491         icon_dest = os.path.join(icon_dir, icon_filename)
1492         resize_icon(icon_dest, density)
1493
1494     # Copy from icons-mdpi to icons since mdpi is the baseline density
1495     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1496     if os.path.isfile(baseline):
1497         apk['icons']['0'] = icon_filename
1498         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1499
1500
1501 def apply_info_from_latest_apk(apps, apks):
1502     """
1503     Some information from the apks needs to be applied up to the application level.
1504     When doing this, we use the info from the most recent version's apk.
1505     We deal with figuring out when the app was added and last updated at the same time.
1506     """
1507     for appid, app in apps.items():
1508         bestver = UNSET_VERSION_CODE
1509         for apk in apks:
1510             if apk['packageName'] == appid:
1511                 if apk['versionCode'] > bestver:
1512                     bestver = apk['versionCode']
1513                     bestapk = apk
1514
1515                 if 'added' in apk:
1516                     if not app.added or apk['added'] < app.added:
1517                         app.added = apk['added']
1518                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1519                         app.lastUpdated = apk['added']
1520
1521         if not app.added:
1522             logging.debug("Don't know when " + appid + " was added")
1523         if not app.lastUpdated:
1524             logging.debug("Don't know when " + appid + " was last updated")
1525
1526         if bestver == UNSET_VERSION_CODE:
1527
1528             if app.Name is None:
1529                 app.Name = app.AutoName or appid
1530             app.icon = None
1531             logging.debug("Application " + appid + " has no packages")
1532         else:
1533             if app.Name is None:
1534                 app.Name = bestapk['name']
1535             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1536             if app.CurrentVersionCode is None:
1537                 app.CurrentVersionCode = str(bestver)
1538
1539
1540 def make_categories_txt(repodir, categories):
1541     '''Write a category list in the repo to allow quick access'''
1542     catdata = ''
1543     for cat in sorted(categories):
1544         catdata += cat + '\n'
1545     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1546         f.write(catdata)
1547
1548
1549 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1550
1551     def filter_apk_list_sorted(apk_list):
1552         res = []
1553         for apk in apk_list:
1554             if apk['packageName'] == appid:
1555                 res.append(apk)
1556
1557         # Sort the apk list by version code. First is highest/newest.
1558         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1559
1560     for appid, app in apps.items():
1561
1562         if app.ArchivePolicy:
1563             keepversions = int(app.ArchivePolicy[:-9])
1564         else:
1565             keepversions = defaultkeepversions
1566
1567         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1568                       .format(appid, len(apks), keepversions, len(archapks)))
1569
1570         current_app_apks = filter_apk_list_sorted(apks)
1571         if len(current_app_apks) > keepversions:
1572             # Move back the ones we don't want.
1573             for apk in current_app_apks[keepversions:]:
1574                 move_apk_between_sections(repodir, archivedir, apk)
1575                 archapks.append(apk)
1576                 apks.remove(apk)
1577
1578         current_app_archapks = filter_apk_list_sorted(archapks)
1579         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1580             kept = 0
1581             # Move forward the ones we want again, except DisableAlgorithm
1582             for apk in current_app_archapks:
1583                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1584                     move_apk_between_sections(archivedir, repodir, apk)
1585                     archapks.remove(apk)
1586                     apks.append(apk)
1587                     kept += 1
1588                 if kept == keepversions:
1589                     break
1590
1591
1592 def move_apk_between_sections(from_dir, to_dir, apk):
1593     """move an APK from repo to archive or vice versa"""
1594
1595     def _move_file(from_dir, to_dir, filename, ignore_missing):
1596         from_path = os.path.join(from_dir, filename)
1597         if ignore_missing and not os.path.exists(from_path):
1598             return
1599         to_path = os.path.join(to_dir, filename)
1600         if not os.path.exists(to_dir):
1601             os.mkdir(to_dir)
1602         shutil.move(from_path, to_path)
1603
1604     if from_dir == to_dir:
1605         return
1606
1607     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1608     _move_file(from_dir, to_dir, apk['apkName'], False)
1609     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1610     for density in all_screen_densities:
1611         from_icon_dir = get_icon_dir(from_dir, density)
1612         to_icon_dir = get_icon_dir(to_dir, density)
1613         if density not in apk['icons']:
1614             continue
1615         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1616     if 'srcname' in apk:
1617         _move_file(from_dir, to_dir, apk['srcname'], False)
1618
1619
1620 def add_apks_to_per_app_repos(repodir, apks):
1621     apks_per_app = dict()
1622     for apk in apks:
1623         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1624         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1625         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1626         apks_per_app[apk['packageName']] = apk
1627
1628         if not os.path.exists(apk['per_app_icons']):
1629             logging.info('Adding new repo for only ' + apk['packageName'])
1630             os.makedirs(apk['per_app_icons'])
1631
1632         apkpath = os.path.join(repodir, apk['apkName'])
1633         shutil.copy(apkpath, apk['per_app_repo'])
1634         apksigpath = apkpath + '.sig'
1635         if os.path.exists(apksigpath):
1636             shutil.copy(apksigpath, apk['per_app_repo'])
1637         apkascpath = apkpath + '.asc'
1638         if os.path.exists(apkascpath):
1639             shutil.copy(apkascpath, apk['per_app_repo'])
1640
1641
1642 def create_metadata_from_template(apk):
1643     '''create a new metadata file using internal or external template
1644
1645     Generate warnings for apk's with no metadata (or create skeleton
1646     metadata files, if requested on the command line).  Though the
1647     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1648     since those impose things on the metadata file made from the
1649     template: field sort order, empty field value, formatting, etc.
1650     '''
1651
1652     import yaml
1653     if os.path.exists('template.yml'):
1654         with open('template.yml') as f:
1655             metatxt = f.read()
1656         if 'name' in apk and apk['name'] != '':
1657             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1658                              r'\1 ' + apk['name'],
1659                              metatxt,
1660                              flags=re.IGNORECASE | re.MULTILINE)
1661         else:
1662             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1663             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1664                              r'\1 ' + apk['packageName'],
1665                              metatxt,
1666                              flags=re.IGNORECASE | re.MULTILINE)
1667         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1668             f.write(metatxt)
1669     else:
1670         app = dict()
1671         app['Categories'] = [os.path.basename(os.getcwd())]
1672         # include some blanks as part of the template
1673         app['AuthorName'] = ''
1674         app['Summary'] = ''
1675         app['WebSite'] = ''
1676         app['IssueTracker'] = ''
1677         app['SourceCode'] = ''
1678         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1679         if 'name' in apk and apk['name'] != '':
1680             app['Name'] = apk['name']
1681         else:
1682             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1683             app['Name'] = apk['packageName']
1684         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1685             yaml.dump(app, f, default_flow_style=False)
1686     logging.info("Generated skeleton metadata for " + apk['packageName'])
1687
1688
1689 config = None
1690 options = None
1691
1692
1693 def main():
1694
1695     global config, options
1696
1697     # Parse command line...
1698     parser = ArgumentParser()
1699     common.setup_global_opts(parser)
1700     parser.add_argument("--create-key", action="store_true", default=False,
1701                         help=_("Create a repo signing key in a keystore"))
1702     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1703                         help=_("Create skeleton metadata files that are missing"))
1704     parser.add_argument("--delete-unknown", action="store_true", default=False,
1705                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1706     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1707                         help=_("Report on build data status"))
1708     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1709                         help=_("Interactively ask about things that need updating."))
1710     parser.add_argument("-I", "--icons", action="store_true", default=False,
1711                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1712     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1713                         help=_("Specify editor to use in interactive mode. Default ") +
1714                         "is /etc/alternatives/editor")
1715     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1716                         help=_("Update the wiki"))
1717     parser.add_argument("--pretty", action="store_true", default=False,
1718                         help=_("Produce human-readable index.xml"))
1719     parser.add_argument("--clean", action="store_true", default=False,
1720                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1721     parser.add_argument("--nosign", action="store_true", default=False,
1722                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1723     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1724                         help=_("Use date from APK instead of current time for newly added APKs"))
1725     parser.add_argument("--rename-apks", action="store_true", default=False,
1726                         help=_("Rename APK files that do not match package.name_123.apk"))
1727     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1728                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1729     metadata.add_metadata_arguments(parser)
1730     options = parser.parse_args()
1731     metadata.warnings_action = options.W
1732
1733     config = common.read_config(options)
1734
1735     if not ('jarsigner' in config and 'keytool' in config):
1736         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1737
1738     repodirs = ['repo']
1739     if config['archive_older'] != 0:
1740         repodirs.append('archive')
1741         if not os.path.exists('archive'):
1742             os.mkdir('archive')
1743
1744     if options.icons:
1745         resize_all_icons(repodirs)
1746         sys.exit(0)
1747
1748     if options.rename_apks:
1749         options.clean = True
1750
1751     # check that icons exist now, rather than fail at the end of `fdroid update`
1752     for k in ['repo_icon', 'archive_icon']:
1753         if k in config:
1754             if not os.path.exists(config[k]):
1755                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1756                 sys.exit(1)
1757
1758     # if the user asks to create a keystore, do it now, reusing whatever it can
1759     if options.create_key:
1760         if os.path.exists(config['keystore']):
1761             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1762             logging.critical("\t'" + config['keystore'] + "'")
1763             sys.exit(1)
1764
1765         if 'repo_keyalias' not in config:
1766             config['repo_keyalias'] = socket.getfqdn()
1767             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1768         if 'keydname' not in config:
1769             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1770             common.write_to_config(config, 'keydname', config['keydname'])
1771         if 'keystore' not in config:
1772             config['keystore'] = common.default_config['keystore']
1773             common.write_to_config(config, 'keystore', config['keystore'])
1774
1775         password = common.genpassword()
1776         if 'keystorepass' not in config:
1777             config['keystorepass'] = password
1778             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1779         if 'keypass' not in config:
1780             config['keypass'] = password
1781             common.write_to_config(config, 'keypass', config['keypass'])
1782         common.genkeystore(config)
1783
1784     # Get all apps...
1785     apps = metadata.read_metadata()
1786
1787     # Generate a list of categories...
1788     categories = set()
1789     for app in apps.values():
1790         categories.update(app.Categories)
1791
1792     # Read known apks data (will be updated and written back when we've finished)
1793     knownapks = common.KnownApks()
1794
1795     # Get APK cache
1796     apkcache = get_cache()
1797
1798     # Delete builds for disabled apps
1799     delete_disabled_builds(apps, apkcache, repodirs)
1800
1801     # Scan all apks in the main repo
1802     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1803
1804     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1805                                            options.use_date_from_apk)
1806     cachechanged = cachechanged or fcachechanged
1807     apks += files
1808     for apk in apks:
1809         if apk['packageName'] not in apps:
1810             if options.create_metadata:
1811                 create_metadata_from_template(apk)
1812                 apps = metadata.read_metadata()
1813             else:
1814                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1815                 if options.delete_unknown:
1816                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1817                     rmf = os.path.join(repodirs[0], apk['apkName'])
1818                     if not os.path.exists(rmf):
1819                         logging.error("Could not find {0} to remove it".format(rmf))
1820                     else:
1821                         os.remove(rmf)
1822                 else:
1823                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1824
1825     copy_triple_t_store_metadata(apps)
1826     insert_obbs(repodirs[0], apps, apks)
1827     insert_localized_app_metadata(apps)
1828     translate_per_build_anti_features(apps, apks)
1829
1830     # Scan the archive repo for apks as well
1831     if len(repodirs) > 1:
1832         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1833         if cc:
1834             cachechanged = True
1835     else:
1836         archapks = []
1837
1838     # Apply information from latest apks to the application and update dates
1839     apply_info_from_latest_apk(apps, apks + archapks)
1840
1841     # Sort the app list by name, then the web site doesn't have to by default.
1842     # (we had to wait until we'd scanned the apks to do this, because mostly the
1843     # name comes from there!)
1844     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1845
1846     # APKs are placed into multiple repos based on the app package, providing
1847     # per-app subscription feeds for nightly builds and things like it
1848     if config['per_app_repos']:
1849         add_apks_to_per_app_repos(repodirs[0], apks)
1850         for appid, app in apps.items():
1851             repodir = os.path.join(appid, 'fdroid', 'repo')
1852             appdict = dict()
1853             appdict[appid] = app
1854             if os.path.isdir(repodir):
1855                 index.make(appdict, [appid], apks, repodir, False)
1856             else:
1857                 logging.info('Skipping index generation for ' + appid)
1858         return
1859
1860     if len(repodirs) > 1:
1861         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1862
1863     # Make the index for the main repo...
1864     index.make(apps, sortedids, apks, repodirs[0], False)
1865     make_categories_txt(repodirs[0], categories)
1866
1867     # If there's an archive repo,  make the index for it. We already scanned it
1868     # earlier on.
1869     if len(repodirs) > 1:
1870         index.make(apps, sortedids, archapks, repodirs[1], True)
1871
1872     git_remote = config.get('binary_transparency_remote')
1873     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1874         from . import btlog
1875         btlog.make_binary_transparency_log(repodirs)
1876
1877     if config['update_stats']:
1878         # Update known apks info...
1879         knownapks.writeifchanged()
1880
1881         # Generate latest apps data for widget
1882         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1883             data = ''
1884             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1885                 for line in f:
1886                     appid = line.rstrip()
1887                     data += appid + "\t"
1888                     app = apps[appid]
1889                     data += app.Name + "\t"
1890                     if app.icon is not None:
1891                         data += app.icon + "\t"
1892                     data += app.License + "\n"
1893             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1894                 f.write(data)
1895
1896     if cachechanged:
1897         write_cache(apkcache)
1898
1899     # Update the wiki...
1900     if options.wiki:
1901         update_wiki(apps, sortedids, apks + archapks)
1902
1903     logging.info(_("Finished"))
1904
1905
1906 if __name__ == "__main__":
1907     main()