fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import common
  41 from . import index
  42 from . import metadata
  43 from .common import SdkToolsPopen
  44 from .exception import BuildException, FDroidException
  45
  46 METADATA_VERSION = 18
  47
  48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  49 UNSET_VERSION_CODE = -0x100000000
  50
  51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85
  86 def dpi_to_px(density):
  87     return (int(density) * 48) / 160
  88
  89
  90 def px_to_dpi(px):
  91     return (int(px) * 160) / 48
  92
  93
  94 def get_icon_dir(repodir, density):
  95     if density == '0':
  96         return os.path.join(repodir, "icons")
  97     return os.path.join(repodir, "icons-%s" % density)
  98
  99
 100 def get_icon_dirs(repodir):
 101     for density in screen_densities:
 102         yield get_icon_dir(repodir, density)
 103
 104
 105 def get_all_icon_dirs(repodir):
 106     for density in all_screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def update_wiki(apps, sortedids, apks):
 111     """Update the wiki
 112
 113     :param apps: fully populated list of all applications
 114     :param apks: all apks, except...
 115     """
 116     logging.info("Updating wiki")
 117     wikicat = 'Apps'
 118     wikiredircat = 'App Redirects'
 119     import mwclient
 120     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 121                          path=config['wiki_path'])
 122     site.login(config['wiki_user'], config['wiki_password'])
 123     generated_pages = {}
 124     generated_redirects = {}
 125
 126     for appid in sortedids:
 127         app = metadata.App(apps[appid])
 128
 129         wikidata = ''
 130         if app.Disabled:
 131             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 132         if app.AntiFeatures:
 133             for af in app.AntiFeatures:
 134                 wikidata += '{{AntiFeature|' + af + '}}\n'
 135         if app.RequiresRoot:
 136             requiresroot = 'Yes'
 137         else:
 138             requiresroot = 'No'
 139         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 140             appid,
 141             app.Name,
 142             app.added.strftime('%Y-%m-%d') if app.added else '',
 143             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 144             app.SourceCode,
 145             app.IssueTracker,
 146             app.WebSite,
 147             app.Changelog,
 148             app.Donate,
 149             app.FlattrID,
 150             app.Bitcoin,
 151             app.Litecoin,
 152             app.License,
 153             requiresroot,
 154             app.AuthorName,
 155             app.AuthorEmail)
 156
 157         if app.Provides:
 158             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 159
 160         wikidata += app.Summary
 161         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 162
 163         wikidata += "=Description=\n"
 164         wikidata += metadata.description_wiki(app.Description) + "\n"
 165
 166         wikidata += "=Maintainer Notes=\n"
 167         if app.MaintainerNotes:
 168             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 169         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 170
 171         # Get a list of all packages for this application...
 172         apklist = []
 173         gotcurrentver = False
 174         cantupdate = False
 175         buildfails = False
 176         for apk in apks:
 177             if apk['packageName'] == appid:
 178                 if str(apk['versionCode']) == app.CurrentVersionCode:
 179                     gotcurrentver = True
 180                 apklist.append(apk)
 181         # Include ones we can't build, as a special case...
 182         for build in app.builds:
 183             if build.disable:
 184                 if build.versionCode == app.CurrentVersionCode:
 185                     cantupdate = True
 186                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 187                 apklist.append({'versionCode': int(build.versionCode),
 188                                 'versionName': build.versionName,
 189                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 190                                 })
 191             else:
 192                 builtit = False
 193                 for apk in apklist:
 194                     if apk['versionCode'] == int(build.versionCode):
 195                         builtit = True
 196                         break
 197                 if not builtit:
 198                     buildfails = True
 199                     apklist.append({'versionCode': int(build.versionCode),
 200                                     'versionName': build.versionName,
 201                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 202                                     })
 203         if app.CurrentVersionCode == '0':
 204             cantupdate = True
 205         # Sort with most recent first...
 206         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 207
 208         wikidata += "=Versions=\n"
 209         if len(apklist) == 0:
 210             wikidata += "We currently have no versions of this app available."
 211         elif not gotcurrentver:
 212             wikidata += "We don't have the current version of this app."
 213         else:
 214             wikidata += "We have the current version of this app."
 215         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 216         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 217         if len(app.NoSourceSince) > 0:
 218             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 219         if len(app.CurrentVersion) > 0:
 220             wikidata += "The current (recommended) version is " + app.CurrentVersion
 221             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 222         validapks = 0
 223         for apk in apklist:
 224             wikidata += "==" + apk['versionName'] + "==\n"
 225
 226             if 'buildproblem' in apk:
 227                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 228             else:
 229                 validapks += 1
 230                 wikidata += "This version is built and signed by "
 231                 if 'srcname' in apk:
 232                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 233                 else:
 234                     wikidata += "the original developer.\n\n"
 235             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 236
 237         wikidata += '\n[[Category:' + wikicat + ']]\n'
 238         if len(app.NoSourceSince) > 0:
 239             wikidata += '\n[[Category:Apps missing source code]]\n'
 240         if validapks == 0 and not app.Disabled:
 241             wikidata += '\n[[Category:Apps with no packages]]\n'
 242         if cantupdate and not app.Disabled:
 243             wikidata += "\n[[Category:Apps we cannot update]]\n"
 244         if buildfails and not app.Disabled:
 245             wikidata += "\n[[Category:Apps with failing builds]]\n"
 246         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 247             wikidata += '\n[[Category:Apps to Update]]\n'
 248         if app.Disabled:
 249             wikidata += '\n[[Category:Apps that are disabled]]\n'
 250         if app.UpdateCheckMode == 'None' and not app.Disabled:
 251             wikidata += '\n[[Category:Apps with no update check]]\n'
 252         for appcat in app.Categories:
 253             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 254
 255         # We can't have underscores in the page name, even if they're in
 256         # the package ID, because MediaWiki messes with them...
 257         pagename = appid.replace('_', ' ')
 258
 259         # Drop a trailing newline, because mediawiki is going to drop it anyway
 260         # and it we don't we'll think the page has changed when it hasn't...
 261         if wikidata.endswith('\n'):
 262             wikidata = wikidata[:-1]
 263
 264         generated_pages[pagename] = wikidata
 265
 266         # Make a redirect from the name to the ID too, unless there's
 267         # already an existing page with the name and it isn't a redirect.
 268         noclobber = False
 269         apppagename = app.Name.replace('_', ' ')
 270         apppagename = apppagename.replace('{', '')
 271         apppagename = apppagename.replace('}', ' ')
 272         apppagename = apppagename.replace(':', ' ')
 273         apppagename = apppagename.replace('[', ' ')
 274         apppagename = apppagename.replace(']', ' ')
 275         # Drop double spaces caused mostly by replacing ':' above
 276         apppagename = apppagename.replace('  ', ' ')
 277         for expagename in site.allpages(prefix=apppagename,
 278                                         filterredir='nonredirects',
 279                                         generator=False):
 280             if expagename == apppagename:
 281                 noclobber = True
 282         # Another reason not to make the redirect page is if the app name
 283         # is the same as it's ID, because that will overwrite the real page
 284         # with an redirect to itself! (Although it seems like an odd
 285         # scenario this happens a lot, e.g. where there is metadata but no
 286         # builds or binaries to extract a name from.
 287         if apppagename == pagename:
 288             noclobber = True
 289         if not noclobber:
 290             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 291
 292     for tcat, genp in [(wikicat, generated_pages),
 293                        (wikiredircat, generated_redirects)]:
 294         catpages = site.Pages['Category:' + tcat]
 295         existingpages = []
 296         for page in catpages:
 297             existingpages.append(page.name)
 298             if page.name in genp:
 299                 pagetxt = page.edit()
 300                 if pagetxt != genp[page.name]:
 301                     logging.debug("Updating modified page " + page.name)
 302                     page.save(genp[page.name], summary='Auto-updated')
 303                 else:
 304                     logging.debug("Page " + page.name + " is unchanged")
 305             else:
 306                 logging.warn("Deleting page " + page.name)
 307                 page.delete('No longer published')
 308         for pagename, text in genp.items():
 309             logging.debug("Checking " + pagename)
 310             if pagename not in existingpages:
 311                 logging.debug("Creating page " + pagename)
 312                 try:
 313                     newpage = site.Pages[pagename]
 314                     newpage.save(text, summary='Auto-created')
 315                 except Exception as e:
 316                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 317
 318     # Purge server cache to ensure counts are up to date
 319     site.pages['Repository Maintenance'].purge()
 320
 321
 322 def delete_disabled_builds(apps, apkcache, repodirs):
 323     """Delete disabled build outputs.
 324
 325     :param apps: list of all applications, as per metadata.read_metadata
 326     :param apkcache: current apk cache information
 327     :param repodirs: the repo directories to process
 328     """
 329     for appid, app in apps.items():
 330         for build in app['builds']:
 331             if not build.disable:
 332                 continue
 333             apkfilename = common.get_release_filename(app, build)
 334             iconfilename = "%s.%s.png" % (
 335                 appid,
 336                 build.versionCode)
 337             for repodir in repodirs:
 338                 files = [
 339                     os.path.join(repodir, apkfilename),
 340                     os.path.join(repodir, apkfilename + '.asc'),
 341                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 342                 ]
 343                 for density in all_screen_densities:
 344                     repo_dir = get_icon_dir(repodir, density)
 345                     files.append(os.path.join(repo_dir, iconfilename))
 346
 347                 for f in files:
 348                     if os.path.exists(f):
 349                         logging.info("Deleting disabled build output " + f)
 350                         os.remove(f)
 351             if apkfilename in apkcache:
 352                 del apkcache[apkfilename]
 353
 354
 355 def resize_icon(iconpath, density):
 356
 357     if not os.path.isfile(iconpath):
 358         return
 359
 360     fp = None
 361     try:
 362         fp = open(iconpath, 'rb')
 363         im = Image.open(fp)
 364         size = dpi_to_px(density)
 365
 366         if any(length > size for length in im.size):
 367             oldsize = im.size
 368             im.thumbnail((size, size), Image.ANTIALIAS)
 369             logging.debug("%s was too large at %s - new size is %s" % (
 370                 iconpath, oldsize, im.size))
 371             im.save(iconpath, "PNG")
 372
 373     except Exception as e:
 374         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 375
 376     finally:
 377         if fp:
 378             fp.close()
 379
 380
 381 def resize_all_icons(repodirs):
 382     """Resize all icons that exceed the max size
 383
 384     :param repodirs: the repo directories to process
 385     """
 386     for repodir in repodirs:
 387         for density in screen_densities:
 388             icon_dir = get_icon_dir(repodir, density)
 389             icon_glob = os.path.join(icon_dir, '*.png')
 390             for iconpath in glob.glob(icon_glob):
 391                 resize_icon(iconpath, density)
 392
 393
 394 def getsig(apkpath):
 395     """ Get the signing certificate of an apk. To get the same md5 has that
 396     Android gets, we encode the .RSA certificate in a specific format and pass
 397     it hex-encoded to the md5 digest algorithm.
 398
 399     :param apkpath: path to the apk
 400     :returns: A string containing the md5 of the signature of the apk or None
 401               if an error occurred.
 402     """
 403
 404     with zipfile.ZipFile(apkpath, 'r') as apk:
 405         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 406
 407         if len(certs) < 1:
 408             logging.error("Found no signing certificates on %s" % apkpath)
 409             return None
 410         if len(certs) > 1:
 411             logging.error("Found multiple signing certificates on %s" % apkpath)
 412             return None
 413
 414         cert = apk.read(certs[0])
 415
 416     cert_encoded = common.get_certificate(cert)
 417
 418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 419
 420
 421 def get_cache_file():
 422     return os.path.join('tmp', 'apkcache')
 423
 424
 425 def get_cache():
 426     """Get the cached dict of the APK index
 427
 428     Gather information about all the apk files in the repo directory,
 429     using cached data if possible. Some of the index operations take a
 430     long time, like calculating the SHA-256 and verifying the APK
 431     signature.
 432
 433     The cache is invalidated if the metadata version is different, or
 434     the 'allow_disabled_algorithms' config/option is different.  In
 435     those cases, there is no easy way to know what has changed from
 436     the cache, so just rerun the whole thing.
 437
 438     :return: apkcache
 439
 440     """
 441     apkcachefile = get_cache_file()
 442     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 443     if not options.clean and os.path.exists(apkcachefile):
 444         with open(apkcachefile, 'rb') as cf:
 445             apkcache = pickle.load(cf, encoding='utf-8')
 446         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 447            or apkcache.get('allow_disabled_algorithms') != ada:
 448             apkcache = {}
 449     else:
 450         apkcache = {}
 451
 452     apkcache["METADATA_VERSION"] = METADATA_VERSION
 453     apkcache['allow_disabled_algorithms'] = ada
 454
 455     return apkcache
 456
 457
 458 def write_cache(apkcache):
 459     apkcachefile = get_cache_file()
 460     cache_path = os.path.dirname(apkcachefile)
 461     if not os.path.exists(cache_path):
 462         os.makedirs(cache_path)
 463     with open(apkcachefile, 'wb') as cf:
 464         pickle.dump(apkcache, cf)
 465
 466
 467 def get_icon_bytes(apkzip, iconsrc):
 468     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 469     try:
 470         return apkzip.read(iconsrc)
 471     except KeyError:
 472         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 473
 474
 475 def sha256sum(filename):
 476     '''Calculate the sha256 of the given file'''
 477     sha = hashlib.sha256()
 478     with open(filename, 'rb') as f:
 479         while True:
 480             t = f.read(16384)
 481             if len(t) == 0:
 482                 break
 483             sha.update(t)
 484     return sha.hexdigest()
 485
 486
 487 def has_known_vulnerability(filename):
 488     """checks for known vulnerabilities in the APK
 489
 490     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 491     version.  Google also enforces this:
 492     https://support.google.com/faqs/answer/6376725?hl=en
 493
 494     Checks whether there are more than one classes.dex or AndroidManifest.xml
 495     files, which is invalid and an essential part of the "Master Key" attack.
 496
 497     http://www.saurik.com/id/17
 498     """
 499
 500     # statically load this pattern
 501     if not hasattr(has_known_vulnerability, "pattern"):
 502         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 503
 504     files_in_apk = set()
 505     with zipfile.ZipFile(filename) as zf:
 506         for name in zf.namelist():
 507             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 508                 lib = zf.open(name)
 509                 while True:
 510                     chunk = lib.read(4096)
 511                     if chunk == b'':
 512                         break
 513                     m = has_known_vulnerability.pattern.search(chunk)
 514                     if m:
 515                         version = m.group(1).decode('ascii')
 516                         if version.startswith('1.0.1') and version[5] >= 'r' \
 517                            or version.startswith('1.0.2') and version[5] >= 'f':
 518                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 519                         else:
 520                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 521                             return True
 522                         break
 523             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 524                 if name in files_in_apk:
 525                     return True
 526                 files_in_apk.add(name)
 527
 528     return False
 529
 530
 531 def insert_obbs(repodir, apps, apks):
 532     """Scans the .obb files in a given repo directory and adds them to the
 533     relevant APK instances.  OBB files have versionCodes like APK
 534     files, and they are loosely associated.  If there is an OBB file
 535     present, then any APK with the same or higher versionCode will use
 536     that OBB file.  There are two OBB types: main and patch, each APK
 537     can only have only have one of each.
 538
 539     https://developer.android.com/google/play/expansion-files.html
 540
 541     :param repodir: repo directory to scan
 542     :param apps: list of current, valid apps
 543     :param apks: current information on all APKs
 544
 545     """
 546
 547     def obbWarnDelete(f, msg):
 548         logging.warning(msg + f)
 549         if options.delete_unknown:
 550             logging.error("Deleting unknown file: " + f)
 551             os.remove(f)
 552
 553     obbs = []
 554     java_Integer_MIN_VALUE = -pow(2, 31)
 555     currentPackageNames = apps.keys()
 556     for f in glob.glob(os.path.join(repodir, '*.obb')):
 557         obbfile = os.path.basename(f)
 558         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 559         chunks = obbfile.split('.')
 560         if chunks[0] != 'main' and chunks[0] != 'patch':
 561             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 562             continue
 563         if not re.match(r'^-?[0-9]+$', chunks[1]):
 564             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 565             continue
 566         versionCode = int(chunks[1])
 567         packagename = ".".join(chunks[2:-1])
 568
 569         highestVersionCode = java_Integer_MIN_VALUE
 570         if packagename not in currentPackageNames:
 571             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 572             continue
 573         for apk in apks:
 574             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 575                 highestVersionCode = apk['versionCode']
 576         if versionCode > highestVersionCode:
 577             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 578                           + ') than any APK: ')
 579             continue
 580         obbsha256 = sha256sum(f)
 581         obbs.append((packagename, versionCode, obbfile, obbsha256))
 582
 583     for apk in apks:
 584         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 585             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 586                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 587                     apk['obbMainFile'] = obbfile
 588                     apk['obbMainFileSha256'] = obbsha256
 589                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 590                     apk['obbPatchFile'] = obbfile
 591                     apk['obbPatchFileSha256'] = obbsha256
 592             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 593                 break
 594
 595
 596 def translate_per_build_anti_features(apps, apks):
 597     """Grab the anti-features list from the build metadata
 598
 599     For most Anti-Features, they are really most applicable per-APK,
 600     not for an app.  An app can fix a vulnerability, add/remove
 601     tracking, etc.  This reads the 'antifeatures' list from the Build
 602     entries in the fdroiddata metadata file, then transforms it into
 603     the 'antiFeatures' list of unique items for the index.
 604
 605     The field key is all lower case in the metadata file to match the
 606     rest of the Build fields.  It is 'antiFeatures' camel case in the
 607     implementation, index, and fdroidclient since it is translated
 608     from the build 'antifeatures' field, not directly included.
 609
 610     """
 611
 612     antiFeatures = dict()
 613     for packageName, app in apps.items():
 614         d = dict()
 615         for build in app['builds']:
 616             afl = build.get('antifeatures')
 617             if afl:
 618                 d[int(build.versionCode)] = afl
 619         if len(d) > 0:
 620             antiFeatures[packageName] = d
 621
 622     for apk in apks:
 623         d = antiFeatures.get(apk['packageName'])
 624         if d:
 625             afl = d.get(apk['versionCode'])
 626             if afl:
 627                 apk['antiFeatures'].update(afl)
 628
 629
 630 def _get_localized_dict(app, locale):
 631     '''get the dict to add localized store metadata to'''
 632     if 'localized' not in app:
 633         app['localized'] = collections.OrderedDict()
 634     if locale not in app['localized']:
 635         app['localized'][locale] = collections.OrderedDict()
 636     return app['localized'][locale]
 637
 638
 639 def _set_localized_text_entry(app, locale, key, f):
 640     limit = config['char_limits'][key]
 641     localized = _get_localized_dict(app, locale)
 642     with open(f) as fp:
 643         text = fp.read()[:limit]
 644         if len(text) > 0:
 645             localized[key] = text
 646
 647
 648 def _set_author_entry(app, key, f):
 649     limit = config['char_limits']['author']
 650     with open(f) as fp:
 651         text = fp.read()[:limit]
 652         if len(text) > 0:
 653             app[key] = text
 654
 655
 656 def copy_triple_t_store_metadata(apps):
 657     """Include store metadata from the app's source repo
 658
 659     The Triple-T Gradle Play Publisher is a plugin that has a standard
 660     file layout for all of the metadata and graphics that the Google
 661     Play Store accepts.  Since F-Droid has the git repo, it can just
 662     pluck those files directly.  This method reads any text files into
 663     the app dict, then copies any graphics into the fdroid repo
 664     directory structure.
 665
 666     This needs to be run before insert_localized_app_metadata() so that
 667     the graphics files that are copied into the fdroid repo get
 668     properly indexed.
 669
 670     https://github.com/Triple-T/gradle-play-publisher#upload-images
 671     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 672
 673     """
 674
 675     if not os.path.isdir('build'):
 676         return  # nothing to do
 677
 678     for packageName, app in apps.items():
 679         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 680             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 681             for root, dirs, files in os.walk(d):
 682                 segments = root.split('/')
 683                 locale = segments[-2]
 684                 for f in files:
 685                     if f == 'fulldescription':
 686                         _set_localized_text_entry(app, locale, 'description',
 687                                                   os.path.join(root, f))
 688                         continue
 689                     elif f == 'shortdescription':
 690                         _set_localized_text_entry(app, locale, 'summary',
 691                                                   os.path.join(root, f))
 692                         continue
 693                     elif f == 'title':
 694                         _set_localized_text_entry(app, locale, 'name',
 695                                                   os.path.join(root, f))
 696                         continue
 697                     elif f == 'video':
 698                         _set_localized_text_entry(app, locale, 'video',
 699                                                   os.path.join(root, f))
 700                         continue
 701                     elif f == 'whatsnew':
 702                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 703                                                   os.path.join(root, f))
 704                         continue
 705                     elif f == 'contactEmail':
 706                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 707                         continue
 708                     elif f == 'contactPhone':
 709                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 710                         continue
 711                     elif f == 'contactWebsite':
 712                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 713                         continue
 714
 715                     base, extension = common.get_extension(f)
 716                     dirname = os.path.basename(root)
 717                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 718                         if segments[-2] == 'listing':
 719                             locale = segments[-3]
 720                         else:
 721                             locale = segments[-2]
 722                         destdir = os.path.join('repo', packageName, locale)
 723                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 724                         sourcefile = os.path.join(root, f)
 725                         destfile = os.path.join(destdir, dirname + '.' + extension)
 726                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 727                         shutil.copy(sourcefile, destfile)
 728
 729
 730 def insert_localized_app_metadata(apps):
 731     """scans standard locations for graphics and localized text
 732
 733     Scans for localized description files, store graphics, and
 734     screenshot PNG files in statically defined screenshots directory
 735     and adds them to the app metadata.  The screenshots and graphic
 736     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 737     and must be in the following layout:
 738     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 739
 740     repo/packageName/locale/featureGraphic.png
 741     repo/packageName/locale/phoneScreenshots/1.png
 742     repo/packageName/locale/phoneScreenshots/2.png
 743
 744     The changelog files must be text files named with the versionCode
 745     ending with ".txt" and must be in the following layout:
 746     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 747
 748     repo/packageName/locale/changelogs/12345.txt
 749
 750     This will scan the each app's source repo then the metadata/ dir
 751     for these standard locations of changelog files.  If it finds
 752     them, they will be added to the dict of all packages, with the
 753     versions in the metadata/ folder taking precendence over the what
 754     is in the app's source repo.
 755
 756     Where "packageName" is the app's packageName and "locale" is the locale
 757     of the graphics, e.g. what language they are in, using the IETF RFC5646
 758     format (en-US, fr-CA, es-MX, etc).
 759
 760     This will also scan the app's git for a fastlane folder, and the
 761     metadata/ folder and the apps' source repos for standard locations
 762     of graphic and screenshot files.  If it finds them, it will copy
 763     them into the repo.  The fastlane files follow this pattern:
 764     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 765
 766     """
 767
 768     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 769     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 770     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 771
 772     for srcd in sorted(sourcedirs):
 773         if not os.path.isdir(srcd):
 774             continue
 775         for root, dirs, files in os.walk(srcd):
 776             segments = root.split('/')
 777             packageName = segments[1]
 778             if packageName not in apps:
 779                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 780                 continue
 781             locale = segments[-1]
 782             destdir = os.path.join('repo', packageName, locale)
 783             for f in files:
 784                 if f in ('description.txt', 'full_description.txt'):
 785                     _set_localized_text_entry(apps[packageName], locale, 'description',
 786                                               os.path.join(root, f))
 787                     continue
 788                 elif f in ('summary.txt', 'short_description.txt'):
 789                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 790                                               os.path.join(root, f))
 791                     continue
 792                 elif f in ('name.txt', 'title.txt'):
 793                     _set_localized_text_entry(apps[packageName], locale, 'name',
 794                                               os.path.join(root, f))
 795                     continue
 796                 elif f == 'video.txt':
 797                     _set_localized_text_entry(apps[packageName], locale, 'video',
 798                                               os.path.join(root, f))
 799                     continue
 800                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 801                     locale = segments[-2]
 802                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 803                                               os.path.join(root, f))
 804                     continue
 805
 806                 base, extension = common.get_extension(f)
 807                 if locale == 'images':
 808                     locale = segments[-2]
 809                     destdir = os.path.join('repo', packageName, locale)
 810                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 811                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 812                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 813                     shutil.copy(os.path.join(root, f), destdir)
 814             for d in dirs:
 815                 if d in SCREENSHOT_DIRS:
 816                     for f in glob.glob(os.path.join(root, d, '*.*')):
 817                         _, extension = common.get_extension(f)
 818                         if extension in ALLOWED_EXTENSIONS:
 819                             screenshotdestdir = os.path.join(destdir, d)
 820                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 821                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 822                             shutil.copy(f, screenshotdestdir)
 823
 824     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 825     for d in repofiles:
 826         if not os.path.isdir(d):
 827             continue
 828         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 829             if not os.path.isfile(f):
 830                 continue
 831             segments = f.split('/')
 832             packageName = segments[1]
 833             locale = segments[2]
 834             screenshotdir = segments[3]
 835             filename = os.path.basename(f)
 836             base, extension = common.get_extension(filename)
 837
 838             if packageName not in apps:
 839                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 840                                 % (filename, packageName))
 841                 continue
 842             graphics = _get_localized_dict(apps[packageName], locale)
 843
 844             if extension not in ALLOWED_EXTENSIONS:
 845                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 846             elif base in GRAPHIC_NAMES:
 847                 # there can only be zero or one of these per locale
 848                 graphics[base] = filename
 849             elif screenshotdir in SCREENSHOT_DIRS:
 850                 # there can any number of these per locale
 851                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 852                 if screenshotdir not in graphics:
 853                     graphics[screenshotdir] = []
 854                 graphics[screenshotdir].append(filename)
 855             else:
 856                 logging.warning('Unsupported graphics file found: ' + f)
 857
 858
 859 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 860     """Scan a repo for all files with an extension except APK/OBB
 861
 862     :param apkcache: current cached info about all repo files
 863     :param repodir: repo directory to scan
 864     :param knownapks: list of all known files, as per metadata.read_metadata
 865     :param use_date_from_file: use date from file (instead of current date)
 866                                for newly added files
 867     """
 868
 869     cachechanged = False
 870     repo_files = []
 871     repodir = repodir.encode('utf-8')
 872     for name in os.listdir(repodir):
 873         file_extension = common.get_file_extension(name)
 874         if file_extension == 'apk' or file_extension == 'obb':
 875             continue
 876         filename = os.path.join(repodir, name)
 877         name_utf8 = name.decode('utf-8')
 878         if filename.endswith(b'_src.tar.gz'):
 879             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 880             continue
 881         if not common.is_repo_file(filename):
 882             continue
 883         stat = os.stat(filename)
 884         if stat.st_size == 0:
 885             raise FDroidException(filename + ' is zero size!')
 886
 887         shasum = sha256sum(filename)
 888         usecache = False
 889         if name in apkcache:
 890             repo_file = apkcache[name]
 891             # added time is cached as tuple but used here as datetime instance
 892             if 'added' in repo_file:
 893                 a = repo_file['added']
 894                 if isinstance(a, datetime):
 895                     repo_file['added'] = a
 896                 else:
 897                     repo_file['added'] = datetime(*a[:6])
 898             if repo_file.get('hash') == shasum:
 899                 logging.debug("Reading " + name_utf8 + " from cache")
 900                 usecache = True
 901             else:
 902                 logging.debug("Ignoring stale cache data for " + name)
 903
 904         if not usecache:
 905             logging.debug("Processing " + name_utf8)
 906             repo_file = collections.OrderedDict()
 907             repo_file['name'] = os.path.splitext(name_utf8)[0]
 908             # TODO rename apkname globally to something more generic
 909             repo_file['apkName'] = name_utf8
 910             repo_file['hash'] = shasum
 911             repo_file['hashType'] = 'sha256'
 912             repo_file['versionCode'] = 0
 913             repo_file['versionName'] = shasum
 914             # the static ID is the SHA256 unless it is set in the metadata
 915             repo_file['packageName'] = shasum
 916
 917             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 918             if m:
 919                 repo_file['packageName'] = m.group(1)
 920                 repo_file['versionCode'] = int(m.group(2))
 921             srcfilename = name + b'_src.tar.gz'
 922             if os.path.exists(os.path.join(repodir, srcfilename)):
 923                 repo_file['srcname'] = srcfilename.decode('utf-8')
 924             repo_file['size'] = stat.st_size
 925
 926             apkcache[name] = repo_file
 927             cachechanged = True
 928
 929         if use_date_from_file:
 930             timestamp = stat.st_ctime
 931             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 932         else:
 933             default_date_param = None
 934
 935         # Record in knownapks, getting the added date at the same time..
 936         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 937                                     default_date=default_date_param)
 938         if added:
 939             repo_file['added'] = added
 940
 941         repo_files.append(repo_file)
 942
 943     return repo_files, cachechanged
 944
 945
 946 def scan_apk(apk_file):
 947     """
 948     Scans an APK file and returns dictionary with metadata of the APK.
 949
 950     Attention: This does *not* verify that the APK signature is correct.
 951
 952     :param apk_file: The (ideally absolute) path to the APK file
 953     :raises BuildException
 954     :return A dict containing APK metadata
 955     """
 956     apk = {
 957         'hash': sha256sum(apk_file),
 958         'hashType': 'sha256',
 959         'uses-permission': [],
 960         'uses-permission-sdk-23': [],
 961         'features': [],
 962         'icons_src': {},
 963         'icons': {},
 964         'antiFeatures': set(),
 965     }
 966
 967     if SdkToolsPopen(['aapt', 'version'], output=False):
 968         scan_apk_aapt(apk, apk_file)
 969     else:
 970         scan_apk_androguard(apk, apk_file)
 971
 972     # Get the signature
 973     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 974     apk['sig'] = getsig(apk_file)
 975     if not apk['sig']:
 976         raise BuildException("Failed to get apk signature")
 977
 978     # Get size of the APK
 979     apk['size'] = os.path.getsize(apk_file)
 980
 981     if 'minSdkVersion' not in apk:
 982         logging.warning("No SDK version information found in {0}".format(apk_file))
 983         apk['minSdkVersion'] = 1
 984
 985     # Check for known vulnerabilities
 986     if has_known_vulnerability(apk_file):
 987         apk['antiFeatures'].add('KnownVuln')
 988
 989     return apk
 990
 991
 992 def scan_apk_aapt(apk, apkfile):
 993     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 994     if p.returncode != 0:
 995         if options.delete_unknown:
 996             if os.path.exists(apkfile):
 997                 logging.error("Failed to get apk information, deleting " + apkfile)
 998                 os.remove(apkfile)
 999             else:
1000                 logging.error("Could not find {0} to remove it".format(apkfile))
1001         else:
1002             logging.error("Failed to get apk information, skipping " + apkfile)
1003         raise BuildException("Invalid APK")
1004     for line in p.output.splitlines():
1005         if line.startswith("package:"):
1006             try:
1007                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1008                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1009                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1010             except Exception as e:
1011                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1012         elif line.startswith("application:"):
1013             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1014             # Keep path to non-dpi icon in case we need it
1015             match = re.match(APK_ICON_PAT_NODPI, line)
1016             if match:
1017                 apk['icons_src']['-1'] = match.group(1)
1018         elif line.startswith("launchable-activity:"):
1019             # Only use launchable-activity as fallback to application
1020             if not apk['name']:
1021                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1022             if '-1' not in apk['icons_src']:
1023                 match = re.match(APK_ICON_PAT_NODPI, line)
1024                 if match:
1025                     apk['icons_src']['-1'] = match.group(1)
1026         elif line.startswith("application-icon-"):
1027             match = re.match(APK_ICON_PAT, line)
1028             if match:
1029                 density = match.group(1)
1030                 path = match.group(2)
1031                 apk['icons_src'][density] = path
1032         elif line.startswith("sdkVersion:"):
1033             m = re.match(APK_SDK_VERSION_PAT, line)
1034             if m is None:
1035                 logging.error(line.replace('sdkVersion:', '')
1036                               + ' is not a valid minSdkVersion!')
1037             else:
1038                 apk['minSdkVersion'] = m.group(1)
1039                 # if target not set, default to min
1040                 if 'targetSdkVersion' not in apk:
1041                     apk['targetSdkVersion'] = m.group(1)
1042         elif line.startswith("targetSdkVersion:"):
1043             m = re.match(APK_SDK_VERSION_PAT, line)
1044             if m is None:
1045                 logging.error(line.replace('targetSdkVersion:', '')
1046                               + ' is not a valid targetSdkVersion!')
1047             else:
1048                 apk['targetSdkVersion'] = m.group(1)
1049         elif line.startswith("maxSdkVersion:"):
1050             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1051         elif line.startswith("native-code:"):
1052             apk['nativecode'] = []
1053             for arch in line[13:].split(' '):
1054                 apk['nativecode'].append(arch[1:-1])
1055         elif line.startswith('uses-permission:'):
1056             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1057             if perm_match['maxSdkVersion']:
1058                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1059             permission = UsesPermission(
1060                 perm_match['name'],
1061                 perm_match['maxSdkVersion']
1062             )
1063
1064             apk['uses-permission'].append(permission)
1065         elif line.startswith('uses-permission-sdk-23:'):
1066             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1067             if perm_match['maxSdkVersion']:
1068                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1069             permission_sdk_23 = UsesPermissionSdk23(
1070                 perm_match['name'],
1071                 perm_match['maxSdkVersion']
1072             )
1073
1074             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1075
1076         elif line.startswith('uses-feature:'):
1077             feature = re.match(APK_FEATURE_PAT, line).group(1)
1078             # Filter out this, it's only added with the latest SDK tools and
1079             # causes problems for lots of apps.
1080             if feature != "android.hardware.screen.portrait" \
1081                     and feature != "android.hardware.screen.landscape":
1082                 if feature.startswith("android.feature."):
1083                     feature = feature[16:]
1084                 apk['features'].add(feature)
1085
1086
1087 def scan_apk_androguard(apk, apkfile):
1088     try:
1089         from androguard.core.bytecodes.apk import APK
1090         apkobject = APK(apkfile)
1091         if apkobject.is_valid_APK():
1092             arsc = apkobject.get_android_resources()
1093         else:
1094             if options.delete_unknown:
1095                 if os.path.exists(apkfile):
1096                     logging.error("Failed to get apk information, deleting " + apkfile)
1097                     os.remove(apkfile)
1098                 else:
1099                     logging.error("Could not find {0} to remove it".format(apkfile))
1100             else:
1101                 logging.error("Failed to get apk information, skipping " + apkfile)
1102             raise BuildException("Invaild APK")
1103     except ImportError:
1104         raise FDroidException("androguard library is not installed and aapt not present")
1105     except FileNotFoundError:
1106         logging.error("Could not open apk file for analysis")
1107         raise BuildException("Invalid APK")
1108
1109     apk['packageName'] = apkobject.get_package()
1110     apk['versionCode'] = int(apkobject.get_androidversion_code())
1111     apk['versionName'] = apkobject.get_androidversion_name()
1112     if apk['versionName'][0] == "@":
1113         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1114         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1115         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1116     apk['name'] = apkobject.get_app_name()
1117
1118     if apkobject.get_max_sdk_version() is not None:
1119         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1120     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1121     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1122
1123     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1124     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1125
1126     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1127
1128     for file in apkobject.get_files():
1129         d_re = density_re.match(file)
1130         if d_re:
1131             folder = d_re.group(1).split('-')
1132             if len(folder) > 1:
1133                 resolution = folder[1]
1134             else:
1135                 resolution = 'mdpi'
1136             density = screen_resolutions[resolution]
1137             apk['icons_src'][density] = d_re.group(0)
1138
1139     if apk['icons_src'].get('-1') is None:
1140         apk['icons_src']['-1'] = apk['icons_src']['160']
1141
1142     arch_re = re.compile("^lib/(.*)/.*$")
1143     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1144     if len(arch) >= 1:
1145         apk['nativecode'] = []
1146         apk['nativecode'].extend(sorted(list(arch)))
1147
1148     xml = apkobject.get_android_manifest_xml()
1149
1150     for item in xml.getElementsByTagName('uses-permission'):
1151         name = str(item.getAttribute("android:name"))
1152         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1153         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1154         permission = UsesPermission(
1155             name,
1156             maxSdkVersion
1157         )
1158         apk['uses-permission'].append(permission)
1159
1160     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1161         name = str(item.getAttribute("android:name"))
1162         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1163         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1164         permission_sdk_23 = UsesPermissionSdk23(
1165             name,
1166             maxSdkVersion
1167         )
1168         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1169
1170     for item in xml.getElementsByTagName('uses-feature'):
1171         feature = str(item.getAttribute("android:name"))
1172         if feature != "android.hardware.screen.portrait" \
1173                 and feature != "android.hardware.screen.landscape":
1174             if feature.startswith("android.feature."):
1175                 feature = feature[16:]
1176         apk['features'].append(feature)
1177
1178
1179 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1180                 allow_disabled_algorithms=False, archive_bad_sig=False):
1181     """Processes the apk with the given filename in the given repo directory.
1182
1183     This also extracts the icons.
1184
1185     :param apkcache: current apk cache information
1186     :param apkfilename: the filename of the apk to scan
1187     :param repodir: repo directory to scan
1188     :param knownapks: known apks info
1189     :param use_date_from_apk: use date from APK (instead of current date)
1190                               for newly added APKs
1191     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1192                                       disabled algorithms in the signature (e.g. MD5)
1193     :param archive_bad_sig: move APKs with a bad signature to the archive
1194     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1195      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1196     """
1197
1198     if ' ' in apkfilename:
1199         if options.rename_apks:
1200             newfilename = apkfilename.replace(' ', '_')
1201             os.rename(os.path.join(repodir, apkfilename),
1202                       os.path.join(repodir, newfilename))
1203             apkfilename = newfilename
1204         else:
1205             logging.critical("Spaces in filenames are not allowed.")
1206             return True, None, False
1207
1208     apk = {}
1209     apkfile = os.path.join(repodir, apkfilename)
1210
1211     cachechanged = False
1212     usecache = False
1213     if apkfilename in apkcache:
1214         apk = apkcache[apkfilename]
1215         if apk.get('hash') == sha256sum(apkfile):
1216             logging.debug("Reading " + apkfilename + " from cache")
1217             usecache = True
1218         else:
1219             logging.debug("Ignoring stale cache data for " + apkfilename)
1220
1221     if not usecache:
1222         logging.debug("Processing " + apkfilename)
1223
1224         try:
1225             apk = scan_apk(apkfile)
1226         except BuildException:
1227             logging.warning('Skipping "%s" with invalid signature!', apkfilename)
1228             return True, None, False
1229
1230         # Check for debuggable apks...
1231         if common.isApkAndDebuggable(apkfile):
1232             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1233
1234         if options.rename_apks:
1235             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1236             std_short_name = os.path.join(repodir, n)
1237             if apkfile != std_short_name:
1238                 if os.path.exists(std_short_name):
1239                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1240                     if apkfile != std_long_name:
1241                         if os.path.exists(std_long_name):
1242                             dupdir = os.path.join('duplicates', repodir)
1243                             if not os.path.isdir(dupdir):
1244                                 os.makedirs(dupdir, exist_ok=True)
1245                             dupfile = os.path.join('duplicates', std_long_name)
1246                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1247                             os.rename(apkfile, dupfile)
1248                             return True, None, False
1249                         else:
1250                             os.rename(apkfile, std_long_name)
1251                     apkfile = std_long_name
1252                 else:
1253                     os.rename(apkfile, std_short_name)
1254                     apkfile = std_short_name
1255                 apkfilename = apkfile[len(repodir) + 1:]
1256
1257         apk['apkName'] = apkfilename
1258         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1259         if os.path.exists(os.path.join(repodir, srcfilename)):
1260             apk['srcname'] = srcfilename
1261
1262         # verify the jar signature is correct, allow deprecated
1263         # algorithms only if the APK is in the archive.
1264         skipapk = False
1265         if not common.verify_apk_signature(apkfile):
1266             if repodir == 'archive' or allow_disabled_algorithms:
1267                 if common.verify_old_apk_signature(apkfile):
1268                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1269                 else:
1270                     skipapk = True
1271             else:
1272                 skipapk = True
1273
1274         if skipapk:
1275             if archive_bad_sig:
1276                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1277                 move_apk_between_sections(repodir, 'archive', apk)
1278             else:
1279                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1280             return True, None, False
1281
1282         apkzip = zipfile.ZipFile(apkfile, 'r')
1283
1284         # if an APK has files newer than the system time, suggest updating
1285         # the system clock.  This is useful for offline systems, used for
1286         # signing, which do not have another source of clock sync info. It
1287         # has to be more than 24 hours newer because ZIP/APK files do not
1288         # store timezone info
1289         manifest = apkzip.getinfo('AndroidManifest.xml')
1290         if manifest.date_time[1] == 0:  # month can't be zero
1291             logging.debug('AndroidManifest.xml has no date')
1292         else:
1293             dt_obj = datetime(*manifest.date_time)
1294             checkdt = dt_obj - timedelta(1)
1295             if datetime.today() < checkdt:
1296                 logging.warning('System clock is older than manifest in: '
1297                                 + apkfilename
1298                                 + '\nSet clock to that time using:\n'
1299                                 + 'sudo date -s "' + str(dt_obj) + '"')
1300
1301         # extract icons from APK zip file
1302         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1303         try:
1304             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1305         finally:
1306             apkzip.close()  # ensure that APK zip file gets closed
1307
1308         # resize existing icons for densities missing in the APK
1309         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1310
1311         if use_date_from_apk and manifest.date_time[1] != 0:
1312             default_date_param = datetime(*manifest.date_time)
1313         else:
1314             default_date_param = None
1315
1316         # Record in known apks, getting the added date at the same time..
1317         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1318                                     default_date=default_date_param)
1319         if added:
1320             apk['added'] = added
1321
1322         apkcache[apkfilename] = apk
1323         cachechanged = True
1324
1325     return False, apk, cachechanged
1326
1327
1328 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1329     """Processes the apks in the given repo directory.
1330
1331     This also extracts the icons.
1332
1333     :param apkcache: current apk cache information
1334     :param repodir: repo directory to scan
1335     :param knownapks: known apks info
1336     :param use_date_from_apk: use date from APK (instead of current date)
1337                               for newly added APKs
1338     :returns: (apks, cachechanged) where apks is a list of apk information,
1339               and cachechanged is True if the apkcache got changed.
1340     """
1341
1342     cachechanged = False
1343
1344     for icon_dir in get_all_icon_dirs(repodir):
1345         if os.path.exists(icon_dir):
1346             if options.clean:
1347                 shutil.rmtree(icon_dir)
1348                 os.makedirs(icon_dir)
1349         else:
1350             os.makedirs(icon_dir)
1351
1352     apks = []
1353     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1354         apkfilename = apkfile[len(repodir) + 1:]
1355         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1356         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1357                                              use_date_from_apk, ada, True)
1358         if skip:
1359             continue
1360         apks.append(apk)
1361         cachechanged = cachechanged or cachethis
1362
1363     return apks, cachechanged
1364
1365
1366 def extract_apk_icons(icon_filename, apk, apk_zip, repo_dir):
1367     """
1368     Extracts icons from the given APK zip in various densities,
1369     saves them into given repo directory
1370     and stores their names in the APK metadata dictionary.
1371
1372     :param icon_filename: A string representing the icon's file name
1373     :param apk: A populated dictionary containing APK metadata.
1374                 Needs to have 'icons_src' key
1375     :param apk_zip: An opened zipfile.ZipFile of the APK file
1376     :param repo_dir: The directory of the APK's repository
1377     :return: A list of icon densities that are missing
1378     """
1379     empty_densities = []
1380     for density in screen_densities:
1381         if density not in apk['icons_src']:
1382             empty_densities.append(density)
1383             continue
1384         icon_src = apk['icons_src'][density]
1385         icon_dir = get_icon_dir(repo_dir, density)
1386         icon_dest = os.path.join(icon_dir, icon_filename)
1387
1388         # Extract the icon files per density
1389         try:
1390             with open(icon_dest, 'wb') as f:
1391                 f.write(get_icon_bytes(apk_zip, icon_src))
1392             apk['icons'][density] = icon_filename
1393         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1394             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1395             del apk['icons_src'][density]
1396             empty_densities.append(density)
1397
1398     if '-1' in apk['icons_src']:
1399         icon_src = apk['icons_src']['-1']
1400         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1401         with open(icon_path, 'wb') as f:
1402             f.write(get_icon_bytes(apk_zip, icon_src))
1403         try:
1404             im = Image.open(icon_path)
1405             dpi = px_to_dpi(im.size[0])
1406             for density in screen_densities:
1407                 if density in apk['icons']:
1408                     break
1409                 if density == screen_densities[-1] or dpi >= int(density):
1410                     apk['icons'][density] = icon_filename
1411                     shutil.move(icon_path,
1412                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1413                     empty_densities.remove(density)
1414                     break
1415         except Exception as e:
1416             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1417
1418     if apk['icons']:
1419         apk['icon'] = icon_filename
1420
1421     return empty_densities
1422
1423
1424 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1425     """
1426     Resize existing icons for densities missing in the APK to ensure all densities are available
1427
1428     :param empty_densities: A list of icon densities that are missing
1429     :param icon_filename: A string representing the icon's file name
1430     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1431     :param repo_dir: The directory of the APK's repository
1432     """
1433     # First try resizing down to not lose quality
1434     last_density = None
1435     for density in screen_densities:
1436         if density not in empty_densities:
1437             last_density = density
1438             continue
1439         if last_density is None:
1440             continue
1441         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1442
1443         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1444         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1445         fp = None
1446         try:
1447             fp = open(last_icon_path, 'rb')
1448             im = Image.open(fp)
1449
1450             size = dpi_to_px(density)
1451
1452             im.thumbnail((size, size), Image.ANTIALIAS)
1453             im.save(icon_path, "PNG")
1454             empty_densities.remove(density)
1455         except Exception as e:
1456             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1457         finally:
1458             if fp:
1459                 fp.close()
1460
1461     # Then just copy from the highest resolution available
1462     last_density = None
1463     for density in reversed(screen_densities):
1464         if density not in empty_densities:
1465             last_density = density
1466             continue
1467
1468         if last_density is None:
1469             continue
1470
1471         shutil.copyfile(
1472             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1473             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1474         )
1475         empty_densities.remove(density)
1476
1477     for density in screen_densities:
1478         icon_dir = get_icon_dir(repo_dir, density)
1479         icon_dest = os.path.join(icon_dir, icon_filename)
1480         resize_icon(icon_dest, density)
1481
1482     # Copy from icons-mdpi to icons since mdpi is the baseline density
1483     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1484     if os.path.isfile(baseline):
1485         apk['icons']['0'] = icon_filename
1486         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1487
1488
1489 def apply_info_from_latest_apk(apps, apks):
1490     """
1491     Some information from the apks needs to be applied up to the application level.
1492     When doing this, we use the info from the most recent version's apk.
1493     We deal with figuring out when the app was added and last updated at the same time.
1494     """
1495     for appid, app in apps.items():
1496         bestver = UNSET_VERSION_CODE
1497         for apk in apks:
1498             if apk['packageName'] == appid:
1499                 if apk['versionCode'] > bestver:
1500                     bestver = apk['versionCode']
1501                     bestapk = apk
1502
1503                 if 'added' in apk:
1504                     if not app.added or apk['added'] < app.added:
1505                         app.added = apk['added']
1506                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1507                         app.lastUpdated = apk['added']
1508
1509         if not app.added:
1510             logging.debug("Don't know when " + appid + " was added")
1511         if not app.lastUpdated:
1512             logging.debug("Don't know when " + appid + " was last updated")
1513
1514         if bestver == UNSET_VERSION_CODE:
1515
1516             if app.Name is None:
1517                 app.Name = app.AutoName or appid
1518             app.icon = None
1519             logging.debug("Application " + appid + " has no packages")
1520         else:
1521             if app.Name is None:
1522                 app.Name = bestapk['name']
1523             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1524             if app.CurrentVersionCode is None:
1525                 app.CurrentVersionCode = str(bestver)
1526
1527
1528 def make_categories_txt(repodir, categories):
1529     '''Write a category list in the repo to allow quick access'''
1530     catdata = ''
1531     for cat in sorted(categories):
1532         catdata += cat + '\n'
1533     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1534         f.write(catdata)
1535
1536
1537 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1538
1539     def filter_apk_list_sorted(apk_list):
1540         res = []
1541         for apk in apk_list:
1542             if apk['packageName'] == appid:
1543                 res.append(apk)
1544
1545         # Sort the apk list by version code. First is highest/newest.
1546         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1547
1548     for appid, app in apps.items():
1549
1550         if app.ArchivePolicy:
1551             keepversions = int(app.ArchivePolicy[:-9])
1552         else:
1553             keepversions = defaultkeepversions
1554
1555         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1556                       .format(appid, len(apks), keepversions, len(archapks)))
1557
1558         current_app_apks = filter_apk_list_sorted(apks)
1559         if len(current_app_apks) > keepversions:
1560             # Move back the ones we don't want.
1561             for apk in current_app_apks[keepversions:]:
1562                 move_apk_between_sections(repodir, archivedir, apk)
1563                 archapks.append(apk)
1564                 apks.remove(apk)
1565
1566         current_app_archapks = filter_apk_list_sorted(archapks)
1567         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1568             kept = 0
1569             # Move forward the ones we want again, except DisableAlgorithm
1570             for apk in current_app_archapks:
1571                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1572                     move_apk_between_sections(archivedir, repodir, apk)
1573                     archapks.remove(apk)
1574                     apks.append(apk)
1575                     kept += 1
1576                 if kept == keepversions:
1577                     break
1578
1579
1580 def move_apk_between_sections(from_dir, to_dir, apk):
1581     """move an APK from repo to archive or vice versa"""
1582
1583     def _move_file(from_dir, to_dir, filename, ignore_missing):
1584         from_path = os.path.join(from_dir, filename)
1585         if ignore_missing and not os.path.exists(from_path):
1586             return
1587         to_path = os.path.join(to_dir, filename)
1588         if not os.path.exists(to_dir):
1589             os.mkdir(to_dir)
1590         shutil.move(from_path, to_path)
1591
1592     if from_dir == to_dir:
1593         return
1594
1595     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1596     _move_file(from_dir, to_dir, apk['apkName'], False)
1597     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1598     for density in all_screen_densities:
1599         from_icon_dir = get_icon_dir(from_dir, density)
1600         to_icon_dir = get_icon_dir(to_dir, density)
1601         if density not in apk['icons']:
1602             continue
1603         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1604     if 'srcname' in apk:
1605         _move_file(from_dir, to_dir, apk['srcname'], False)
1606
1607
1608 def add_apks_to_per_app_repos(repodir, apks):
1609     apks_per_app = dict()
1610     for apk in apks:
1611         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1612         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1613         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1614         apks_per_app[apk['packageName']] = apk
1615
1616         if not os.path.exists(apk['per_app_icons']):
1617             logging.info('Adding new repo for only ' + apk['packageName'])
1618             os.makedirs(apk['per_app_icons'])
1619
1620         apkpath = os.path.join(repodir, apk['apkName'])
1621         shutil.copy(apkpath, apk['per_app_repo'])
1622         apksigpath = apkpath + '.sig'
1623         if os.path.exists(apksigpath):
1624             shutil.copy(apksigpath, apk['per_app_repo'])
1625         apkascpath = apkpath + '.asc'
1626         if os.path.exists(apkascpath):
1627             shutil.copy(apkascpath, apk['per_app_repo'])
1628
1629
1630 config = None
1631 options = None
1632
1633
1634 def main():
1635
1636     global config, options
1637
1638     # Parse command line...
1639     parser = ArgumentParser()
1640     common.setup_global_opts(parser)
1641     parser.add_argument("--create-key", action="store_true", default=False,
1642                         help="Create a repo signing key in a keystore")
1643     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1644                         help="Create skeleton metadata files that are missing")
1645     parser.add_argument("--delete-unknown", action="store_true", default=False,
1646                         help="Delete APKs and/or OBBs without metadata from the repo")
1647     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1648                         help="Report on build data status")
1649     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1650                         help="Interactively ask about things that need updating.")
1651     parser.add_argument("-I", "--icons", action="store_true", default=False,
1652                         help="Resize all the icons exceeding the max pixel size and exit")
1653     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1654                         help="Specify editor to use in interactive mode. Default " +
1655                         "is /etc/alternatives/editor")
1656     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1657                         help="Update the wiki")
1658     parser.add_argument("--pretty", action="store_true", default=False,
1659                         help="Produce human-readable index.xml")
1660     parser.add_argument("--clean", action="store_true", default=False,
1661                         help="Clean update - don't uses caches, reprocess all apks")
1662     parser.add_argument("--nosign", action="store_true", default=False,
1663                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1664     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1665                         help="Use date from apk instead of current time for newly added apks")
1666     parser.add_argument("--rename-apks", action="store_true", default=False,
1667                         help="Rename APK files that do not match package.name_123.apk")
1668     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1669                         help="Include APKs that are signed with disabled algorithms like MD5")
1670     metadata.add_metadata_arguments(parser)
1671     options = parser.parse_args()
1672     metadata.warnings_action = options.W
1673
1674     config = common.read_config(options)
1675
1676     if not ('jarsigner' in config and 'keytool' in config):
1677         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1678
1679     repodirs = ['repo']
1680     if config['archive_older'] != 0:
1681         repodirs.append('archive')
1682         if not os.path.exists('archive'):
1683             os.mkdir('archive')
1684
1685     if options.icons:
1686         resize_all_icons(repodirs)
1687         sys.exit(0)
1688
1689     if options.rename_apks:
1690         options.clean = True
1691
1692     # check that icons exist now, rather than fail at the end of `fdroid update`
1693     for k in ['repo_icon', 'archive_icon']:
1694         if k in config:
1695             if not os.path.exists(config[k]):
1696                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1697                 sys.exit(1)
1698
1699     # if the user asks to create a keystore, do it now, reusing whatever it can
1700     if options.create_key:
1701         if os.path.exists(config['keystore']):
1702             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1703             logging.critical("\t'" + config['keystore'] + "'")
1704             sys.exit(1)
1705
1706         if 'repo_keyalias' not in config:
1707             config['repo_keyalias'] = socket.getfqdn()
1708             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1709         if 'keydname' not in config:
1710             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1711             common.write_to_config(config, 'keydname', config['keydname'])
1712         if 'keystore' not in config:
1713             config['keystore'] = common.default_config['keystore']
1714             common.write_to_config(config, 'keystore', config['keystore'])
1715
1716         password = common.genpassword()
1717         if 'keystorepass' not in config:
1718             config['keystorepass'] = password
1719             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1720         if 'keypass' not in config:
1721             config['keypass'] = password
1722             common.write_to_config(config, 'keypass', config['keypass'])
1723         common.genkeystore(config)
1724
1725     # Get all apps...
1726     apps = metadata.read_metadata()
1727
1728     # Generate a list of categories...
1729     categories = set()
1730     for app in apps.values():
1731         categories.update(app.Categories)
1732
1733     # Read known apks data (will be updated and written back when we've finished)
1734     knownapks = common.KnownApks()
1735
1736     # Get APK cache
1737     apkcache = get_cache()
1738
1739     # Delete builds for disabled apps
1740     delete_disabled_builds(apps, apkcache, repodirs)
1741
1742     # Scan all apks in the main repo
1743     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1744
1745     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1746                                            options.use_date_from_apk)
1747     cachechanged = cachechanged or fcachechanged
1748     apks += files
1749     # Generate warnings for apk's with no metadata (or create skeleton
1750     # metadata files, if requested on the command line)
1751     newmetadata = False
1752     for apk in apks:
1753         if apk['packageName'] not in apps:
1754             if options.create_metadata:
1755                 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1756                     app = metadata.App()
1757                     if 'name' in apk:
1758                         app.Name = apk['name']
1759                         app.Summary = apk['name']
1760                     else:
1761                         logging.warn(apk['packageName'] + ' does not have a name! Using package name instead.')
1762                         app.Name = apk['packageName']
1763                         app.Summary = apk['packageName']
1764                     app.CurrentVersionCode = 2147483647  # Java's Integer.MAX_VALUE
1765                     app.Categories = [os.path.basename(os.path.dirname(os.getcwd()))]
1766                     metadata.write_yaml(f, app)
1767                     logging.info("Generated skeleton metadata for " + apk['packageName'])
1768                     newmetadata = True
1769             else:
1770                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1771                 if options.delete_unknown:
1772                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1773                     rmf = os.path.join(repodirs[0], apk['apkName'])
1774                     if not os.path.exists(rmf):
1775                         logging.error("Could not find {0} to remove it".format(rmf))
1776                     else:
1777                         os.remove(rmf)
1778                 else:
1779                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1780
1781     # update the metadata with the newly created ones included
1782     if newmetadata:
1783         apps = metadata.read_metadata()
1784
1785     copy_triple_t_store_metadata(apps)
1786     insert_obbs(repodirs[0], apps, apks)
1787     insert_localized_app_metadata(apps)
1788     translate_per_build_anti_features(apps, apks)
1789
1790     # Scan the archive repo for apks as well
1791     if len(repodirs) > 1:
1792         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1793         if cc:
1794             cachechanged = True
1795     else:
1796         archapks = []
1797
1798     # Apply information from latest apks to the application and update dates
1799     apply_info_from_latest_apk(apps, apks + archapks)
1800
1801     # Sort the app list by name, then the web site doesn't have to by default.
1802     # (we had to wait until we'd scanned the apks to do this, because mostly the
1803     # name comes from there!)
1804     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1805
1806     # APKs are placed into multiple repos based on the app package, providing
1807     # per-app subscription feeds for nightly builds and things like it
1808     if config['per_app_repos']:
1809         add_apks_to_per_app_repos(repodirs[0], apks)
1810         for appid, app in apps.items():
1811             repodir = os.path.join(appid, 'fdroid', 'repo')
1812             appdict = dict()
1813             appdict[appid] = app
1814             if os.path.isdir(repodir):
1815                 index.make(appdict, [appid], apks, repodir, False)
1816             else:
1817                 logging.info('Skipping index generation for ' + appid)
1818         return
1819
1820     if len(repodirs) > 1:
1821         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1822
1823     # Make the index for the main repo...
1824     index.make(apps, sortedids, apks, repodirs[0], False)
1825     make_categories_txt(repodirs[0], categories)
1826
1827     # If there's an archive repo,  make the index for it. We already scanned it
1828     # earlier on.
1829     if len(repodirs) > 1:
1830         index.make(apps, sortedids, archapks, repodirs[1], True)
1831
1832     git_remote = config.get('binary_transparency_remote')
1833     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1834         from . import btlog
1835         btlog.make_binary_transparency_log(repodirs)
1836
1837     if config['update_stats']:
1838         # Update known apks info...
1839         knownapks.writeifchanged()
1840
1841         # Generate latest apps data for widget
1842         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1843             data = ''
1844             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1845                 for line in f:
1846                     appid = line.rstrip()
1847                     data += appid + "\t"
1848                     app = apps[appid]
1849                     data += app.Name + "\t"
1850                     if app.icon is not None:
1851                         data += app.icon + "\t"
1852                     data += app.License + "\n"
1853             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1854                 f.write(data)
1855
1856     if cachechanged:
1857         write_cache(apkcache)
1858
1859     # Update the wiki...
1860     if options.wiki:
1861         update_wiki(apps, sortedids, apks + archapks)
1862
1863     logging.info("Finished.")
1864
1865
1866 if __name__ == "__main__":
1867     main()