fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import common
  41 from . import index
  42 from . import metadata
  43 from .common import SdkToolsPopen
  44 from .exception import BuildException, FDroidException
  45
  46 METADATA_VERSION = 18
  47
  48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  49 UNSET_VERSION_CODE = -0x100000000
  50
  51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85
  86 def dpi_to_px(density):
  87     return (int(density) * 48) / 160
  88
  89
  90 def px_to_dpi(px):
  91     return (int(px) * 160) / 48
  92
  93
  94 def get_icon_dir(repodir, density):
  95     if density == '0':
  96         return os.path.join(repodir, "icons")
  97     return os.path.join(repodir, "icons-%s" % density)
  98
  99
 100 def get_icon_dirs(repodir):
 101     for density in screen_densities:
 102         yield get_icon_dir(repodir, density)
 103
 104
 105 def get_all_icon_dirs(repodir):
 106     for density in all_screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def update_wiki(apps, sortedids, apks):
 111     """Update the wiki
 112
 113     :param apps: fully populated list of all applications
 114     :param apks: all apks, except...
 115     """
 116     logging.info("Updating wiki")
 117     wikicat = 'Apps'
 118     wikiredircat = 'App Redirects'
 119     import mwclient
 120     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 121                          path=config['wiki_path'])
 122     site.login(config['wiki_user'], config['wiki_password'])
 123     generated_pages = {}
 124     generated_redirects = {}
 125
 126     for appid in sortedids:
 127         app = metadata.App(apps[appid])
 128
 129         wikidata = ''
 130         if app.Disabled:
 131             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 132         if app.AntiFeatures:
 133             for af in app.AntiFeatures:
 134                 wikidata += '{{AntiFeature|' + af + '}}\n'
 135         if app.RequiresRoot:
 136             requiresroot = 'Yes'
 137         else:
 138             requiresroot = 'No'
 139         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 140             appid,
 141             app.Name,
 142             app.added.strftime('%Y-%m-%d') if app.added else '',
 143             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 144             app.SourceCode,
 145             app.IssueTracker,
 146             app.WebSite,
 147             app.Changelog,
 148             app.Donate,
 149             app.FlattrID,
 150             app.Bitcoin,
 151             app.Litecoin,
 152             app.License,
 153             requiresroot,
 154             app.AuthorName,
 155             app.AuthorEmail)
 156
 157         if app.Provides:
 158             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 159
 160         wikidata += app.Summary
 161         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 162
 163         wikidata += "=Description=\n"
 164         wikidata += metadata.description_wiki(app.Description) + "\n"
 165
 166         wikidata += "=Maintainer Notes=\n"
 167         if app.MaintainerNotes:
 168             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 169         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 170
 171         # Get a list of all packages for this application...
 172         apklist = []
 173         gotcurrentver = False
 174         cantupdate = False
 175         buildfails = False
 176         for apk in apks:
 177             if apk['packageName'] == appid:
 178                 if str(apk['versionCode']) == app.CurrentVersionCode:
 179                     gotcurrentver = True
 180                 apklist.append(apk)
 181         # Include ones we can't build, as a special case...
 182         for build in app.builds:
 183             if build.disable:
 184                 if build.versionCode == app.CurrentVersionCode:
 185                     cantupdate = True
 186                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 187                 apklist.append({'versionCode': int(build.versionCode),
 188                                 'versionName': build.versionName,
 189                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 190                                 })
 191             else:
 192                 builtit = False
 193                 for apk in apklist:
 194                     if apk['versionCode'] == int(build.versionCode):
 195                         builtit = True
 196                         break
 197                 if not builtit:
 198                     buildfails = True
 199                     apklist.append({'versionCode': int(build.versionCode),
 200                                     'versionName': build.versionName,
 201                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 202                                     })
 203         if app.CurrentVersionCode == '0':
 204             cantupdate = True
 205         # Sort with most recent first...
 206         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 207
 208         wikidata += "=Versions=\n"
 209         if len(apklist) == 0:
 210             wikidata += "We currently have no versions of this app available."
 211         elif not gotcurrentver:
 212             wikidata += "We don't have the current version of this app."
 213         else:
 214             wikidata += "We have the current version of this app."
 215         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 216         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 217         if len(app.NoSourceSince) > 0:
 218             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 219         if len(app.CurrentVersion) > 0:
 220             wikidata += "The current (recommended) version is " + app.CurrentVersion
 221             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 222         validapks = 0
 223         for apk in apklist:
 224             wikidata += "==" + apk['versionName'] + "==\n"
 225
 226             if 'buildproblem' in apk:
 227                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 228             else:
 229                 validapks += 1
 230                 wikidata += "This version is built and signed by "
 231                 if 'srcname' in apk:
 232                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 233                 else:
 234                     wikidata += "the original developer.\n\n"
 235             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 236
 237         wikidata += '\n[[Category:' + wikicat + ']]\n'
 238         if len(app.NoSourceSince) > 0:
 239             wikidata += '\n[[Category:Apps missing source code]]\n'
 240         if validapks == 0 and not app.Disabled:
 241             wikidata += '\n[[Category:Apps with no packages]]\n'
 242         if cantupdate and not app.Disabled:
 243             wikidata += "\n[[Category:Apps we cannot update]]\n"
 244         if buildfails and not app.Disabled:
 245             wikidata += "\n[[Category:Apps with failing builds]]\n"
 246         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 247             wikidata += '\n[[Category:Apps to Update]]\n'
 248         if app.Disabled:
 249             wikidata += '\n[[Category:Apps that are disabled]]\n'
 250         if app.UpdateCheckMode == 'None' and not app.Disabled:
 251             wikidata += '\n[[Category:Apps with no update check]]\n'
 252         for appcat in app.Categories:
 253             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 254
 255         # We can't have underscores in the page name, even if they're in
 256         # the package ID, because MediaWiki messes with them...
 257         pagename = appid.replace('_', ' ')
 258
 259         # Drop a trailing newline, because mediawiki is going to drop it anyway
 260         # and it we don't we'll think the page has changed when it hasn't...
 261         if wikidata.endswith('\n'):
 262             wikidata = wikidata[:-1]
 263
 264         generated_pages[pagename] = wikidata
 265
 266         # Make a redirect from the name to the ID too, unless there's
 267         # already an existing page with the name and it isn't a redirect.
 268         noclobber = False
 269         apppagename = app.Name.replace('_', ' ')
 270         apppagename = apppagename.replace('{', '')
 271         apppagename = apppagename.replace('}', ' ')
 272         apppagename = apppagename.replace(':', ' ')
 273         apppagename = apppagename.replace('[', ' ')
 274         apppagename = apppagename.replace(']', ' ')
 275         # Drop double spaces caused mostly by replacing ':' above
 276         apppagename = apppagename.replace('  ', ' ')
 277         for expagename in site.allpages(prefix=apppagename,
 278                                         filterredir='nonredirects',
 279                                         generator=False):
 280             if expagename == apppagename:
 281                 noclobber = True
 282         # Another reason not to make the redirect page is if the app name
 283         # is the same as it's ID, because that will overwrite the real page
 284         # with an redirect to itself! (Although it seems like an odd
 285         # scenario this happens a lot, e.g. where there is metadata but no
 286         # builds or binaries to extract a name from.
 287         if apppagename == pagename:
 288             noclobber = True
 289         if not noclobber:
 290             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 291
 292     for tcat, genp in [(wikicat, generated_pages),
 293                        (wikiredircat, generated_redirects)]:
 294         catpages = site.Pages['Category:' + tcat]
 295         existingpages = []
 296         for page in catpages:
 297             existingpages.append(page.name)
 298             if page.name in genp:
 299                 pagetxt = page.edit()
 300                 if pagetxt != genp[page.name]:
 301                     logging.debug("Updating modified page " + page.name)
 302                     page.save(genp[page.name], summary='Auto-updated')
 303                 else:
 304                     logging.debug("Page " + page.name + " is unchanged")
 305             else:
 306                 logging.warn("Deleting page " + page.name)
 307                 page.delete('No longer published')
 308         for pagename, text in genp.items():
 309             logging.debug("Checking " + pagename)
 310             if pagename not in existingpages:
 311                 logging.debug("Creating page " + pagename)
 312                 try:
 313                     newpage = site.Pages[pagename]
 314                     newpage.save(text, summary='Auto-created')
 315                 except Exception as e:
 316                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 317
 318     # Purge server cache to ensure counts are up to date
 319     site.pages['Repository Maintenance'].purge()
 320
 321
 322 def delete_disabled_builds(apps, apkcache, repodirs):
 323     """Delete disabled build outputs.
 324
 325     :param apps: list of all applications, as per metadata.read_metadata
 326     :param apkcache: current apk cache information
 327     :param repodirs: the repo directories to process
 328     """
 329     for appid, app in apps.items():
 330         for build in app['builds']:
 331             if not build.disable:
 332                 continue
 333             apkfilename = common.get_release_filename(app, build)
 334             iconfilename = "%s.%s.png" % (
 335                 appid,
 336                 build.versionCode)
 337             for repodir in repodirs:
 338                 files = [
 339                     os.path.join(repodir, apkfilename),
 340                     os.path.join(repodir, apkfilename + '.asc'),
 341                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 342                 ]
 343                 for density in all_screen_densities:
 344                     repo_dir = get_icon_dir(repodir, density)
 345                     files.append(os.path.join(repo_dir, iconfilename))
 346
 347                 for f in files:
 348                     if os.path.exists(f):
 349                         logging.info("Deleting disabled build output " + f)
 350                         os.remove(f)
 351             if apkfilename in apkcache:
 352                 del apkcache[apkfilename]
 353
 354
 355 def resize_icon(iconpath, density):
 356
 357     if not os.path.isfile(iconpath):
 358         return
 359
 360     fp = None
 361     try:
 362         fp = open(iconpath, 'rb')
 363         im = Image.open(fp)
 364         size = dpi_to_px(density)
 365
 366         if any(length > size for length in im.size):
 367             oldsize = im.size
 368             im.thumbnail((size, size), Image.ANTIALIAS)
 369             logging.debug("%s was too large at %s - new size is %s" % (
 370                 iconpath, oldsize, im.size))
 371             im.save(iconpath, "PNG")
 372
 373     except Exception as e:
 374         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 375
 376     finally:
 377         if fp:
 378             fp.close()
 379
 380
 381 def resize_all_icons(repodirs):
 382     """Resize all icons that exceed the max size
 383
 384     :param repodirs: the repo directories to process
 385     """
 386     for repodir in repodirs:
 387         for density in screen_densities:
 388             icon_dir = get_icon_dir(repodir, density)
 389             icon_glob = os.path.join(icon_dir, '*.png')
 390             for iconpath in glob.glob(icon_glob):
 391                 resize_icon(iconpath, density)
 392
 393
 394 def getsig(apkpath):
 395     """ Get the signing certificate of an apk. To get the same md5 has that
 396     Android gets, we encode the .RSA certificate in a specific format and pass
 397     it hex-encoded to the md5 digest algorithm.
 398
 399     :param apkpath: path to the apk
 400     :returns: A string containing the md5 of the signature of the apk or None
 401               if an error occurred.
 402     """
 403
 404     with zipfile.ZipFile(apkpath, 'r') as apk:
 405         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 406
 407         if len(certs) < 1:
 408             logging.error("Found no signing certificates on %s" % apkpath)
 409             return None
 410         if len(certs) > 1:
 411             logging.error("Found multiple signing certificates on %s" % apkpath)
 412             return None
 413
 414         cert = apk.read(certs[0])
 415
 416     cert_encoded = common.get_certificate(cert)
 417
 418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 419
 420
 421 def get_cache_file():
 422     return os.path.join('tmp', 'apkcache')
 423
 424
 425 def get_cache():
 426     """Get the cached dict of the APK index
 427
 428     Gather information about all the apk files in the repo directory,
 429     using cached data if possible. Some of the index operations take a
 430     long time, like calculating the SHA-256 and verifying the APK
 431     signature.
 432
 433     The cache is invalidated if the metadata version is different, or
 434     the 'allow_disabled_algorithms' config/option is different.  In
 435     those cases, there is no easy way to know what has changed from
 436     the cache, so just rerun the whole thing.
 437
 438     :return: apkcache
 439
 440     """
 441     apkcachefile = get_cache_file()
 442     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 443     if not options.clean and os.path.exists(apkcachefile):
 444         with open(apkcachefile, 'rb') as cf:
 445             apkcache = pickle.load(cf, encoding='utf-8')
 446         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 447            or apkcache.get('allow_disabled_algorithms') != ada:
 448             apkcache = {}
 449     else:
 450         apkcache = {}
 451
 452     apkcache["METADATA_VERSION"] = METADATA_VERSION
 453     apkcache['allow_disabled_algorithms'] = ada
 454
 455     return apkcache
 456
 457
 458 def write_cache(apkcache):
 459     apkcachefile = get_cache_file()
 460     cache_path = os.path.dirname(apkcachefile)
 461     if not os.path.exists(cache_path):
 462         os.makedirs(cache_path)
 463     with open(apkcachefile, 'wb') as cf:
 464         pickle.dump(apkcache, cf)
 465
 466
 467 def get_icon_bytes(apkzip, iconsrc):
 468     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 469     try:
 470         return apkzip.read(iconsrc)
 471     except KeyError:
 472         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 473
 474
 475 def sha256sum(filename):
 476     '''Calculate the sha256 of the given file'''
 477     sha = hashlib.sha256()
 478     with open(filename, 'rb') as f:
 479         while True:
 480             t = f.read(16384)
 481             if len(t) == 0:
 482                 break
 483             sha.update(t)
 484     return sha.hexdigest()
 485
 486
 487 def has_known_vulnerability(filename):
 488     """checks for known vulnerabilities in the APK
 489
 490     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 491     version.  Google also enforces this:
 492     https://support.google.com/faqs/answer/6376725?hl=en
 493
 494     Checks whether there are more than one classes.dex or AndroidManifest.xml
 495     files, which is invalid and an essential part of the "Master Key" attack.
 496
 497     http://www.saurik.com/id/17
 498     """
 499
 500     # statically load this pattern
 501     if not hasattr(has_known_vulnerability, "pattern"):
 502         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 503
 504     files_in_apk = set()
 505     with zipfile.ZipFile(filename) as zf:
 506         for name in zf.namelist():
 507             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 508                 lib = zf.open(name)
 509                 while True:
 510                     chunk = lib.read(4096)
 511                     if chunk == b'':
 512                         break
 513                     m = has_known_vulnerability.pattern.search(chunk)
 514                     if m:
 515                         version = m.group(1).decode('ascii')
 516                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 517                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 518                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 519                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 520                         else:
 521                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 522                             return True
 523                         break
 524             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 525                 if name in files_in_apk:
 526                     return True
 527                 files_in_apk.add(name)
 528
 529     return False
 530
 531
 532 def insert_obbs(repodir, apps, apks):
 533     """Scans the .obb files in a given repo directory and adds them to the
 534     relevant APK instances.  OBB files have versionCodes like APK
 535     files, and they are loosely associated.  If there is an OBB file
 536     present, then any APK with the same or higher versionCode will use
 537     that OBB file.  There are two OBB types: main and patch, each APK
 538     can only have only have one of each.
 539
 540     https://developer.android.com/google/play/expansion-files.html
 541
 542     :param repodir: repo directory to scan
 543     :param apps: list of current, valid apps
 544     :param apks: current information on all APKs
 545
 546     """
 547
 548     def obbWarnDelete(f, msg):
 549         logging.warning(msg + f)
 550         if options.delete_unknown:
 551             logging.error("Deleting unknown file: " + f)
 552             os.remove(f)
 553
 554     obbs = []
 555     java_Integer_MIN_VALUE = -pow(2, 31)
 556     currentPackageNames = apps.keys()
 557     for f in glob.glob(os.path.join(repodir, '*.obb')):
 558         obbfile = os.path.basename(f)
 559         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 560         chunks = obbfile.split('.')
 561         if chunks[0] != 'main' and chunks[0] != 'patch':
 562             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 563             continue
 564         if not re.match(r'^-?[0-9]+$', chunks[1]):
 565             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 566             continue
 567         versionCode = int(chunks[1])
 568         packagename = ".".join(chunks[2:-1])
 569
 570         highestVersionCode = java_Integer_MIN_VALUE
 571         if packagename not in currentPackageNames:
 572             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 573             continue
 574         for apk in apks:
 575             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 576                 highestVersionCode = apk['versionCode']
 577         if versionCode > highestVersionCode:
 578             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 579                           + ') than any APK: ')
 580             continue
 581         obbsha256 = sha256sum(f)
 582         obbs.append((packagename, versionCode, obbfile, obbsha256))
 583
 584     for apk in apks:
 585         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 586             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 587                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 588                     apk['obbMainFile'] = obbfile
 589                     apk['obbMainFileSha256'] = obbsha256
 590                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 591                     apk['obbPatchFile'] = obbfile
 592                     apk['obbPatchFileSha256'] = obbsha256
 593             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 594                 break
 595
 596
 597 def translate_per_build_anti_features(apps, apks):
 598     """Grab the anti-features list from the build metadata
 599
 600     For most Anti-Features, they are really most applicable per-APK,
 601     not for an app.  An app can fix a vulnerability, add/remove
 602     tracking, etc.  This reads the 'antifeatures' list from the Build
 603     entries in the fdroiddata metadata file, then transforms it into
 604     the 'antiFeatures' list of unique items for the index.
 605
 606     The field key is all lower case in the metadata file to match the
 607     rest of the Build fields.  It is 'antiFeatures' camel case in the
 608     implementation, index, and fdroidclient since it is translated
 609     from the build 'antifeatures' field, not directly included.
 610
 611     """
 612
 613     antiFeatures = dict()
 614     for packageName, app in apps.items():
 615         d = dict()
 616         for build in app['builds']:
 617             afl = build.get('antifeatures')
 618             if afl:
 619                 d[int(build.versionCode)] = afl
 620         if len(d) > 0:
 621             antiFeatures[packageName] = d
 622
 623     for apk in apks:
 624         d = antiFeatures.get(apk['packageName'])
 625         if d:
 626             afl = d.get(apk['versionCode'])
 627             if afl:
 628                 apk['antiFeatures'].update(afl)
 629
 630
 631 def _get_localized_dict(app, locale):
 632     '''get the dict to add localized store metadata to'''
 633     if 'localized' not in app:
 634         app['localized'] = collections.OrderedDict()
 635     if locale not in app['localized']:
 636         app['localized'][locale] = collections.OrderedDict()
 637     return app['localized'][locale]
 638
 639
 640 def _set_localized_text_entry(app, locale, key, f):
 641     limit = config['char_limits'][key]
 642     localized = _get_localized_dict(app, locale)
 643     with open(f) as fp:
 644         text = fp.read()[:limit]
 645         if len(text) > 0:
 646             localized[key] = text
 647
 648
 649 def _set_author_entry(app, key, f):
 650     limit = config['char_limits']['author']
 651     with open(f) as fp:
 652         text = fp.read()[:limit]
 653         if len(text) > 0:
 654             app[key] = text
 655
 656
 657 def copy_triple_t_store_metadata(apps):
 658     """Include store metadata from the app's source repo
 659
 660     The Triple-T Gradle Play Publisher is a plugin that has a standard
 661     file layout for all of the metadata and graphics that the Google
 662     Play Store accepts.  Since F-Droid has the git repo, it can just
 663     pluck those files directly.  This method reads any text files into
 664     the app dict, then copies any graphics into the fdroid repo
 665     directory structure.
 666
 667     This needs to be run before insert_localized_app_metadata() so that
 668     the graphics files that are copied into the fdroid repo get
 669     properly indexed.
 670
 671     https://github.com/Triple-T/gradle-play-publisher#upload-images
 672     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 673
 674     """
 675
 676     if not os.path.isdir('build'):
 677         return  # nothing to do
 678
 679     for packageName, app in apps.items():
 680         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 681             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 682             for root, dirs, files in os.walk(d):
 683                 segments = root.split('/')
 684                 locale = segments[-2]
 685                 for f in files:
 686                     if f == 'fulldescription':
 687                         _set_localized_text_entry(app, locale, 'description',
 688                                                   os.path.join(root, f))
 689                         continue
 690                     elif f == 'shortdescription':
 691                         _set_localized_text_entry(app, locale, 'summary',
 692                                                   os.path.join(root, f))
 693                         continue
 694                     elif f == 'title':
 695                         _set_localized_text_entry(app, locale, 'name',
 696                                                   os.path.join(root, f))
 697                         continue
 698                     elif f == 'video':
 699                         _set_localized_text_entry(app, locale, 'video',
 700                                                   os.path.join(root, f))
 701                         continue
 702                     elif f == 'whatsnew':
 703                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 704                                                   os.path.join(root, f))
 705                         continue
 706                     elif f == 'contactEmail':
 707                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 708                         continue
 709                     elif f == 'contactPhone':
 710                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 711                         continue
 712                     elif f == 'contactWebsite':
 713                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 714                         continue
 715
 716                     base, extension = common.get_extension(f)
 717                     dirname = os.path.basename(root)
 718                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 719                         if segments[-2] == 'listing':
 720                             locale = segments[-3]
 721                         else:
 722                             locale = segments[-2]
 723                         destdir = os.path.join('repo', packageName, locale)
 724                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 725                         sourcefile = os.path.join(root, f)
 726                         destfile = os.path.join(destdir, dirname + '.' + extension)
 727                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 728                         shutil.copy(sourcefile, destfile)
 729
 730
 731 def insert_localized_app_metadata(apps):
 732     """scans standard locations for graphics and localized text
 733
 734     Scans for localized description files, store graphics, and
 735     screenshot PNG files in statically defined screenshots directory
 736     and adds them to the app metadata.  The screenshots and graphic
 737     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 738     and must be in the following layout:
 739     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 740
 741     repo/packageName/locale/featureGraphic.png
 742     repo/packageName/locale/phoneScreenshots/1.png
 743     repo/packageName/locale/phoneScreenshots/2.png
 744
 745     The changelog files must be text files named with the versionCode
 746     ending with ".txt" and must be in the following layout:
 747     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 748
 749     repo/packageName/locale/changelogs/12345.txt
 750
 751     This will scan the each app's source repo then the metadata/ dir
 752     for these standard locations of changelog files.  If it finds
 753     them, they will be added to the dict of all packages, with the
 754     versions in the metadata/ folder taking precendence over the what
 755     is in the app's source repo.
 756
 757     Where "packageName" is the app's packageName and "locale" is the locale
 758     of the graphics, e.g. what language they are in, using the IETF RFC5646
 759     format (en-US, fr-CA, es-MX, etc).
 760
 761     This will also scan the app's git for a fastlane folder, and the
 762     metadata/ folder and the apps' source repos for standard locations
 763     of graphic and screenshot files.  If it finds them, it will copy
 764     them into the repo.  The fastlane files follow this pattern:
 765     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 766
 767     """
 768
 769     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 770     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 771     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 772
 773     for srcd in sorted(sourcedirs):
 774         if not os.path.isdir(srcd):
 775             continue
 776         for root, dirs, files in os.walk(srcd):
 777             segments = root.split('/')
 778             packageName = segments[1]
 779             if packageName not in apps:
 780                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 781                 continue
 782             locale = segments[-1]
 783             destdir = os.path.join('repo', packageName, locale)
 784             for f in files:
 785                 if f in ('description.txt', 'full_description.txt'):
 786                     _set_localized_text_entry(apps[packageName], locale, 'description',
 787                                               os.path.join(root, f))
 788                     continue
 789                 elif f in ('summary.txt', 'short_description.txt'):
 790                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 791                                               os.path.join(root, f))
 792                     continue
 793                 elif f in ('name.txt', 'title.txt'):
 794                     _set_localized_text_entry(apps[packageName], locale, 'name',
 795                                               os.path.join(root, f))
 796                     continue
 797                 elif f == 'video.txt':
 798                     _set_localized_text_entry(apps[packageName], locale, 'video',
 799                                               os.path.join(root, f))
 800                     continue
 801                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 802                     locale = segments[-2]
 803                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 804                                               os.path.join(root, f))
 805                     continue
 806
 807                 base, extension = common.get_extension(f)
 808                 if locale == 'images':
 809                     locale = segments[-2]
 810                     destdir = os.path.join('repo', packageName, locale)
 811                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 812                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 813                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 814                     shutil.copy(os.path.join(root, f), destdir)
 815             for d in dirs:
 816                 if d in SCREENSHOT_DIRS:
 817                     for f in glob.glob(os.path.join(root, d, '*.*')):
 818                         _, extension = common.get_extension(f)
 819                         if extension in ALLOWED_EXTENSIONS:
 820                             screenshotdestdir = os.path.join(destdir, d)
 821                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 822                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 823                             shutil.copy(f, screenshotdestdir)
 824
 825     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 826     for d in repofiles:
 827         if not os.path.isdir(d):
 828             continue
 829         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 830             if not os.path.isfile(f):
 831                 continue
 832             segments = f.split('/')
 833             packageName = segments[1]
 834             locale = segments[2]
 835             screenshotdir = segments[3]
 836             filename = os.path.basename(f)
 837             base, extension = common.get_extension(filename)
 838
 839             if packageName not in apps:
 840                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 841                                 % (filename, packageName))
 842                 continue
 843             graphics = _get_localized_dict(apps[packageName], locale)
 844
 845             if extension not in ALLOWED_EXTENSIONS:
 846                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 847             elif base in GRAPHIC_NAMES:
 848                 # there can only be zero or one of these per locale
 849                 graphics[base] = filename
 850             elif screenshotdir in SCREENSHOT_DIRS:
 851                 # there can any number of these per locale
 852                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 853                 if screenshotdir not in graphics:
 854                     graphics[screenshotdir] = []
 855                 graphics[screenshotdir].append(filename)
 856             else:
 857                 logging.warning('Unsupported graphics file found: ' + f)
 858
 859
 860 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 861     """Scan a repo for all files with an extension except APK/OBB
 862
 863     :param apkcache: current cached info about all repo files
 864     :param repodir: repo directory to scan
 865     :param knownapks: list of all known files, as per metadata.read_metadata
 866     :param use_date_from_file: use date from file (instead of current date)
 867                                for newly added files
 868     """
 869
 870     cachechanged = False
 871     repo_files = []
 872     repodir = repodir.encode('utf-8')
 873     for name in os.listdir(repodir):
 874         file_extension = common.get_file_extension(name)
 875         if file_extension == 'apk' or file_extension == 'obb':
 876             continue
 877         filename = os.path.join(repodir, name)
 878         name_utf8 = name.decode('utf-8')
 879         if filename.endswith(b'_src.tar.gz'):
 880             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 881             continue
 882         if not common.is_repo_file(filename):
 883             continue
 884         stat = os.stat(filename)
 885         if stat.st_size == 0:
 886             raise FDroidException(filename + ' is zero size!')
 887
 888         shasum = sha256sum(filename)
 889         usecache = False
 890         if name in apkcache:
 891             repo_file = apkcache[name]
 892             # added time is cached as tuple but used here as datetime instance
 893             if 'added' in repo_file:
 894                 a = repo_file['added']
 895                 if isinstance(a, datetime):
 896                     repo_file['added'] = a
 897                 else:
 898                     repo_file['added'] = datetime(*a[:6])
 899             if repo_file.get('hash') == shasum:
 900                 logging.debug("Reading " + name_utf8 + " from cache")
 901                 usecache = True
 902             else:
 903                 logging.debug("Ignoring stale cache data for " + name)
 904
 905         if not usecache:
 906             logging.debug("Processing " + name_utf8)
 907             repo_file = collections.OrderedDict()
 908             repo_file['name'] = os.path.splitext(name_utf8)[0]
 909             # TODO rename apkname globally to something more generic
 910             repo_file['apkName'] = name_utf8
 911             repo_file['hash'] = shasum
 912             repo_file['hashType'] = 'sha256'
 913             repo_file['versionCode'] = 0
 914             repo_file['versionName'] = shasum
 915             # the static ID is the SHA256 unless it is set in the metadata
 916             repo_file['packageName'] = shasum
 917
 918             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 919             if m:
 920                 repo_file['packageName'] = m.group(1)
 921                 repo_file['versionCode'] = int(m.group(2))
 922             srcfilename = name + b'_src.tar.gz'
 923             if os.path.exists(os.path.join(repodir, srcfilename)):
 924                 repo_file['srcname'] = srcfilename.decode('utf-8')
 925             repo_file['size'] = stat.st_size
 926
 927             apkcache[name] = repo_file
 928             cachechanged = True
 929
 930         if use_date_from_file:
 931             timestamp = stat.st_ctime
 932             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 933         else:
 934             default_date_param = None
 935
 936         # Record in knownapks, getting the added date at the same time..
 937         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 938                                     default_date=default_date_param)
 939         if added:
 940             repo_file['added'] = added
 941
 942         repo_files.append(repo_file)
 943
 944     return repo_files, cachechanged
 945
 946
 947 def scan_apk(apk_file):
 948     """
 949     Scans an APK file and returns dictionary with metadata of the APK.
 950
 951     Attention: This does *not* verify that the APK signature is correct.
 952
 953     :param apk_file: The (ideally absolute) path to the APK file
 954     :raises BuildException
 955     :return A dict containing APK metadata
 956     """
 957     apk = {
 958         'hash': sha256sum(apk_file),
 959         'hashType': 'sha256',
 960         'uses-permission': [],
 961         'uses-permission-sdk-23': [],
 962         'features': [],
 963         'icons_src': {},
 964         'icons': {},
 965         'antiFeatures': set(),
 966     }
 967
 968     if SdkToolsPopen(['aapt', 'version'], output=False):
 969         scan_apk_aapt(apk, apk_file)
 970     else:
 971         scan_apk_androguard(apk, apk_file)
 972
 973     # Get the signature
 974     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 975     apk['sig'] = getsig(apk_file)
 976     if not apk['sig']:
 977         raise BuildException("Failed to get apk signature")
 978
 979     # Get size of the APK
 980     apk['size'] = os.path.getsize(apk_file)
 981
 982     if 'minSdkVersion' not in apk:
 983         logging.warning("No SDK version information found in {0}".format(apk_file))
 984         apk['minSdkVersion'] = 1
 985
 986     # Check for known vulnerabilities
 987     if has_known_vulnerability(apk_file):
 988         apk['antiFeatures'].add('KnownVuln')
 989
 990     return apk
 991
 992
 993 def scan_apk_aapt(apk, apkfile):
 994     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 995     if p.returncode != 0:
 996         if options.delete_unknown:
 997             if os.path.exists(apkfile):
 998                 logging.error("Failed to get apk information, deleting " + apkfile)
 999                 os.remove(apkfile)
1000             else:
1001                 logging.error("Could not find {0} to remove it".format(apkfile))
1002         else:
1003             logging.error("Failed to get apk information, skipping " + apkfile)
1004         raise BuildException("Invalid APK")
1005     for line in p.output.splitlines():
1006         if line.startswith("package:"):
1007             try:
1008                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1009                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1010                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1011             except Exception as e:
1012                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1013         elif line.startswith("application:"):
1014             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1015             # Keep path to non-dpi icon in case we need it
1016             match = re.match(APK_ICON_PAT_NODPI, line)
1017             if match:
1018                 apk['icons_src']['-1'] = match.group(1)
1019         elif line.startswith("launchable-activity:"):
1020             # Only use launchable-activity as fallback to application
1021             if not apk['name']:
1022                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1023             if '-1' not in apk['icons_src']:
1024                 match = re.match(APK_ICON_PAT_NODPI, line)
1025                 if match:
1026                     apk['icons_src']['-1'] = match.group(1)
1027         elif line.startswith("application-icon-"):
1028             match = re.match(APK_ICON_PAT, line)
1029             if match:
1030                 density = match.group(1)
1031                 path = match.group(2)
1032                 apk['icons_src'][density] = path
1033         elif line.startswith("sdkVersion:"):
1034             m = re.match(APK_SDK_VERSION_PAT, line)
1035             if m is None:
1036                 logging.error(line.replace('sdkVersion:', '')
1037                               + ' is not a valid minSdkVersion!')
1038             else:
1039                 apk['minSdkVersion'] = m.group(1)
1040                 # if target not set, default to min
1041                 if 'targetSdkVersion' not in apk:
1042                     apk['targetSdkVersion'] = m.group(1)
1043         elif line.startswith("targetSdkVersion:"):
1044             m = re.match(APK_SDK_VERSION_PAT, line)
1045             if m is None:
1046                 logging.error(line.replace('targetSdkVersion:', '')
1047                               + ' is not a valid targetSdkVersion!')
1048             else:
1049                 apk['targetSdkVersion'] = m.group(1)
1050         elif line.startswith("maxSdkVersion:"):
1051             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1052         elif line.startswith("native-code:"):
1053             apk['nativecode'] = []
1054             for arch in line[13:].split(' '):
1055                 apk['nativecode'].append(arch[1:-1])
1056         elif line.startswith('uses-permission:'):
1057             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1058             if perm_match['maxSdkVersion']:
1059                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1060             permission = UsesPermission(
1061                 perm_match['name'],
1062                 perm_match['maxSdkVersion']
1063             )
1064
1065             apk['uses-permission'].append(permission)
1066         elif line.startswith('uses-permission-sdk-23:'):
1067             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1068             if perm_match['maxSdkVersion']:
1069                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1070             permission_sdk_23 = UsesPermissionSdk23(
1071                 perm_match['name'],
1072                 perm_match['maxSdkVersion']
1073             )
1074
1075             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1076
1077         elif line.startswith('uses-feature:'):
1078             feature = re.match(APK_FEATURE_PAT, line).group(1)
1079             # Filter out this, it's only added with the latest SDK tools and
1080             # causes problems for lots of apps.
1081             if feature != "android.hardware.screen.portrait" \
1082                     and feature != "android.hardware.screen.landscape":
1083                 if feature.startswith("android.feature."):
1084                     feature = feature[16:]
1085                 apk['features'].add(feature)
1086
1087
1088 def scan_apk_androguard(apk, apkfile):
1089     try:
1090         from androguard.core.bytecodes.apk import APK
1091         apkobject = APK(apkfile)
1092         if apkobject.is_valid_APK():
1093             arsc = apkobject.get_android_resources()
1094         else:
1095             if options.delete_unknown:
1096                 if os.path.exists(apkfile):
1097                     logging.error("Failed to get apk information, deleting " + apkfile)
1098                     os.remove(apkfile)
1099                 else:
1100                     logging.error("Could not find {0} to remove it".format(apkfile))
1101             else:
1102                 logging.error("Failed to get apk information, skipping " + apkfile)
1103             raise BuildException("Invaild APK")
1104     except ImportError:
1105         raise FDroidException("androguard library is not installed and aapt not present")
1106     except FileNotFoundError:
1107         logging.error("Could not open apk file for analysis")
1108         raise BuildException("Invalid APK")
1109
1110     apk['packageName'] = apkobject.get_package()
1111     apk['versionCode'] = int(apkobject.get_androidversion_code())
1112     apk['versionName'] = apkobject.get_androidversion_name()
1113     if apk['versionName'][0] == "@":
1114         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1115         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1116         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1117     apk['name'] = apkobject.get_app_name()
1118
1119     if apkobject.get_max_sdk_version() is not None:
1120         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1121     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1122     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1123
1124     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1125     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1126
1127     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1128
1129     for file in apkobject.get_files():
1130         d_re = density_re.match(file)
1131         if d_re:
1132             folder = d_re.group(1).split('-')
1133             if len(folder) > 1:
1134                 resolution = folder[1]
1135             else:
1136                 resolution = 'mdpi'
1137             density = screen_resolutions[resolution]
1138             apk['icons_src'][density] = d_re.group(0)
1139
1140     if apk['icons_src'].get('-1') is None:
1141         apk['icons_src']['-1'] = apk['icons_src']['160']
1142
1143     arch_re = re.compile("^lib/(.*)/.*$")
1144     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1145     if len(arch) >= 1:
1146         apk['nativecode'] = []
1147         apk['nativecode'].extend(sorted(list(arch)))
1148
1149     xml = apkobject.get_android_manifest_xml()
1150
1151     for item in xml.getElementsByTagName('uses-permission'):
1152         name = str(item.getAttribute("android:name"))
1153         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1154         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1155         permission = UsesPermission(
1156             name,
1157             maxSdkVersion
1158         )
1159         apk['uses-permission'].append(permission)
1160
1161     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1162         name = str(item.getAttribute("android:name"))
1163         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1164         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1165         permission_sdk_23 = UsesPermissionSdk23(
1166             name,
1167             maxSdkVersion
1168         )
1169         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1170
1171     for item in xml.getElementsByTagName('uses-feature'):
1172         feature = str(item.getAttribute("android:name"))
1173         if feature != "android.hardware.screen.portrait" \
1174                 and feature != "android.hardware.screen.landscape":
1175             if feature.startswith("android.feature."):
1176                 feature = feature[16:]
1177         apk['features'].append(feature)
1178
1179
1180 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1181                 allow_disabled_algorithms=False, archive_bad_sig=False):
1182     """Processes the apk with the given filename in the given repo directory.
1183
1184     This also extracts the icons.
1185
1186     :param apkcache: current apk cache information
1187     :param apkfilename: the filename of the apk to scan
1188     :param repodir: repo directory to scan
1189     :param knownapks: known apks info
1190     :param use_date_from_apk: use date from APK (instead of current date)
1191                               for newly added APKs
1192     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1193                                       disabled algorithms in the signature (e.g. MD5)
1194     :param archive_bad_sig: move APKs with a bad signature to the archive
1195     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1196      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1197     """
1198
1199     if ' ' in apkfilename:
1200         if options.rename_apks:
1201             newfilename = apkfilename.replace(' ', '_')
1202             os.rename(os.path.join(repodir, apkfilename),
1203                       os.path.join(repodir, newfilename))
1204             apkfilename = newfilename
1205         else:
1206             logging.critical("Spaces in filenames are not allowed.")
1207             return True, None, False
1208
1209     apk = {}
1210     apkfile = os.path.join(repodir, apkfilename)
1211
1212     cachechanged = False
1213     usecache = False
1214     if apkfilename in apkcache:
1215         apk = apkcache[apkfilename]
1216         if apk.get('hash') == sha256sum(apkfile):
1217             logging.debug("Reading " + apkfilename + " from cache")
1218             usecache = True
1219         else:
1220             logging.debug("Ignoring stale cache data for " + apkfilename)
1221
1222     if not usecache:
1223         logging.debug("Processing " + apkfilename)
1224
1225         try:
1226             apk = scan_apk(apkfile)
1227         except BuildException:
1228             logging.warning('Skipping "%s" with invalid signature!', apkfilename)
1229             return True, None, False
1230
1231         # Check for debuggable apks...
1232         if common.isApkAndDebuggable(apkfile):
1233             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1234
1235         if options.rename_apks:
1236             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1237             std_short_name = os.path.join(repodir, n)
1238             if apkfile != std_short_name:
1239                 if os.path.exists(std_short_name):
1240                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1241                     if apkfile != std_long_name:
1242                         if os.path.exists(std_long_name):
1243                             dupdir = os.path.join('duplicates', repodir)
1244                             if not os.path.isdir(dupdir):
1245                                 os.makedirs(dupdir, exist_ok=True)
1246                             dupfile = os.path.join('duplicates', std_long_name)
1247                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1248                             os.rename(apkfile, dupfile)
1249                             return True, None, False
1250                         else:
1251                             os.rename(apkfile, std_long_name)
1252                     apkfile = std_long_name
1253                 else:
1254                     os.rename(apkfile, std_short_name)
1255                     apkfile = std_short_name
1256                 apkfilename = apkfile[len(repodir) + 1:]
1257
1258         apk['apkName'] = apkfilename
1259         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1260         if os.path.exists(os.path.join(repodir, srcfilename)):
1261             apk['srcname'] = srcfilename
1262
1263         # verify the jar signature is correct, allow deprecated
1264         # algorithms only if the APK is in the archive.
1265         skipapk = False
1266         if not common.verify_apk_signature(apkfile):
1267             if repodir == 'archive' or allow_disabled_algorithms:
1268                 if common.verify_old_apk_signature(apkfile):
1269                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1270                 else:
1271                     skipapk = True
1272             else:
1273                 skipapk = True
1274
1275         if skipapk:
1276             if archive_bad_sig:
1277                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1278                 move_apk_between_sections(repodir, 'archive', apk)
1279             else:
1280                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1281             return True, None, False
1282
1283         apkzip = zipfile.ZipFile(apkfile, 'r')
1284
1285         # if an APK has files newer than the system time, suggest updating
1286         # the system clock.  This is useful for offline systems, used for
1287         # signing, which do not have another source of clock sync info. It
1288         # has to be more than 24 hours newer because ZIP/APK files do not
1289         # store timezone info
1290         manifest = apkzip.getinfo('AndroidManifest.xml')
1291         if manifest.date_time[1] == 0:  # month can't be zero
1292             logging.debug('AndroidManifest.xml has no date')
1293         else:
1294             dt_obj = datetime(*manifest.date_time)
1295             checkdt = dt_obj - timedelta(1)
1296             if datetime.today() < checkdt:
1297                 logging.warning('System clock is older than manifest in: '
1298                                 + apkfilename
1299                                 + '\nSet clock to that time using:\n'
1300                                 + 'sudo date -s "' + str(dt_obj) + '"')
1301
1302         # extract icons from APK zip file
1303         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1304         try:
1305             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1306         finally:
1307             apkzip.close()  # ensure that APK zip file gets closed
1308
1309         # resize existing icons for densities missing in the APK
1310         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1311
1312         if use_date_from_apk and manifest.date_time[1] != 0:
1313             default_date_param = datetime(*manifest.date_time)
1314         else:
1315             default_date_param = None
1316
1317         # Record in known apks, getting the added date at the same time..
1318         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1319                                     default_date=default_date_param)
1320         if added:
1321             apk['added'] = added
1322
1323         apkcache[apkfilename] = apk
1324         cachechanged = True
1325
1326     return False, apk, cachechanged
1327
1328
1329 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1330     """Processes the apks in the given repo directory.
1331
1332     This also extracts the icons.
1333
1334     :param apkcache: current apk cache information
1335     :param repodir: repo directory to scan
1336     :param knownapks: known apks info
1337     :param use_date_from_apk: use date from APK (instead of current date)
1338                               for newly added APKs
1339     :returns: (apks, cachechanged) where apks is a list of apk information,
1340               and cachechanged is True if the apkcache got changed.
1341     """
1342
1343     cachechanged = False
1344
1345     for icon_dir in get_all_icon_dirs(repodir):
1346         if os.path.exists(icon_dir):
1347             if options.clean:
1348                 shutil.rmtree(icon_dir)
1349                 os.makedirs(icon_dir)
1350         else:
1351             os.makedirs(icon_dir)
1352
1353     apks = []
1354     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1355         apkfilename = apkfile[len(repodir) + 1:]
1356         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1357         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1358                                              use_date_from_apk, ada, True)
1359         if skip:
1360             continue
1361         apks.append(apk)
1362         cachechanged = cachechanged or cachethis
1363
1364     return apks, cachechanged
1365
1366
1367 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1368     """
1369     Extracts icons from the given APK zip in various densities,
1370     saves them into given repo directory
1371     and stores their names in the APK metadata dictionary.
1372
1373     :param icon_filename: A string representing the icon's file name
1374     :param apk: A populated dictionary containing APK metadata.
1375                 Needs to have 'icons_src' key
1376     :param apkzip: An opened zipfile.ZipFile of the APK file
1377     :param repo_dir: The directory of the APK's repository
1378     :return: A list of icon densities that are missing
1379     """
1380     empty_densities = []
1381     for density in screen_densities:
1382         if density not in apk['icons_src']:
1383             empty_densities.append(density)
1384             continue
1385         icon_src = apk['icons_src'][density]
1386         icon_dir = get_icon_dir(repo_dir, density)
1387         icon_dest = os.path.join(icon_dir, icon_filename)
1388
1389         # Extract the icon files per density
1390         if icon_src.endswith('.xml'):
1391             png = os.path.basename(icon_src)[:-4] + '.png'
1392             for f in apkzip.namelist():
1393                 if f.endswith(png):
1394                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1395                     if m and screen_resolutions[m.group(2)] == density:
1396                         icon_src = f
1397             if icon_src.endswith('.xml'):
1398                 empty_densities.append(density)
1399                 continue
1400         try:
1401             with open(icon_dest, 'wb') as f:
1402                 f.write(get_icon_bytes(apkzip, icon_src))
1403             apk['icons'][density] = icon_filename
1404         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1405             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1406             del apk['icons_src'][density]
1407             empty_densities.append(density)
1408
1409     if '-1' in apk['icons_src']:
1410         icon_src = apk['icons_src']['-1']
1411         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1412         with open(icon_path, 'wb') as f:
1413             f.write(get_icon_bytes(apkzip, icon_src))
1414         try:
1415             im = Image.open(icon_path)
1416             dpi = px_to_dpi(im.size[0])
1417             for density in screen_densities:
1418                 if density in apk['icons']:
1419                     break
1420                 if density == screen_densities[-1] or dpi >= int(density):
1421                     apk['icons'][density] = icon_filename
1422                     shutil.move(icon_path,
1423                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1424                     empty_densities.remove(density)
1425                     break
1426         except Exception as e:
1427             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1428
1429     if apk['icons']:
1430         apk['icon'] = icon_filename
1431
1432     return empty_densities
1433
1434
1435 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1436     """
1437     Resize existing icons for densities missing in the APK to ensure all densities are available
1438
1439     :param empty_densities: A list of icon densities that are missing
1440     :param icon_filename: A string representing the icon's file name
1441     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1442     :param repo_dir: The directory of the APK's repository
1443     """
1444     # First try resizing down to not lose quality
1445     last_density = None
1446     for density in screen_densities:
1447         if density not in empty_densities:
1448             last_density = density
1449             continue
1450         if last_density is None:
1451             continue
1452         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1453
1454         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1455         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1456         fp = None
1457         try:
1458             fp = open(last_icon_path, 'rb')
1459             im = Image.open(fp)
1460
1461             size = dpi_to_px(density)
1462
1463             im.thumbnail((size, size), Image.ANTIALIAS)
1464             im.save(icon_path, "PNG")
1465             empty_densities.remove(density)
1466         except Exception as e:
1467             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1468         finally:
1469             if fp:
1470                 fp.close()
1471
1472     # Then just copy from the highest resolution available
1473     last_density = None
1474     for density in reversed(screen_densities):
1475         if density not in empty_densities:
1476             last_density = density
1477             continue
1478
1479         if last_density is None:
1480             continue
1481
1482         shutil.copyfile(
1483             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1484             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1485         )
1486         empty_densities.remove(density)
1487
1488     for density in screen_densities:
1489         icon_dir = get_icon_dir(repo_dir, density)
1490         icon_dest = os.path.join(icon_dir, icon_filename)
1491         resize_icon(icon_dest, density)
1492
1493     # Copy from icons-mdpi to icons since mdpi is the baseline density
1494     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1495     if os.path.isfile(baseline):
1496         apk['icons']['0'] = icon_filename
1497         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1498
1499
1500 def apply_info_from_latest_apk(apps, apks):
1501     """
1502     Some information from the apks needs to be applied up to the application level.
1503     When doing this, we use the info from the most recent version's apk.
1504     We deal with figuring out when the app was added and last updated at the same time.
1505     """
1506     for appid, app in apps.items():
1507         bestver = UNSET_VERSION_CODE
1508         for apk in apks:
1509             if apk['packageName'] == appid:
1510                 if apk['versionCode'] > bestver:
1511                     bestver = apk['versionCode']
1512                     bestapk = apk
1513
1514                 if 'added' in apk:
1515                     if not app.added or apk['added'] < app.added:
1516                         app.added = apk['added']
1517                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1518                         app.lastUpdated = apk['added']
1519
1520         if not app.added:
1521             logging.debug("Don't know when " + appid + " was added")
1522         if not app.lastUpdated:
1523             logging.debug("Don't know when " + appid + " was last updated")
1524
1525         if bestver == UNSET_VERSION_CODE:
1526
1527             if app.Name is None:
1528                 app.Name = app.AutoName or appid
1529             app.icon = None
1530             logging.debug("Application " + appid + " has no packages")
1531         else:
1532             if app.Name is None:
1533                 app.Name = bestapk['name']
1534             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1535             if app.CurrentVersionCode is None:
1536                 app.CurrentVersionCode = str(bestver)
1537
1538
1539 def make_categories_txt(repodir, categories):
1540     '''Write a category list in the repo to allow quick access'''
1541     catdata = ''
1542     for cat in sorted(categories):
1543         catdata += cat + '\n'
1544     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1545         f.write(catdata)
1546
1547
1548 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1549
1550     def filter_apk_list_sorted(apk_list):
1551         res = []
1552         for apk in apk_list:
1553             if apk['packageName'] == appid:
1554                 res.append(apk)
1555
1556         # Sort the apk list by version code. First is highest/newest.
1557         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1558
1559     for appid, app in apps.items():
1560
1561         if app.ArchivePolicy:
1562             keepversions = int(app.ArchivePolicy[:-9])
1563         else:
1564             keepversions = defaultkeepversions
1565
1566         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1567                       .format(appid, len(apks), keepversions, len(archapks)))
1568
1569         current_app_apks = filter_apk_list_sorted(apks)
1570         if len(current_app_apks) > keepversions:
1571             # Move back the ones we don't want.
1572             for apk in current_app_apks[keepversions:]:
1573                 move_apk_between_sections(repodir, archivedir, apk)
1574                 archapks.append(apk)
1575                 apks.remove(apk)
1576
1577         current_app_archapks = filter_apk_list_sorted(archapks)
1578         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1579             kept = 0
1580             # Move forward the ones we want again, except DisableAlgorithm
1581             for apk in current_app_archapks:
1582                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1583                     move_apk_between_sections(archivedir, repodir, apk)
1584                     archapks.remove(apk)
1585                     apks.append(apk)
1586                     kept += 1
1587                 if kept == keepversions:
1588                     break
1589
1590
1591 def move_apk_between_sections(from_dir, to_dir, apk):
1592     """move an APK from repo to archive or vice versa"""
1593
1594     def _move_file(from_dir, to_dir, filename, ignore_missing):
1595         from_path = os.path.join(from_dir, filename)
1596         if ignore_missing and not os.path.exists(from_path):
1597             return
1598         to_path = os.path.join(to_dir, filename)
1599         if not os.path.exists(to_dir):
1600             os.mkdir(to_dir)
1601         shutil.move(from_path, to_path)
1602
1603     if from_dir == to_dir:
1604         return
1605
1606     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1607     _move_file(from_dir, to_dir, apk['apkName'], False)
1608     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1609     for density in all_screen_densities:
1610         from_icon_dir = get_icon_dir(from_dir, density)
1611         to_icon_dir = get_icon_dir(to_dir, density)
1612         if density not in apk['icons']:
1613             continue
1614         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1615     if 'srcname' in apk:
1616         _move_file(from_dir, to_dir, apk['srcname'], False)
1617
1618
1619 def add_apks_to_per_app_repos(repodir, apks):
1620     apks_per_app = dict()
1621     for apk in apks:
1622         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1623         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1624         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1625         apks_per_app[apk['packageName']] = apk
1626
1627         if not os.path.exists(apk['per_app_icons']):
1628             logging.info('Adding new repo for only ' + apk['packageName'])
1629             os.makedirs(apk['per_app_icons'])
1630
1631         apkpath = os.path.join(repodir, apk['apkName'])
1632         shutil.copy(apkpath, apk['per_app_repo'])
1633         apksigpath = apkpath + '.sig'
1634         if os.path.exists(apksigpath):
1635             shutil.copy(apksigpath, apk['per_app_repo'])
1636         apkascpath = apkpath + '.asc'
1637         if os.path.exists(apkascpath):
1638             shutil.copy(apkascpath, apk['per_app_repo'])
1639
1640
1641 config = None
1642 options = None
1643
1644
1645 def main():
1646
1647     global config, options
1648
1649     # Parse command line...
1650     parser = ArgumentParser()
1651     common.setup_global_opts(parser)
1652     parser.add_argument("--create-key", action="store_true", default=False,
1653                         help="Create a repo signing key in a keystore")
1654     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1655                         help="Create skeleton metadata files that are missing")
1656     parser.add_argument("--delete-unknown", action="store_true", default=False,
1657                         help="Delete APKs and/or OBBs without metadata from the repo")
1658     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1659                         help="Report on build data status")
1660     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1661                         help="Interactively ask about things that need updating.")
1662     parser.add_argument("-I", "--icons", action="store_true", default=False,
1663                         help="Resize all the icons exceeding the max pixel size and exit")
1664     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1665                         help="Specify editor to use in interactive mode. Default " +
1666                         "is /etc/alternatives/editor")
1667     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1668                         help="Update the wiki")
1669     parser.add_argument("--pretty", action="store_true", default=False,
1670                         help="Produce human-readable index.xml")
1671     parser.add_argument("--clean", action="store_true", default=False,
1672                         help="Clean update - don't uses caches, reprocess all apks")
1673     parser.add_argument("--nosign", action="store_true", default=False,
1674                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1675     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1676                         help="Use date from apk instead of current time for newly added apks")
1677     parser.add_argument("--rename-apks", action="store_true", default=False,
1678                         help="Rename APK files that do not match package.name_123.apk")
1679     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1680                         help="Include APKs that are signed with disabled algorithms like MD5")
1681     metadata.add_metadata_arguments(parser)
1682     options = parser.parse_args()
1683     metadata.warnings_action = options.W
1684
1685     config = common.read_config(options)
1686
1687     if not ('jarsigner' in config and 'keytool' in config):
1688         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1689
1690     repodirs = ['repo']
1691     if config['archive_older'] != 0:
1692         repodirs.append('archive')
1693         if not os.path.exists('archive'):
1694             os.mkdir('archive')
1695
1696     if options.icons:
1697         resize_all_icons(repodirs)
1698         sys.exit(0)
1699
1700     if options.rename_apks:
1701         options.clean = True
1702
1703     # check that icons exist now, rather than fail at the end of `fdroid update`
1704     for k in ['repo_icon', 'archive_icon']:
1705         if k in config:
1706             if not os.path.exists(config[k]):
1707                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1708                 sys.exit(1)
1709
1710     # if the user asks to create a keystore, do it now, reusing whatever it can
1711     if options.create_key:
1712         if os.path.exists(config['keystore']):
1713             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1714             logging.critical("\t'" + config['keystore'] + "'")
1715             sys.exit(1)
1716
1717         if 'repo_keyalias' not in config:
1718             config['repo_keyalias'] = socket.getfqdn()
1719             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1720         if 'keydname' not in config:
1721             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1722             common.write_to_config(config, 'keydname', config['keydname'])
1723         if 'keystore' not in config:
1724             config['keystore'] = common.default_config['keystore']
1725             common.write_to_config(config, 'keystore', config['keystore'])
1726
1727         password = common.genpassword()
1728         if 'keystorepass' not in config:
1729             config['keystorepass'] = password
1730             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1731         if 'keypass' not in config:
1732             config['keypass'] = password
1733             common.write_to_config(config, 'keypass', config['keypass'])
1734         common.genkeystore(config)
1735
1736     # Get all apps...
1737     apps = metadata.read_metadata()
1738
1739     # Generate a list of categories...
1740     categories = set()
1741     for app in apps.values():
1742         categories.update(app.Categories)
1743
1744     # Read known apks data (will be updated and written back when we've finished)
1745     knownapks = common.KnownApks()
1746
1747     # Get APK cache
1748     apkcache = get_cache()
1749
1750     # Delete builds for disabled apps
1751     delete_disabled_builds(apps, apkcache, repodirs)
1752
1753     # Scan all apks in the main repo
1754     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1755
1756     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1757                                            options.use_date_from_apk)
1758     cachechanged = cachechanged or fcachechanged
1759     apks += files
1760     # Generate warnings for apk's with no metadata (or create skeleton
1761     # metadata files, if requested on the command line)
1762     newmetadata = False
1763     for apk in apks:
1764         if apk['packageName'] not in apps:
1765             if options.create_metadata:
1766                 import yaml
1767                 with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1768                     # this should use metadata.App() and
1769                     # metadata.write_yaml(), but since ruamel.yaml
1770                     # 0.13 is not widely distributed yet, and it's
1771                     # special tricks are not really needed here, this
1772                     # uses the plain YAML lib
1773                     if os.path.exists('template.yml'):
1774                         with open('template.yml') as f:
1775                             metatxt = f.read()
1776                         if 'name' in apk and apk['name'] != '':
1777                             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$', r'\1 ' + apk['name'], metatxt, flags=re.IGNORECASE | re.MULTILINE)
1778                         else:
1779                             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1780                             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$', r'\1 ' + apk['packageName'], metatxt, flags=re.IGNORECASE | re.MULTILINE)
1781                         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1782                             f.write(metatxt)
1783                     else:
1784                         app = dict()
1785                         app['Categories'] = [os.path.basename(os.getcwd())]
1786                         # include some blanks as part of the template
1787                         app['AuthorName'] = ''
1788                         app['Summary'] = ''
1789                         app['WebSite'] = ''
1790                         app['IssueTracker'] = ''
1791                         app['SourceCode'] = ''
1792                         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1793                         if 'name' in apk and apk['name'] != '':
1794                             app['Name'] = apk['name']
1795                         else:
1796                             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1797                             app['Name'] = apk['packageName']
1798                         yaml.dump(app, f, default_flow_style=False)
1799                     logging.info("Generated skeleton metadata for " + apk['packageName'])
1800                     newmetadata = True
1801             else:
1802                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1803                 if options.delete_unknown:
1804                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1805                     rmf = os.path.join(repodirs[0], apk['apkName'])
1806                     if not os.path.exists(rmf):
1807                         logging.error("Could not find {0} to remove it".format(rmf))
1808                     else:
1809                         os.remove(rmf)
1810                 else:
1811                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1812
1813     # update the metadata with the newly created ones included
1814     if newmetadata:
1815         apps = metadata.read_metadata()
1816
1817     copy_triple_t_store_metadata(apps)
1818     insert_obbs(repodirs[0], apps, apks)
1819     insert_localized_app_metadata(apps)
1820     translate_per_build_anti_features(apps, apks)
1821
1822     # Scan the archive repo for apks as well
1823     if len(repodirs) > 1:
1824         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1825         if cc:
1826             cachechanged = True
1827     else:
1828         archapks = []
1829
1830     # Apply information from latest apks to the application and update dates
1831     apply_info_from_latest_apk(apps, apks + archapks)
1832
1833     # Sort the app list by name, then the web site doesn't have to by default.
1834     # (we had to wait until we'd scanned the apks to do this, because mostly the
1835     # name comes from there!)
1836     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1837
1838     # APKs are placed into multiple repos based on the app package, providing
1839     # per-app subscription feeds for nightly builds and things like it
1840     if config['per_app_repos']:
1841         add_apks_to_per_app_repos(repodirs[0], apks)
1842         for appid, app in apps.items():
1843             repodir = os.path.join(appid, 'fdroid', 'repo')
1844             appdict = dict()
1845             appdict[appid] = app
1846             if os.path.isdir(repodir):
1847                 index.make(appdict, [appid], apks, repodir, False)
1848             else:
1849                 logging.info('Skipping index generation for ' + appid)
1850         return
1851
1852     if len(repodirs) > 1:
1853         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1854
1855     # Make the index for the main repo...
1856     index.make(apps, sortedids, apks, repodirs[0], False)
1857     make_categories_txt(repodirs[0], categories)
1858
1859     # If there's an archive repo,  make the index for it. We already scanned it
1860     # earlier on.
1861     if len(repodirs) > 1:
1862         index.make(apps, sortedids, archapks, repodirs[1], True)
1863
1864     git_remote = config.get('binary_transparency_remote')
1865     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1866         from . import btlog
1867         btlog.make_binary_transparency_log(repodirs)
1868
1869     if config['update_stats']:
1870         # Update known apks info...
1871         knownapks.writeifchanged()
1872
1873         # Generate latest apps data for widget
1874         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1875             data = ''
1876             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1877                 for line in f:
1878                     appid = line.rstrip()
1879                     data += appid + "\t"
1880                     app = apps[appid]
1881                     data += app.Name + "\t"
1882                     if app.icon is not None:
1883                         data += app.icon + "\t"
1884                     data += app.License + "\n"
1885             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1886                 f.write(data)
1887
1888     if cachechanged:
1889         write_cache(apkcache)
1890
1891     # Update the wiki...
1892     if options.wiki:
1893         update_wiki(apps, sortedids, apks + archapks)
1894
1895     logging.info("Finished.")
1896
1897
1898 if __name__ == "__main__":
1899     main()