fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0':
 100         return os.path.join(repodir, "icons")
 101     return os.path.join(repodir, "icons-%s" % density)
 102
 103
 104 def get_icon_dirs(repodir):
 105     for density in screen_densities:
 106         yield get_icon_dir(repodir, density)
 107
 108
 109 def get_all_icon_dirs(repodir):
 110     for density in all_screen_densities:
 111         yield get_icon_dir(repodir, density)
 112
 113
 114 def update_wiki(apps, sortedids, apks):
 115     """Update the wiki
 116
 117     :param apps: fully populated list of all applications
 118     :param apks: all apks, except...
 119     """
 120     logging.info("Updating wiki")
 121     wikicat = 'Apps'
 122     wikiredircat = 'App Redirects'
 123     import mwclient
 124     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 125                          path=config['wiki_path'])
 126     site.login(config['wiki_user'], config['wiki_password'])
 127     generated_pages = {}
 128     generated_redirects = {}
 129
 130     for appid in sortedids:
 131         app = metadata.App(apps[appid])
 132
 133         wikidata = ''
 134         if app.Disabled:
 135             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 136         if app.AntiFeatures:
 137             for af in sorted(app.AntiFeatures):
 138                 wikidata += '{{AntiFeature|' + af + '}}\n'
 139         if app.RequiresRoot:
 140             requiresroot = 'Yes'
 141         else:
 142             requiresroot = 'No'
 143         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 144             appid,
 145             app.Name,
 146             app.added.strftime('%Y-%m-%d') if app.added else '',
 147             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 148             app.SourceCode,
 149             app.IssueTracker,
 150             app.WebSite,
 151             app.Changelog,
 152             app.Donate,
 153             app.FlattrID,
 154             app.LiberapayID,
 155             app.Bitcoin,
 156             app.Litecoin,
 157             app.License,
 158             requiresroot,
 159             app.AuthorName,
 160             app.AuthorEmail)
 161
 162         if app.Provides:
 163             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 164
 165         wikidata += app.Summary
 166         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 167
 168         wikidata += "=Description=\n"
 169         wikidata += metadata.description_wiki(app.Description) + "\n"
 170
 171         wikidata += "=Maintainer Notes=\n"
 172         if app.MaintainerNotes:
 173             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 174         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 175
 176         # Get a list of all packages for this application...
 177         apklist = []
 178         gotcurrentver = False
 179         cantupdate = False
 180         buildfails = False
 181         for apk in apks:
 182             if apk['packageName'] == appid:
 183                 if str(apk['versionCode']) == app.CurrentVersionCode:
 184                     gotcurrentver = True
 185                 apklist.append(apk)
 186         # Include ones we can't build, as a special case...
 187         for build in app.builds:
 188             if build.disable:
 189                 if build.versionCode == app.CurrentVersionCode:
 190                     cantupdate = True
 191                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 192                 apklist.append({'versionCode': int(build.versionCode),
 193                                 'versionName': build.versionName,
 194                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 195                                 })
 196             else:
 197                 builtit = False
 198                 for apk in apklist:
 199                     if apk['versionCode'] == int(build.versionCode):
 200                         builtit = True
 201                         break
 202                 if not builtit:
 203                     buildfails = True
 204                     apklist.append({'versionCode': int(build.versionCode),
 205                                     'versionName': build.versionName,
 206                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 207                                     })
 208         if app.CurrentVersionCode == '0':
 209             cantupdate = True
 210         # Sort with most recent first...
 211         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 212
 213         wikidata += "=Versions=\n"
 214         if len(apklist) == 0:
 215             wikidata += "We currently have no versions of this app available."
 216         elif not gotcurrentver:
 217             wikidata += "We don't have the current version of this app."
 218         else:
 219             wikidata += "We have the current version of this app."
 220         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 221         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 222         if len(app.NoSourceSince) > 0:
 223             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 224         if len(app.CurrentVersion) > 0:
 225             wikidata += "The current (recommended) version is " + app.CurrentVersion
 226             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 227         validapks = 0
 228         for apk in apklist:
 229             wikidata += "==" + apk['versionName'] + "==\n"
 230
 231             if 'buildproblem' in apk:
 232                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 233             else:
 234                 validapks += 1
 235                 wikidata += "This version is built and signed by "
 236                 if 'srcname' in apk:
 237                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 238                 else:
 239                     wikidata += "the original developer.\n\n"
 240             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 241
 242         wikidata += '\n[[Category:' + wikicat + ']]\n'
 243         if len(app.NoSourceSince) > 0:
 244             wikidata += '\n[[Category:Apps missing source code]]\n'
 245         if validapks == 0 and not app.Disabled:
 246             wikidata += '\n[[Category:Apps with no packages]]\n'
 247         if cantupdate and not app.Disabled:
 248             wikidata += "\n[[Category:Apps we cannot update]]\n"
 249         if buildfails and not app.Disabled:
 250             wikidata += "\n[[Category:Apps with failing builds]]\n"
 251         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 252             wikidata += '\n[[Category:Apps to Update]]\n'
 253         if app.Disabled:
 254             wikidata += '\n[[Category:Apps that are disabled]]\n'
 255         if app.UpdateCheckMode == 'None' and not app.Disabled:
 256             wikidata += '\n[[Category:Apps with no update check]]\n'
 257         for appcat in app.Categories:
 258             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 259
 260         # We can't have underscores in the page name, even if they're in
 261         # the package ID, because MediaWiki messes with them...
 262         pagename = appid.replace('_', ' ')
 263
 264         # Drop a trailing newline, because mediawiki is going to drop it anyway
 265         # and it we don't we'll think the page has changed when it hasn't...
 266         if wikidata.endswith('\n'):
 267             wikidata = wikidata[:-1]
 268
 269         generated_pages[pagename] = wikidata
 270
 271         # Make a redirect from the name to the ID too, unless there's
 272         # already an existing page with the name and it isn't a redirect.
 273         noclobber = False
 274         apppagename = app.Name.replace('_', ' ')
 275         apppagename = apppagename.replace('{', '')
 276         apppagename = apppagename.replace('}', ' ')
 277         apppagename = apppagename.replace(':', ' ')
 278         apppagename = apppagename.replace('[', ' ')
 279         apppagename = apppagename.replace(']', ' ')
 280         # Drop double spaces caused mostly by replacing ':' above
 281         apppagename = apppagename.replace('  ', ' ')
 282         for expagename in site.allpages(prefix=apppagename,
 283                                         filterredir='nonredirects',
 284                                         generator=False):
 285             if expagename == apppagename:
 286                 noclobber = True
 287         # Another reason not to make the redirect page is if the app name
 288         # is the same as it's ID, because that will overwrite the real page
 289         # with an redirect to itself! (Although it seems like an odd
 290         # scenario this happens a lot, e.g. where there is metadata but no
 291         # builds or binaries to extract a name from.
 292         if apppagename == pagename:
 293             noclobber = True
 294         if not noclobber:
 295             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 296
 297     for tcat, genp in [(wikicat, generated_pages),
 298                        (wikiredircat, generated_redirects)]:
 299         catpages = site.Pages['Category:' + tcat]
 300         existingpages = []
 301         for page in catpages:
 302             existingpages.append(page.name)
 303             if page.name in genp:
 304                 pagetxt = page.edit()
 305                 if pagetxt != genp[page.name]:
 306                     logging.debug("Updating modified page " + page.name)
 307                     page.save(genp[page.name], summary='Auto-updated')
 308                 else:
 309                     logging.debug("Page " + page.name + " is unchanged")
 310             else:
 311                 logging.warn("Deleting page " + page.name)
 312                 page.delete('No longer published')
 313         for pagename, text in genp.items():
 314             logging.debug("Checking " + pagename)
 315             if pagename not in existingpages:
 316                 logging.debug("Creating page " + pagename)
 317                 try:
 318                     newpage = site.Pages[pagename]
 319                     newpage.save(text, summary='Auto-created')
 320                 except Exception as e:
 321                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 322
 323     # Purge server cache to ensure counts are up to date
 324     site.pages['Repository Maintenance'].purge()
 325
 326
 327 def delete_disabled_builds(apps, apkcache, repodirs):
 328     """Delete disabled build outputs.
 329
 330     :param apps: list of all applications, as per metadata.read_metadata
 331     :param apkcache: current apk cache information
 332     :param repodirs: the repo directories to process
 333     """
 334     for appid, app in apps.items():
 335         for build in app['builds']:
 336             if not build.disable:
 337                 continue
 338             apkfilename = common.get_release_filename(app, build)
 339             iconfilename = "%s.%s.png" % (
 340                 appid,
 341                 build.versionCode)
 342             for repodir in repodirs:
 343                 files = [
 344                     os.path.join(repodir, apkfilename),
 345                     os.path.join(repodir, apkfilename + '.asc'),
 346                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 347                 ]
 348                 for density in all_screen_densities:
 349                     repo_dir = get_icon_dir(repodir, density)
 350                     files.append(os.path.join(repo_dir, iconfilename))
 351
 352                 for f in files:
 353                     if os.path.exists(f):
 354                         logging.info("Deleting disabled build output " + f)
 355                         os.remove(f)
 356             if apkfilename in apkcache:
 357                 del apkcache[apkfilename]
 358
 359
 360 def resize_icon(iconpath, density):
 361
 362     if not os.path.isfile(iconpath):
 363         return
 364
 365     fp = None
 366     try:
 367         fp = open(iconpath, 'rb')
 368         im = Image.open(fp)
 369         size = dpi_to_px(density)
 370
 371         if any(length > size for length in im.size):
 372             oldsize = im.size
 373             im.thumbnail((size, size), Image.ANTIALIAS)
 374             logging.debug("%s was too large at %s - new size is %s" % (
 375                 iconpath, oldsize, im.size))
 376             im.save(iconpath, "PNG", optimize=True,
 377                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 378
 379     except Exception as e:
 380         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 381
 382     finally:
 383         if fp:
 384             fp.close()
 385
 386
 387 def resize_all_icons(repodirs):
 388     """Resize all icons that exceed the max size
 389
 390     :param repodirs: the repo directories to process
 391     """
 392     for repodir in repodirs:
 393         for density in screen_densities:
 394             icon_dir = get_icon_dir(repodir, density)
 395             icon_glob = os.path.join(icon_dir, '*.png')
 396             for iconpath in glob.glob(icon_glob):
 397                 resize_icon(iconpath, density)
 398
 399
 400 def getsig(apkpath):
 401     """ Get the signing certificate of an apk. To get the same md5 has that
 402     Android gets, we encode the .RSA certificate in a specific format and pass
 403     it hex-encoded to the md5 digest algorithm.
 404
 405     :param apkpath: path to the apk
 406     :returns: A string containing the md5 of the signature of the apk or None
 407               if an error occurred.
 408     """
 409
 410     with zipfile.ZipFile(apkpath, 'r') as apk:
 411         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 412
 413         if len(certs) < 1:
 414             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 415             return None
 416         if len(certs) > 1:
 417             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 418             return None
 419
 420         cert = apk.read(certs[0])
 421
 422     cert_encoded = common.get_certificate(cert)
 423
 424     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 425
 426
 427 def get_cache_file():
 428     return os.path.join('tmp', 'apkcache')
 429
 430
 431 def get_cache():
 432     """Get the cached dict of the APK index
 433
 434     Gather information about all the apk files in the repo directory,
 435     using cached data if possible. Some of the index operations take a
 436     long time, like calculating the SHA-256 and verifying the APK
 437     signature.
 438
 439     The cache is invalidated if the metadata version is different, or
 440     the 'allow_disabled_algorithms' config/option is different.  In
 441     those cases, there is no easy way to know what has changed from
 442     the cache, so just rerun the whole thing.
 443
 444     :return: apkcache
 445
 446     """
 447     apkcachefile = get_cache_file()
 448     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 449     if not options.clean and os.path.exists(apkcachefile):
 450         with open(apkcachefile, 'rb') as cf:
 451             apkcache = pickle.load(cf, encoding='utf-8')
 452         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 453            or apkcache.get('allow_disabled_algorithms') != ada:
 454             apkcache = {}
 455     else:
 456         apkcache = {}
 457
 458     apkcache["METADATA_VERSION"] = METADATA_VERSION
 459     apkcache['allow_disabled_algorithms'] = ada
 460
 461     return apkcache
 462
 463
 464 def write_cache(apkcache):
 465     apkcachefile = get_cache_file()
 466     cache_path = os.path.dirname(apkcachefile)
 467     if not os.path.exists(cache_path):
 468         os.makedirs(cache_path)
 469     with open(apkcachefile, 'wb') as cf:
 470         pickle.dump(apkcache, cf)
 471
 472
 473 def get_icon_bytes(apkzip, iconsrc):
 474     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 475     try:
 476         return apkzip.read(iconsrc)
 477     except KeyError:
 478         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 479
 480
 481 def sha256sum(filename):
 482     '''Calculate the sha256 of the given file'''
 483     sha = hashlib.sha256()
 484     with open(filename, 'rb') as f:
 485         while True:
 486             t = f.read(16384)
 487             if len(t) == 0:
 488                 break
 489             sha.update(t)
 490     return sha.hexdigest()
 491
 492
 493 def has_known_vulnerability(filename):
 494     """checks for known vulnerabilities in the APK
 495
 496     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 497     version.  Google also enforces this:
 498     https://support.google.com/faqs/answer/6376725?hl=en
 499
 500     Checks whether there are more than one classes.dex or AndroidManifest.xml
 501     files, which is invalid and an essential part of the "Master Key" attack.
 502     http://www.saurik.com/id/17
 503
 504     Janus is similar to Master Key but is perhaps easier to scan for.
 505     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 506     """
 507
 508     found_vuln = False
 509
 510     # statically load this pattern
 511     if not hasattr(has_known_vulnerability, "pattern"):
 512         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 513
 514     with open(filename.encode(), 'rb') as fp:
 515         first4 = fp.read(4)
 516     if first4 != b'\x50\x4b\x03\x04':
 517         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 518                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 519                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 520
 521     files_in_apk = set()
 522     with zipfile.ZipFile(filename) as zf:
 523         for name in zf.namelist():
 524             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 525                 lib = zf.open(name)
 526                 while True:
 527                     chunk = lib.read(4096)
 528                     if chunk == b'':
 529                         break
 530                     m = has_known_vulnerability.pattern.search(chunk)
 531                     if m:
 532                         version = m.group(1).decode('ascii')
 533                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 534                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 535                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 536                             logging.debug(_('"{path}" contains recent {name} ({version})')
 537                                           .format(path=filename, name=name, version=version))
 538                         else:
 539                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 540                                             .format(path=filename, name=name, version=version))
 541                             found_vuln = True
 542                         break
 543             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 544                 if name in files_in_apk:
 545                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 546                                     .format(apkfilename=filename, name=name))
 547                     found_vuln = True
 548                 files_in_apk.add(name)
 549     return found_vuln
 550
 551
 552 def insert_obbs(repodir, apps, apks):
 553     """Scans the .obb files in a given repo directory and adds them to the
 554     relevant APK instances.  OBB files have versionCodes like APK
 555     files, and they are loosely associated.  If there is an OBB file
 556     present, then any APK with the same or higher versionCode will use
 557     that OBB file.  There are two OBB types: main and patch, each APK
 558     can only have only have one of each.
 559
 560     https://developer.android.com/google/play/expansion-files.html
 561
 562     :param repodir: repo directory to scan
 563     :param apps: list of current, valid apps
 564     :param apks: current information on all APKs
 565
 566     """
 567
 568     def obbWarnDelete(f, msg):
 569         logging.warning(msg + ' ' + f)
 570         if options.delete_unknown:
 571             logging.error(_("Deleting unknown file: {path}").format(path=f))
 572             os.remove(f)
 573
 574     obbs = []
 575     java_Integer_MIN_VALUE = -pow(2, 31)
 576     currentPackageNames = apps.keys()
 577     for f in glob.glob(os.path.join(repodir, '*.obb')):
 578         obbfile = os.path.basename(f)
 579         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 580         chunks = obbfile.split('.')
 581         if chunks[0] != 'main' and chunks[0] != 'patch':
 582             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 583             continue
 584         if not re.match(r'^-?[0-9]+$', chunks[1]):
 585             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 586                           .format(name=chunks[0]))
 587             continue
 588         versionCode = int(chunks[1])
 589         packagename = ".".join(chunks[2:-1])
 590
 591         highestVersionCode = java_Integer_MIN_VALUE
 592         if packagename not in currentPackageNames:
 593             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 594             continue
 595         for apk in apks:
 596             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 597                 highestVersionCode = apk['versionCode']
 598         if versionCode > highestVersionCode:
 599             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 600                           .format(integer=str(versionCode)))
 601             continue
 602         obbsha256 = sha256sum(f)
 603         obbs.append((packagename, versionCode, obbfile, obbsha256))
 604
 605     for apk in apks:
 606         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 607             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 608                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 609                     apk['obbMainFile'] = obbfile
 610                     apk['obbMainFileSha256'] = obbsha256
 611                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 612                     apk['obbPatchFile'] = obbfile
 613                     apk['obbPatchFileSha256'] = obbsha256
 614             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 615                 break
 616
 617
 618 def translate_per_build_anti_features(apps, apks):
 619     """Grab the anti-features list from the build metadata
 620
 621     For most Anti-Features, they are really most applicable per-APK,
 622     not for an app.  An app can fix a vulnerability, add/remove
 623     tracking, etc.  This reads the 'antifeatures' list from the Build
 624     entries in the fdroiddata metadata file, then transforms it into
 625     the 'antiFeatures' list of unique items for the index.
 626
 627     The field key is all lower case in the metadata file to match the
 628     rest of the Build fields.  It is 'antiFeatures' camel case in the
 629     implementation, index, and fdroidclient since it is translated
 630     from the build 'antifeatures' field, not directly included.
 631
 632     """
 633
 634     antiFeatures = dict()
 635     for packageName, app in apps.items():
 636         d = dict()
 637         for build in app['builds']:
 638             afl = build.get('antifeatures')
 639             if afl:
 640                 d[int(build.versionCode)] = afl
 641         if len(d) > 0:
 642             antiFeatures[packageName] = d
 643
 644     for apk in apks:
 645         d = antiFeatures.get(apk['packageName'])
 646         if d:
 647             afl = d.get(apk['versionCode'])
 648             if afl:
 649                 apk['antiFeatures'].update(afl)
 650
 651
 652 def _get_localized_dict(app, locale):
 653     '''get the dict to add localized store metadata to'''
 654     if 'localized' not in app:
 655         app['localized'] = collections.OrderedDict()
 656     if locale not in app['localized']:
 657         app['localized'][locale] = collections.OrderedDict()
 658     return app['localized'][locale]
 659
 660
 661 def _set_localized_text_entry(app, locale, key, f):
 662     limit = config['char_limits'][key]
 663     localized = _get_localized_dict(app, locale)
 664     with open(f) as fp:
 665         text = fp.read()[:limit]
 666         if len(text) > 0:
 667             localized[key] = text
 668
 669
 670 def _set_author_entry(app, key, f):
 671     limit = config['char_limits']['author']
 672     with open(f) as fp:
 673         text = fp.read()[:limit]
 674         if len(text) > 0:
 675             app[key] = text
 676
 677
 678 def _strip_and_copy_image(inpath, outpath):
 679     """Remove any metadata from image and copy it to new path
 680
 681     Sadly, image metadata like EXIF can be used to exploit devices.
 682     It is not used at all in the F-Droid ecosystem, so its much safer
 683     just to remove it entirely.
 684
 685     """
 686
 687     extension = common.get_extension(inpath)[1]
 688     if os.path.isdir(outpath):
 689         outpath = os.path.join(outpath, os.path.basename(inpath))
 690     if extension == 'png':
 691         with open(inpath, 'rb') as fp:
 692             in_image = Image.open(fp)
 693             in_image.save(outpath, "PNG", optimize=True,
 694                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 695     elif extension == 'jpg' or extension == 'jpeg':
 696         with open(inpath, 'rb') as fp:
 697             in_image = Image.open(fp)
 698             data = list(in_image.getdata())
 699             out_image = Image.new(in_image.mode, in_image.size)
 700         out_image.putdata(data)
 701         out_image.save(outpath, "JPEG", optimize=True)
 702     else:
 703         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 704                               .format(extension=extension))
 705
 706
 707 def copy_triple_t_store_metadata(apps):
 708     """Include store metadata from the app's source repo
 709
 710     The Triple-T Gradle Play Publisher is a plugin that has a standard
 711     file layout for all of the metadata and graphics that the Google
 712     Play Store accepts.  Since F-Droid has the git repo, it can just
 713     pluck those files directly.  This method reads any text files into
 714     the app dict, then copies any graphics into the fdroid repo
 715     directory structure.
 716
 717     This needs to be run before insert_localized_app_metadata() so that
 718     the graphics files that are copied into the fdroid repo get
 719     properly indexed.
 720
 721     https://github.com/Triple-T/gradle-play-publisher#upload-images
 722     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 723
 724     """
 725
 726     if not os.path.isdir('build'):
 727         return  # nothing to do
 728
 729     for packageName, app in apps.items():
 730         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 731             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 732             for root, dirs, files in os.walk(d):
 733                 segments = root.split('/')
 734                 locale = segments[-2]
 735                 for f in files:
 736                     if f == 'fulldescription':
 737                         _set_localized_text_entry(app, locale, 'description',
 738                                                   os.path.join(root, f))
 739                         continue
 740                     elif f == 'shortdescription':
 741                         _set_localized_text_entry(app, locale, 'summary',
 742                                                   os.path.join(root, f))
 743                         continue
 744                     elif f == 'title':
 745                         _set_localized_text_entry(app, locale, 'name',
 746                                                   os.path.join(root, f))
 747                         continue
 748                     elif f == 'video':
 749                         _set_localized_text_entry(app, locale, 'video',
 750                                                   os.path.join(root, f))
 751                         continue
 752                     elif f == 'whatsnew':
 753                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 754                                                   os.path.join(root, f))
 755                         continue
 756                     elif f == 'contactEmail':
 757                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 758                         continue
 759                     elif f == 'contactPhone':
 760                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 761                         continue
 762                     elif f == 'contactWebsite':
 763                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 764                         continue
 765
 766                     base, extension = common.get_extension(f)
 767                     dirname = os.path.basename(root)
 768                     if extension in ALLOWED_EXTENSIONS \
 769                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 770                         if segments[-2] == 'listing':
 771                             locale = segments[-3]
 772                         else:
 773                             locale = segments[-2]
 774                         destdir = os.path.join('repo', packageName, locale, dirname)
 775                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 776                         sourcefile = os.path.join(root, f)
 777                         destfile = os.path.join(destdir, os.path.basename(f))
 778                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 779                         _strip_and_copy_image(sourcefile, destfile)
 780
 781
 782 def insert_localized_app_metadata(apps):
 783     """scans standard locations for graphics and localized text
 784
 785     Scans for localized description files, store graphics, and
 786     screenshot PNG files in statically defined screenshots directory
 787     and adds them to the app metadata.  The screenshots and graphic
 788     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 789     and must be in the following layout:
 790     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 791
 792     repo/packageName/locale/featureGraphic.png
 793     repo/packageName/locale/phoneScreenshots/1.png
 794     repo/packageName/locale/phoneScreenshots/2.png
 795
 796     The changelog files must be text files named with the versionCode
 797     ending with ".txt" and must be in the following layout:
 798     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 799
 800     repo/packageName/locale/changelogs/12345.txt
 801
 802     This will scan the each app's source repo then the metadata/ dir
 803     for these standard locations of changelog files.  If it finds
 804     them, they will be added to the dict of all packages, with the
 805     versions in the metadata/ folder taking precendence over the what
 806     is in the app's source repo.
 807
 808     Where "packageName" is the app's packageName and "locale" is the locale
 809     of the graphics, e.g. what language they are in, using the IETF RFC5646
 810     format (en-US, fr-CA, es-MX, etc).
 811
 812     This will also scan the app's git for a fastlane folder, and the
 813     metadata/ folder and the apps' source repos for standard locations
 814     of graphic and screenshot files.  If it finds them, it will copy
 815     them into the repo.  The fastlane files follow this pattern:
 816     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 817
 818     """
 819
 820     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 821     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 822     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 823     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 824
 825     for srcd in sorted(sourcedirs):
 826         if not os.path.isdir(srcd):
 827             continue
 828         for root, dirs, files in os.walk(srcd):
 829             segments = root.split('/')
 830             packageName = segments[1]
 831             if packageName not in apps:
 832                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 833                 continue
 834             locale = segments[-1]
 835             destdir = os.path.join('repo', packageName, locale)
 836
 837             # flavours specified in build receipt
 838             build_flavours = ""
 839             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 840                     and 'gradle' in apps[packageName].builds[-1]:
 841                 build_flavours = apps[packageName].builds[-1].gradle
 842
 843             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 844                 logging.debug("ignoring due to wrong flavour")
 845                 continue
 846
 847             for f in files:
 848                 if f in ('description.txt', 'full_description.txt'):
 849                     _set_localized_text_entry(apps[packageName], locale, 'description',
 850                                               os.path.join(root, f))
 851                     continue
 852                 elif f in ('summary.txt', 'short_description.txt'):
 853                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 854                                               os.path.join(root, f))
 855                     continue
 856                 elif f in ('name.txt', 'title.txt'):
 857                     _set_localized_text_entry(apps[packageName], locale, 'name',
 858                                               os.path.join(root, f))
 859                     continue
 860                 elif f == 'video.txt':
 861                     _set_localized_text_entry(apps[packageName], locale, 'video',
 862                                               os.path.join(root, f))
 863                     continue
 864                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 865                     locale = segments[-2]
 866                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 867                                               os.path.join(root, f))
 868                     continue
 869
 870                 base, extension = common.get_extension(f)
 871                 if locale == 'images':
 872                     locale = segments[-2]
 873                     destdir = os.path.join('repo', packageName, locale)
 874                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 875                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 876                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 877                     _strip_and_copy_image(os.path.join(root, f), destdir)
 878             for d in dirs:
 879                 if d in SCREENSHOT_DIRS:
 880                     if locale == 'images':
 881                         locale = segments[-2]
 882                         destdir = os.path.join('repo', packageName, locale)
 883                     for f in glob.glob(os.path.join(root, d, '*.*')):
 884                         _ignored, extension = common.get_extension(f)
 885                         if extension in ALLOWED_EXTENSIONS:
 886                             screenshotdestdir = os.path.join(destdir, d)
 887                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 888                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 889                             _strip_and_copy_image(f, screenshotdestdir)
 890
 891     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 892     for d in repofiles:
 893         if not os.path.isdir(d):
 894             continue
 895         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 896             if not os.path.isfile(f):
 897                 continue
 898             segments = f.split('/')
 899             packageName = segments[1]
 900             locale = segments[2]
 901             screenshotdir = segments[3]
 902             filename = os.path.basename(f)
 903             base, extension = common.get_extension(filename)
 904
 905             if packageName not in apps:
 906                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 907                                 .format(path=filename, name=packageName))
 908                 continue
 909             graphics = _get_localized_dict(apps[packageName], locale)
 910
 911             if extension not in ALLOWED_EXTENSIONS:
 912                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 913             elif base in GRAPHIC_NAMES:
 914                 # there can only be zero or one of these per locale
 915                 graphics[base] = filename
 916             elif screenshotdir in SCREENSHOT_DIRS:
 917                 # there can any number of these per locale
 918                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 919                 if screenshotdir not in graphics:
 920                     graphics[screenshotdir] = []
 921                 graphics[screenshotdir].append(filename)
 922             else:
 923                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 924
 925
 926 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 927     """Scan a repo for all files with an extension except APK/OBB
 928
 929     :param apkcache: current cached info about all repo files
 930     :param repodir: repo directory to scan
 931     :param knownapks: list of all known files, as per metadata.read_metadata
 932     :param use_date_from_file: use date from file (instead of current date)
 933                                for newly added files
 934     """
 935
 936     cachechanged = False
 937     repo_files = []
 938     repodir = repodir.encode('utf-8')
 939     for name in os.listdir(repodir):
 940         file_extension = common.get_file_extension(name)
 941         if file_extension == 'apk' or file_extension == 'obb':
 942             continue
 943         filename = os.path.join(repodir, name)
 944         name_utf8 = name.decode('utf-8')
 945         if filename.endswith(b'_src.tar.gz'):
 946             logging.debug(_('skipping source tarball: {path}')
 947                           .format(path=filename.decode('utf-8')))
 948             continue
 949         if not common.is_repo_file(filename):
 950             continue
 951         stat = os.stat(filename)
 952         if stat.st_size == 0:
 953             raise FDroidException(_('{path} is zero size!')
 954                                   .format(path=filename))
 955
 956         shasum = sha256sum(filename)
 957         usecache = False
 958         if name in apkcache:
 959             repo_file = apkcache[name]
 960             # added time is cached as tuple but used here as datetime instance
 961             if 'added' in repo_file:
 962                 a = repo_file['added']
 963                 if isinstance(a, datetime):
 964                     repo_file['added'] = a
 965                 else:
 966                     repo_file['added'] = datetime(*a[:6])
 967             if repo_file.get('hash') == shasum:
 968                 logging.debug(_("Reading {apkfilename} from cache")
 969                               .format(apkfilename=name_utf8))
 970                 usecache = True
 971             else:
 972                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 973                               .format(apkfilename=name_utf8))
 974
 975         if not usecache:
 976             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 977             repo_file = collections.OrderedDict()
 978             repo_file['name'] = os.path.splitext(name_utf8)[0]
 979             # TODO rename apkname globally to something more generic
 980             repo_file['apkName'] = name_utf8
 981             repo_file['hash'] = shasum
 982             repo_file['hashType'] = 'sha256'
 983             repo_file['versionCode'] = 0
 984             repo_file['versionName'] = shasum
 985             # the static ID is the SHA256 unless it is set in the metadata
 986             repo_file['packageName'] = shasum
 987
 988             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 989             if m:
 990                 repo_file['packageName'] = m.group(1)
 991                 repo_file['versionCode'] = int(m.group(2))
 992             srcfilename = name + b'_src.tar.gz'
 993             if os.path.exists(os.path.join(repodir, srcfilename)):
 994                 repo_file['srcname'] = srcfilename.decode('utf-8')
 995             repo_file['size'] = stat.st_size
 996
 997             apkcache[name] = repo_file
 998             cachechanged = True
 999
1000         if use_date_from_file:
1001             timestamp = stat.st_ctime
1002             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1003         else:
1004             default_date_param = None
1005
1006         # Record in knownapks, getting the added date at the same time..
1007         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1008                                     default_date=default_date_param)
1009         if added:
1010             repo_file['added'] = added
1011
1012         repo_files.append(repo_file)
1013
1014     return repo_files, cachechanged
1015
1016
1017 def scan_apk(apk_file):
1018     """
1019     Scans an APK file and returns dictionary with metadata of the APK.
1020
1021     Attention: This does *not* verify that the APK signature is correct.
1022
1023     :param apk_file: The (ideally absolute) path to the APK file
1024     :raises BuildException
1025     :return A dict containing APK metadata
1026     """
1027     apk = {
1028         'hash': sha256sum(apk_file),
1029         'hashType': 'sha256',
1030         'uses-permission': [],
1031         'uses-permission-sdk-23': [],
1032         'features': [],
1033         'icons_src': {},
1034         'icons': {},
1035         'antiFeatures': set(),
1036     }
1037
1038     if SdkToolsPopen(['aapt', 'version'], output=False):
1039         scan_apk_aapt(apk, apk_file)
1040     else:
1041         scan_apk_androguard(apk, apk_file)
1042
1043     # Get the signature, or rather the signing key fingerprints
1044     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1045     apk['sig'] = getsig(apk_file)
1046     if not apk['sig']:
1047         raise BuildException("Failed to get apk signature")
1048     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1049                                                                apk_file))
1050     if not apk.get('signer'):
1051         raise BuildException("Failed to get apk signing key fingerprint")
1052
1053     # Get size of the APK
1054     apk['size'] = os.path.getsize(apk_file)
1055
1056     if 'minSdkVersion' not in apk:
1057         logging.warning("No SDK version information found in {0}".format(apk_file))
1058         apk['minSdkVersion'] = 1
1059
1060     # Check for known vulnerabilities
1061     if has_known_vulnerability(apk_file):
1062         apk['antiFeatures'].add('KnownVuln')
1063
1064     return apk
1065
1066
1067 def scan_apk_aapt(apk, apkfile):
1068     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1069     if p.returncode != 0:
1070         if options.delete_unknown:
1071             if os.path.exists(apkfile):
1072                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1073                 os.remove(apkfile)
1074             else:
1075                 logging.error("Could not find {0} to remove it".format(apkfile))
1076         else:
1077             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1078         raise BuildException(_("Invalid APK"))
1079     for line in p.output.splitlines():
1080         if line.startswith("package:"):
1081             try:
1082                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1083                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1084                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1085             except Exception as e:
1086                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1087         elif line.startswith("application:"):
1088             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1089             # Keep path to non-dpi icon in case we need it
1090             match = re.match(APK_ICON_PAT_NODPI, line)
1091             if match:
1092                 apk['icons_src']['-1'] = match.group(1)
1093         elif line.startswith("launchable-activity:"):
1094             # Only use launchable-activity as fallback to application
1095             if not apk['name']:
1096                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1097             if '-1' not in apk['icons_src']:
1098                 match = re.match(APK_ICON_PAT_NODPI, line)
1099                 if match:
1100                     apk['icons_src']['-1'] = match.group(1)
1101         elif line.startswith("application-icon-"):
1102             match = re.match(APK_ICON_PAT, line)
1103             if match:
1104                 density = match.group(1)
1105                 path = match.group(2)
1106                 apk['icons_src'][density] = path
1107         elif line.startswith("sdkVersion:"):
1108             m = re.match(APK_SDK_VERSION_PAT, line)
1109             if m is None:
1110                 logging.error(line.replace('sdkVersion:', '')
1111                               + ' is not a valid minSdkVersion!')
1112             else:
1113                 apk['minSdkVersion'] = m.group(1)
1114                 # if target not set, default to min
1115                 if 'targetSdkVersion' not in apk:
1116                     apk['targetSdkVersion'] = m.group(1)
1117         elif line.startswith("targetSdkVersion:"):
1118             m = re.match(APK_SDK_VERSION_PAT, line)
1119             if m is None:
1120                 logging.error(line.replace('targetSdkVersion:', '')
1121                               + ' is not a valid targetSdkVersion!')
1122             else:
1123                 apk['targetSdkVersion'] = m.group(1)
1124         elif line.startswith("maxSdkVersion:"):
1125             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1126         elif line.startswith("native-code:"):
1127             apk['nativecode'] = []
1128             for arch in line[13:].split(' '):
1129                 apk['nativecode'].append(arch[1:-1])
1130         elif line.startswith('uses-permission:'):
1131             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1132             if perm_match['maxSdkVersion']:
1133                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1134             permission = UsesPermission(
1135                 perm_match['name'],
1136                 perm_match['maxSdkVersion']
1137             )
1138
1139             apk['uses-permission'].append(permission)
1140         elif line.startswith('uses-permission-sdk-23:'):
1141             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1142             if perm_match['maxSdkVersion']:
1143                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1144             permission_sdk_23 = UsesPermissionSdk23(
1145                 perm_match['name'],
1146                 perm_match['maxSdkVersion']
1147             )
1148
1149             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1150
1151         elif line.startswith('uses-feature:'):
1152             feature = re.match(APK_FEATURE_PAT, line).group(1)
1153             # Filter out this, it's only added with the latest SDK tools and
1154             # causes problems for lots of apps.
1155             if feature != "android.hardware.screen.portrait" \
1156                     and feature != "android.hardware.screen.landscape":
1157                 if feature.startswith("android.feature."):
1158                     feature = feature[16:]
1159                 apk['features'].add(feature)
1160
1161
1162 def scan_apk_androguard(apk, apkfile):
1163     try:
1164         from androguard.core.bytecodes.apk import APK
1165         apkobject = APK(apkfile)
1166         if apkobject.is_valid_APK():
1167             arsc = apkobject.get_android_resources()
1168         else:
1169             if options.delete_unknown:
1170                 if os.path.exists(apkfile):
1171                     logging.error(_("Failed to get apk information, deleting {path}")
1172                                   .format(path=apkfile))
1173                     os.remove(apkfile)
1174                 else:
1175                     logging.error(_("Could not find {path} to remove it")
1176                                   .format(path=apkfile))
1177             else:
1178                 logging.error(_("Failed to get apk information, skipping {path}")
1179                               .format(path=apkfile))
1180             raise BuildException(_("Invalid APK"))
1181     except ImportError:
1182         raise FDroidException("androguard library is not installed and aapt not present")
1183     except FileNotFoundError:
1184         logging.error(_("Could not open apk file for analysis"))
1185         raise BuildException(_("Invalid APK"))
1186
1187     apk['packageName'] = apkobject.get_package()
1188     apk['versionCode'] = int(apkobject.get_androidversion_code())
1189     apk['versionName'] = apkobject.get_androidversion_name()
1190     if apk['versionName'][0] == "@":
1191         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1192         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1193         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1194     apk['name'] = apkobject.get_app_name()
1195
1196     if apkobject.get_max_sdk_version() is not None:
1197         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1198     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1199     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1200
1201     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1202     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1203
1204     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1205
1206     for file in apkobject.get_files():
1207         d_re = density_re.match(file)
1208         if d_re:
1209             folder = d_re.group(1).split('-')
1210             if len(folder) > 1:
1211                 resolution = folder[1]
1212             else:
1213                 resolution = 'mdpi'
1214             density = screen_resolutions[resolution]
1215             apk['icons_src'][density] = d_re.group(0)
1216
1217     if apk['icons_src'].get('-1') is None:
1218         apk['icons_src']['-1'] = apk['icons_src']['160']
1219
1220     arch_re = re.compile("^lib/(.*)/.*$")
1221     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1222     if len(arch) >= 1:
1223         apk['nativecode'] = []
1224         apk['nativecode'].extend(sorted(list(arch)))
1225
1226     xml = apkobject.get_android_manifest_xml()
1227
1228     for item in xml.getElementsByTagName('uses-permission'):
1229         name = str(item.getAttribute("android:name"))
1230         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1231         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1232         permission = UsesPermission(
1233             name,
1234             maxSdkVersion
1235         )
1236         apk['uses-permission'].append(permission)
1237
1238     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1239         name = str(item.getAttribute("android:name"))
1240         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1241         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1242         permission_sdk_23 = UsesPermissionSdk23(
1243             name,
1244             maxSdkVersion
1245         )
1246         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1247
1248     for item in xml.getElementsByTagName('uses-feature'):
1249         feature = str(item.getAttribute("android:name"))
1250         if feature != "android.hardware.screen.portrait" \
1251                 and feature != "android.hardware.screen.landscape":
1252             if feature.startswith("android.feature."):
1253                 feature = feature[16:]
1254         apk['features'].append(feature)
1255
1256
1257 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1258                 allow_disabled_algorithms=False, archive_bad_sig=False):
1259     """Processes the apk with the given filename in the given repo directory.
1260
1261     This also extracts the icons.
1262
1263     :param apkcache: current apk cache information
1264     :param apkfilename: the filename of the apk to scan
1265     :param repodir: repo directory to scan
1266     :param knownapks: known apks info
1267     :param use_date_from_apk: use date from APK (instead of current date)
1268                               for newly added APKs
1269     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1270                                       disabled algorithms in the signature (e.g. MD5)
1271     :param archive_bad_sig: move APKs with a bad signature to the archive
1272     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1273      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1274     """
1275
1276     apk = {}
1277     apkfile = os.path.join(repodir, apkfilename)
1278
1279     cachechanged = False
1280     usecache = False
1281     if apkfilename in apkcache:
1282         apk = apkcache[apkfilename]
1283         if apk.get('hash') == sha256sum(apkfile):
1284             logging.debug(_("Reading {apkfilename} from cache")
1285                           .format(apkfilename=apkfilename))
1286             usecache = True
1287         else:
1288             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1289                           .format(apkfilename=apkfilename))
1290
1291     if not usecache:
1292         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1293
1294         try:
1295             apk = scan_apk(apkfile)
1296         except BuildException:
1297             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1298                             .format(apkfilename=apkfilename))
1299             return True, None, False
1300
1301         # Check for debuggable apks...
1302         if common.isApkAndDebuggable(apkfile):
1303             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1304
1305         if options.rename_apks:
1306             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1307             std_short_name = os.path.join(repodir, n)
1308             if apkfile != std_short_name:
1309                 if os.path.exists(std_short_name):
1310                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1311                     if apkfile != std_long_name:
1312                         if os.path.exists(std_long_name):
1313                             dupdir = os.path.join('duplicates', repodir)
1314                             if not os.path.isdir(dupdir):
1315                                 os.makedirs(dupdir, exist_ok=True)
1316                             dupfile = os.path.join('duplicates', std_long_name)
1317                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1318                             os.rename(apkfile, dupfile)
1319                             return True, None, False
1320                         else:
1321                             os.rename(apkfile, std_long_name)
1322                     apkfile = std_long_name
1323                 else:
1324                     os.rename(apkfile, std_short_name)
1325                     apkfile = std_short_name
1326                 apkfilename = apkfile[len(repodir) + 1:]
1327
1328         apk['apkName'] = apkfilename
1329         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1330         if os.path.exists(os.path.join(repodir, srcfilename)):
1331             apk['srcname'] = srcfilename
1332
1333         # verify the jar signature is correct, allow deprecated
1334         # algorithms only if the APK is in the archive.
1335         skipapk = False
1336         if not common.verify_apk_signature(apkfile):
1337             if repodir == 'archive' or allow_disabled_algorithms:
1338                 if common.verify_old_apk_signature(apkfile):
1339                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1340                 else:
1341                     skipapk = True
1342             else:
1343                 skipapk = True
1344
1345         if skipapk:
1346             if archive_bad_sig:
1347                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1348                                 .format(apkfilename=apkfilename))
1349                 move_apk_between_sections(repodir, 'archive', apk)
1350             else:
1351                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1352                                 .format(apkfilename=apkfilename))
1353             return True, None, False
1354
1355         apkzip = zipfile.ZipFile(apkfile, 'r')
1356
1357         manifest = apkzip.getinfo('AndroidManifest.xml')
1358         if manifest.date_time[1] == 0:  # month can't be zero
1359             logging.debug(_('AndroidManifest.xml has no date'))
1360         else:
1361             common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1362
1363         # extract icons from APK zip file
1364         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1365         try:
1366             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1367         finally:
1368             apkzip.close()  # ensure that APK zip file gets closed
1369
1370         # resize existing icons for densities missing in the APK
1371         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1372
1373         if use_date_from_apk and manifest.date_time[1] != 0:
1374             default_date_param = datetime(*manifest.date_time)
1375         else:
1376             default_date_param = None
1377
1378         # Record in known apks, getting the added date at the same time..
1379         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1380                                     default_date=default_date_param)
1381         if added:
1382             apk['added'] = added
1383
1384         apkcache[apkfilename] = apk
1385         cachechanged = True
1386
1387     return False, apk, cachechanged
1388
1389
1390 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1391     """Processes the apks in the given repo directory.
1392
1393     This also extracts the icons.
1394
1395     :param apkcache: current apk cache information
1396     :param repodir: repo directory to scan
1397     :param knownapks: known apks info
1398     :param use_date_from_apk: use date from APK (instead of current date)
1399                               for newly added APKs
1400     :returns: (apks, cachechanged) where apks is a list of apk information,
1401               and cachechanged is True if the apkcache got changed.
1402     """
1403
1404     cachechanged = False
1405
1406     for icon_dir in get_all_icon_dirs(repodir):
1407         if os.path.exists(icon_dir):
1408             if options.clean:
1409                 shutil.rmtree(icon_dir)
1410                 os.makedirs(icon_dir)
1411         else:
1412             os.makedirs(icon_dir)
1413
1414     apks = []
1415     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1416         apkfilename = apkfile[len(repodir) + 1:]
1417         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1418         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1419                                              use_date_from_apk, ada, True)
1420         if skip:
1421             continue
1422         apks.append(apk)
1423         cachechanged = cachechanged or cachethis
1424
1425     return apks, cachechanged
1426
1427
1428 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1429     """
1430     Extracts icons from the given APK zip in various densities,
1431     saves them into given repo directory
1432     and stores their names in the APK metadata dictionary.
1433
1434     :param icon_filename: A string representing the icon's file name
1435     :param apk: A populated dictionary containing APK metadata.
1436                 Needs to have 'icons_src' key
1437     :param apkzip: An opened zipfile.ZipFile of the APK file
1438     :param repo_dir: The directory of the APK's repository
1439     :return: A list of icon densities that are missing
1440     """
1441     empty_densities = []
1442     for density in screen_densities:
1443         if density not in apk['icons_src']:
1444             empty_densities.append(density)
1445             continue
1446         icon_src = apk['icons_src'][density]
1447         icon_dir = get_icon_dir(repo_dir, density)
1448         icon_dest = os.path.join(icon_dir, icon_filename)
1449
1450         # Extract the icon files per density
1451         if icon_src.endswith('.xml'):
1452             png = os.path.basename(icon_src)[:-4] + '.png'
1453             for f in apkzip.namelist():
1454                 if f.endswith(png):
1455                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1456                     if m and screen_resolutions[m.group(2)] == density:
1457                         icon_src = f
1458             if icon_src.endswith('.xml'):
1459                 empty_densities.append(density)
1460                 continue
1461         try:
1462             with open(icon_dest, 'wb') as f:
1463                 f.write(get_icon_bytes(apkzip, icon_src))
1464             apk['icons'][density] = icon_filename
1465         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1466             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1467             del apk['icons_src'][density]
1468             empty_densities.append(density)
1469
1470     if '-1' in apk['icons_src']:
1471         icon_src = apk['icons_src']['-1']
1472         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1473         with open(icon_path, 'wb') as f:
1474             f.write(get_icon_bytes(apkzip, icon_src))
1475         try:
1476             im = Image.open(icon_path)
1477             dpi = px_to_dpi(im.size[0])
1478             for density in screen_densities:
1479                 if density in apk['icons']:
1480                     break
1481                 if density == screen_densities[-1] or dpi >= int(density):
1482                     apk['icons'][density] = icon_filename
1483                     shutil.move(icon_path,
1484                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1485                     empty_densities.remove(density)
1486                     break
1487         except Exception as e:
1488             logging.warning(_("Failed reading {path}: {error}")
1489                             .format(path=icon_path, error=e))
1490         finally:
1491             im.close()
1492
1493     if apk['icons']:
1494         apk['icon'] = icon_filename
1495
1496     return empty_densities
1497
1498
1499 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1500     """
1501     Resize existing icons for densities missing in the APK to ensure all densities are available
1502
1503     :param empty_densities: A list of icon densities that are missing
1504     :param icon_filename: A string representing the icon's file name
1505     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1506     :param repo_dir: The directory of the APK's repository
1507     """
1508     # First try resizing down to not lose quality
1509     last_density = None
1510     for density in screen_densities:
1511         if density not in empty_densities:
1512             last_density = density
1513             continue
1514         if last_density is None:
1515             continue
1516         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1517
1518         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1519         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1520         fp = None
1521         try:
1522             fp = open(last_icon_path, 'rb')
1523             im = Image.open(fp)
1524
1525             size = dpi_to_px(density)
1526
1527             im.thumbnail((size, size), Image.ANTIALIAS)
1528             im.save(icon_path, "PNG", optimize=True,
1529                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1530             empty_densities.remove(density)
1531         except Exception as e:
1532             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1533         finally:
1534             if fp:
1535                 fp.close()
1536
1537     # Then just copy from the highest resolution available
1538     last_density = None
1539     for density in reversed(screen_densities):
1540         if density not in empty_densities:
1541             last_density = density
1542             continue
1543
1544         if last_density is None:
1545             continue
1546
1547         shutil.copyfile(
1548             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1549             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1550         )
1551         empty_densities.remove(density)
1552
1553     for density in screen_densities:
1554         icon_dir = get_icon_dir(repo_dir, density)
1555         icon_dest = os.path.join(icon_dir, icon_filename)
1556         resize_icon(icon_dest, density)
1557
1558     # Copy from icons-mdpi to icons since mdpi is the baseline density
1559     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1560     if os.path.isfile(baseline):
1561         apk['icons']['0'] = icon_filename
1562         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1563
1564
1565 def apply_info_from_latest_apk(apps, apks):
1566     """
1567     Some information from the apks needs to be applied up to the application level.
1568     When doing this, we use the info from the most recent version's apk.
1569     We deal with figuring out when the app was added and last updated at the same time.
1570     """
1571     for appid, app in apps.items():
1572         bestver = UNSET_VERSION_CODE
1573         for apk in apks:
1574             if apk['packageName'] == appid:
1575                 if apk['versionCode'] > bestver:
1576                     bestver = apk['versionCode']
1577                     bestapk = apk
1578
1579                 if 'added' in apk:
1580                     if not app.added or apk['added'] < app.added:
1581                         app.added = apk['added']
1582                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1583                         app.lastUpdated = apk['added']
1584
1585         if not app.added:
1586             logging.debug("Don't know when " + appid + " was added")
1587         if not app.lastUpdated:
1588             logging.debug("Don't know when " + appid + " was last updated")
1589
1590         if bestver == UNSET_VERSION_CODE:
1591
1592             if app.Name is None:
1593                 app.Name = app.AutoName or appid
1594             app.icon = None
1595             logging.debug("Application " + appid + " has no packages")
1596         else:
1597             if app.Name is None:
1598                 app.Name = bestapk['name']
1599             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1600             if app.CurrentVersionCode is None:
1601                 app.CurrentVersionCode = str(bestver)
1602
1603
1604 def make_categories_txt(repodir, categories):
1605     '''Write a category list in the repo to allow quick access'''
1606     catdata = ''
1607     for cat in sorted(categories):
1608         catdata += cat + '\n'
1609     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1610         f.write(catdata)
1611
1612
1613 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1614
1615     def filter_apk_list_sorted(apk_list):
1616         res = []
1617         for apk in apk_list:
1618             if apk['packageName'] == appid:
1619                 res.append(apk)
1620
1621         # Sort the apk list by version code. First is highest/newest.
1622         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1623
1624     for appid, app in apps.items():
1625
1626         if app.ArchivePolicy:
1627             keepversions = int(app.ArchivePolicy[:-9])
1628         else:
1629             keepversions = defaultkeepversions
1630
1631         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1632                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1633
1634         current_app_apks = filter_apk_list_sorted(apks)
1635         if len(current_app_apks) > keepversions:
1636             # Move back the ones we don't want.
1637             for apk in current_app_apks[keepversions:]:
1638                 move_apk_between_sections(repodir, archivedir, apk)
1639                 archapks.append(apk)
1640                 apks.remove(apk)
1641
1642         current_app_archapks = filter_apk_list_sorted(archapks)
1643         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1644             kept = 0
1645             # Move forward the ones we want again, except DisableAlgorithm
1646             for apk in current_app_archapks:
1647                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1648                     move_apk_between_sections(archivedir, repodir, apk)
1649                     archapks.remove(apk)
1650                     apks.append(apk)
1651                     kept += 1
1652                 if kept == keepversions:
1653                     break
1654
1655
1656 def move_apk_between_sections(from_dir, to_dir, apk):
1657     """move an APK from repo to archive or vice versa"""
1658
1659     def _move_file(from_dir, to_dir, filename, ignore_missing):
1660         from_path = os.path.join(from_dir, filename)
1661         if ignore_missing and not os.path.exists(from_path):
1662             return
1663         to_path = os.path.join(to_dir, filename)
1664         if not os.path.exists(to_dir):
1665             os.mkdir(to_dir)
1666         shutil.move(from_path, to_path)
1667
1668     if from_dir == to_dir:
1669         return
1670
1671     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1672     _move_file(from_dir, to_dir, apk['apkName'], False)
1673     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1674     for density in all_screen_densities:
1675         from_icon_dir = get_icon_dir(from_dir, density)
1676         to_icon_dir = get_icon_dir(to_dir, density)
1677         if density not in apk.get('icons', []):
1678             continue
1679         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1680     if 'srcname' in apk:
1681         _move_file(from_dir, to_dir, apk['srcname'], False)
1682
1683
1684 def add_apks_to_per_app_repos(repodir, apks):
1685     apks_per_app = dict()
1686     for apk in apks:
1687         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1688         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1689         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1690         apks_per_app[apk['packageName']] = apk
1691
1692         if not os.path.exists(apk['per_app_icons']):
1693             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1694             os.makedirs(apk['per_app_icons'])
1695
1696         apkpath = os.path.join(repodir, apk['apkName'])
1697         shutil.copy(apkpath, apk['per_app_repo'])
1698         apksigpath = apkpath + '.sig'
1699         if os.path.exists(apksigpath):
1700             shutil.copy(apksigpath, apk['per_app_repo'])
1701         apkascpath = apkpath + '.asc'
1702         if os.path.exists(apkascpath):
1703             shutil.copy(apkascpath, apk['per_app_repo'])
1704
1705
1706 def create_metadata_from_template(apk):
1707     '''create a new metadata file using internal or external template
1708
1709     Generate warnings for apk's with no metadata (or create skeleton
1710     metadata files, if requested on the command line).  Though the
1711     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1712     since those impose things on the metadata file made from the
1713     template: field sort order, empty field value, formatting, etc.
1714     '''
1715
1716     import yaml
1717     if os.path.exists('template.yml'):
1718         with open('template.yml') as f:
1719             metatxt = f.read()
1720         if 'name' in apk and apk['name'] != '':
1721             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1722                              r'\1 ' + apk['name'],
1723                              metatxt,
1724                              flags=re.IGNORECASE | re.MULTILINE)
1725         else:
1726             logging.warning(_('{appid} does not have a name! Using package name instead.')
1727                             .format(appid=apk['packageName']))
1728             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1729                              r'\1 ' + apk['packageName'],
1730                              metatxt,
1731                              flags=re.IGNORECASE | re.MULTILINE)
1732         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1733             f.write(metatxt)
1734     else:
1735         app = dict()
1736         app['Categories'] = [os.path.basename(os.getcwd())]
1737         # include some blanks as part of the template
1738         app['AuthorName'] = ''
1739         app['Summary'] = ''
1740         app['WebSite'] = ''
1741         app['IssueTracker'] = ''
1742         app['SourceCode'] = ''
1743         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1744         if 'name' in apk and apk['name'] != '':
1745             app['Name'] = apk['name']
1746         else:
1747             logging.warning(_('{appid} does not have a name! Using package name instead.')
1748                             .format(appid=apk['packageName']))
1749             app['Name'] = apk['packageName']
1750         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1751             yaml.dump(app, f, default_flow_style=False)
1752     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1753
1754
1755 config = None
1756 options = None
1757
1758
1759 def main():
1760
1761     global config, options
1762
1763     # Parse command line...
1764     parser = ArgumentParser()
1765     common.setup_global_opts(parser)
1766     parser.add_argument("--create-key", action="store_true", default=False,
1767                         help=_("Add a repo signing key to an unsigned repo"))
1768     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1769                         help=_("Add skeleton metadata files for APKs that are missing them"))
1770     parser.add_argument("--delete-unknown", action="store_true", default=False,
1771                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1772     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1773                         help=_("Report on build data status"))
1774     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1775                         help=_("Interactively ask about things that need updating."))
1776     parser.add_argument("-I", "--icons", action="store_true", default=False,
1777                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1778     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1779                         help=_("Specify editor to use in interactive mode. Default " +
1780                                "is {path}").format(path='/etc/alternatives/editor'))
1781     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1782                         help=_("Update the wiki"))
1783     parser.add_argument("--pretty", action="store_true", default=False,
1784                         help=_("Produce human-readable XML/JSON for index files"))
1785     parser.add_argument("--clean", action="store_true", default=False,
1786                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1787     parser.add_argument("--nosign", action="store_true", default=False,
1788                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1789     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1790                         help=_("Use date from APK instead of current time for newly added APKs"))
1791     parser.add_argument("--rename-apks", action="store_true", default=False,
1792                         help=_("Rename APK files that do not match package.name_123.apk"))
1793     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1794                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1795     metadata.add_metadata_arguments(parser)
1796     options = parser.parse_args()
1797     metadata.warnings_action = options.W
1798
1799     config = common.read_config(options)
1800
1801     if not ('jarsigner' in config and 'keytool' in config):
1802         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1803
1804     repodirs = ['repo']
1805     if config['archive_older'] != 0:
1806         repodirs.append('archive')
1807         if not os.path.exists('archive'):
1808             os.mkdir('archive')
1809
1810     if options.icons:
1811         resize_all_icons(repodirs)
1812         sys.exit(0)
1813
1814     if options.rename_apks:
1815         options.clean = True
1816
1817     # check that icons exist now, rather than fail at the end of `fdroid update`
1818     for k in ['repo_icon', 'archive_icon']:
1819         if k in config:
1820             if not os.path.exists(config[k]):
1821                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1822                                  .format(name=k, path=config[k]))
1823                 sys.exit(1)
1824
1825     # if the user asks to create a keystore, do it now, reusing whatever it can
1826     if options.create_key:
1827         if os.path.exists(config['keystore']):
1828             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1829             logging.critical("\t'" + config['keystore'] + "'")
1830             sys.exit(1)
1831
1832         if 'repo_keyalias' not in config:
1833             config['repo_keyalias'] = socket.getfqdn()
1834             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1835         if 'keydname' not in config:
1836             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1837             common.write_to_config(config, 'keydname', config['keydname'])
1838         if 'keystore' not in config:
1839             config['keystore'] = common.default_config['keystore']
1840             common.write_to_config(config, 'keystore', config['keystore'])
1841
1842         password = common.genpassword()
1843         if 'keystorepass' not in config:
1844             config['keystorepass'] = password
1845             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1846         if 'keypass' not in config:
1847             config['keypass'] = password
1848             common.write_to_config(config, 'keypass', config['keypass'])
1849         common.genkeystore(config)
1850
1851     # Get all apps...
1852     apps = metadata.read_metadata()
1853
1854     # Generate a list of categories...
1855     categories = set()
1856     for app in apps.values():
1857         categories.update(app.Categories)
1858
1859     # Read known apks data (will be updated and written back when we've finished)
1860     knownapks = common.KnownApks()
1861
1862     # Get APK cache
1863     apkcache = get_cache()
1864
1865     # Delete builds for disabled apps
1866     delete_disabled_builds(apps, apkcache, repodirs)
1867
1868     # Scan all apks in the main repo
1869     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1870
1871     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1872                                            options.use_date_from_apk)
1873     cachechanged = cachechanged or fcachechanged
1874     apks += files
1875     for apk in apks:
1876         if apk['packageName'] not in apps:
1877             if options.create_metadata:
1878                 create_metadata_from_template(apk)
1879                 apps = metadata.read_metadata()
1880             else:
1881                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1882                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1883                 if options.delete_unknown:
1884                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1885                                  .format(apkfilename=apk['apkName']))
1886                     rmf = os.path.join(repodirs[0], apk['apkName'])
1887                     if not os.path.exists(rmf):
1888                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1889                     else:
1890                         os.remove(rmf)
1891                 else:
1892                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1893
1894     copy_triple_t_store_metadata(apps)
1895     insert_obbs(repodirs[0], apps, apks)
1896     insert_localized_app_metadata(apps)
1897     translate_per_build_anti_features(apps, apks)
1898
1899     # Scan the archive repo for apks as well
1900     if len(repodirs) > 1:
1901         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1902         if cc:
1903             cachechanged = True
1904     else:
1905         archapks = []
1906
1907     # Apply information from latest apks to the application and update dates
1908     apply_info_from_latest_apk(apps, apks + archapks)
1909
1910     # Sort the app list by name, then the web site doesn't have to by default.
1911     # (we had to wait until we'd scanned the apks to do this, because mostly the
1912     # name comes from there!)
1913     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1914
1915     # APKs are placed into multiple repos based on the app package, providing
1916     # per-app subscription feeds for nightly builds and things like it
1917     if config['per_app_repos']:
1918         add_apks_to_per_app_repos(repodirs[0], apks)
1919         for appid, app in apps.items():
1920             repodir = os.path.join(appid, 'fdroid', 'repo')
1921             appdict = dict()
1922             appdict[appid] = app
1923             if os.path.isdir(repodir):
1924                 index.make(appdict, [appid], apks, repodir, False)
1925             else:
1926                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1927         return
1928
1929     if len(repodirs) > 1:
1930         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1931
1932     # Make the index for the main repo...
1933     index.make(apps, sortedids, apks, repodirs[0], False)
1934     make_categories_txt(repodirs[0], categories)
1935
1936     # If there's an archive repo,  make the index for it. We already scanned it
1937     # earlier on.
1938     if len(repodirs) > 1:
1939         index.make(apps, sortedids, archapks, repodirs[1], True)
1940
1941     git_remote = config.get('binary_transparency_remote')
1942     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1943         from . import btlog
1944         btlog.make_binary_transparency_log(repodirs)
1945
1946     if config['update_stats']:
1947         # Update known apks info...
1948         knownapks.writeifchanged()
1949
1950         # Generate latest apps data for widget
1951         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1952             data = ''
1953             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1954                 for line in f:
1955                     appid = line.rstrip()
1956                     data += appid + "\t"
1957                     app = apps[appid]
1958                     data += app.Name + "\t"
1959                     if app.icon is not None:
1960                         data += app.icon + "\t"
1961                     data += app.License + "\n"
1962             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1963                 f.write(data)
1964
1965     if cachechanged:
1966         write_cache(apkcache)
1967
1968     # Update the wiki...
1969     if options.wiki:
1970         update_wiki(apps, sortedids, apks + archapks)
1971
1972     logging.info(_("Finished"))
1973
1974
1975 if __name__ == "__main__":
1976     main()