fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0':
 100         return os.path.join(repodir, "icons")
 101     return os.path.join(repodir, "icons-%s" % density)
 102
 103
 104 def get_icon_dirs(repodir):
 105     for density in screen_densities:
 106         yield get_icon_dir(repodir, density)
 107
 108
 109 def get_all_icon_dirs(repodir):
 110     for density in all_screen_densities:
 111         yield get_icon_dir(repodir, density)
 112
 113
 114 def update_wiki(apps, sortedids, apks):
 115     """Update the wiki
 116
 117     :param apps: fully populated list of all applications
 118     :param apks: all apks, except...
 119     """
 120     logging.info("Updating wiki")
 121     wikicat = 'Apps'
 122     wikiredircat = 'App Redirects'
 123     import mwclient
 124     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 125                          path=config['wiki_path'])
 126     site.login(config['wiki_user'], config['wiki_password'])
 127     generated_pages = {}
 128     generated_redirects = {}
 129
 130     for appid in sortedids:
 131         app = metadata.App(apps[appid])
 132
 133         wikidata = ''
 134         if app.Disabled:
 135             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 136         if app.AntiFeatures:
 137             for af in sorted(app.AntiFeatures):
 138                 wikidata += '{{AntiFeature|' + af + '}}\n'
 139         if app.RequiresRoot:
 140             requiresroot = 'Yes'
 141         else:
 142             requiresroot = 'No'
 143         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 144             appid,
 145             app.Name,
 146             app.added.strftime('%Y-%m-%d') if app.added else '',
 147             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 148             app.SourceCode,
 149             app.IssueTracker,
 150             app.WebSite,
 151             app.Changelog,
 152             app.Donate,
 153             app.FlattrID,
 154             app.LiberapayID,
 155             app.Bitcoin,
 156             app.Litecoin,
 157             app.License,
 158             requiresroot,
 159             app.AuthorName,
 160             app.AuthorEmail)
 161
 162         if app.Provides:
 163             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 164
 165         wikidata += app.Summary
 166         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 167
 168         wikidata += "=Description=\n"
 169         wikidata += metadata.description_wiki(app.Description) + "\n"
 170
 171         wikidata += "=Maintainer Notes=\n"
 172         if app.MaintainerNotes:
 173             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 174         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 175
 176         # Get a list of all packages for this application...
 177         apklist = []
 178         gotcurrentver = False
 179         cantupdate = False
 180         buildfails = False
 181         for apk in apks:
 182             if apk['packageName'] == appid:
 183                 if str(apk['versionCode']) == app.CurrentVersionCode:
 184                     gotcurrentver = True
 185                 apklist.append(apk)
 186         # Include ones we can't build, as a special case...
 187         for build in app.builds:
 188             if build.disable:
 189                 if build.versionCode == app.CurrentVersionCode:
 190                     cantupdate = True
 191                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 192                 apklist.append({'versionCode': int(build.versionCode),
 193                                 'versionName': build.versionName,
 194                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 195                                 })
 196             else:
 197                 builtit = False
 198                 for apk in apklist:
 199                     if apk['versionCode'] == int(build.versionCode):
 200                         builtit = True
 201                         break
 202                 if not builtit:
 203                     buildfails = True
 204                     apklist.append({'versionCode': int(build.versionCode),
 205                                     'versionName': build.versionName,
 206                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 207                                     })
 208         if app.CurrentVersionCode == '0':
 209             cantupdate = True
 210         # Sort with most recent first...
 211         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 212
 213         wikidata += "=Versions=\n"
 214         if len(apklist) == 0:
 215             wikidata += "We currently have no versions of this app available."
 216         elif not gotcurrentver:
 217             wikidata += "We don't have the current version of this app."
 218         else:
 219             wikidata += "We have the current version of this app."
 220         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 221         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 222         if len(app.NoSourceSince) > 0:
 223             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 224         if len(app.CurrentVersion) > 0:
 225             wikidata += "The current (recommended) version is " + app.CurrentVersion
 226             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 227         validapks = 0
 228         for apk in apklist:
 229             wikidata += "==" + apk['versionName'] + "==\n"
 230
 231             if 'buildproblem' in apk:
 232                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 233             else:
 234                 validapks += 1
 235                 wikidata += "This version is built and signed by "
 236                 if 'srcname' in apk:
 237                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 238                 else:
 239                     wikidata += "the original developer.\n\n"
 240             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 241
 242         wikidata += '\n[[Category:' + wikicat + ']]\n'
 243         if len(app.NoSourceSince) > 0:
 244             wikidata += '\n[[Category:Apps missing source code]]\n'
 245         if validapks == 0 and not app.Disabled:
 246             wikidata += '\n[[Category:Apps with no packages]]\n'
 247         if cantupdate and not app.Disabled:
 248             wikidata += "\n[[Category:Apps we cannot update]]\n"
 249         if buildfails and not app.Disabled:
 250             wikidata += "\n[[Category:Apps with failing builds]]\n"
 251         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 252             wikidata += '\n[[Category:Apps to Update]]\n'
 253         if app.Disabled:
 254             wikidata += '\n[[Category:Apps that are disabled]]\n'
 255         if app.UpdateCheckMode == 'None' and not app.Disabled:
 256             wikidata += '\n[[Category:Apps with no update check]]\n'
 257         for appcat in app.Categories:
 258             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 259
 260         # We can't have underscores in the page name, even if they're in
 261         # the package ID, because MediaWiki messes with them...
 262         pagename = appid.replace('_', ' ')
 263
 264         # Drop a trailing newline, because mediawiki is going to drop it anyway
 265         # and it we don't we'll think the page has changed when it hasn't...
 266         if wikidata.endswith('\n'):
 267             wikidata = wikidata[:-1]
 268
 269         generated_pages[pagename] = wikidata
 270
 271         # Make a redirect from the name to the ID too, unless there's
 272         # already an existing page with the name and it isn't a redirect.
 273         noclobber = False
 274         apppagename = app.Name.replace('_', ' ')
 275         apppagename = apppagename.replace('{', '')
 276         apppagename = apppagename.replace('}', ' ')
 277         apppagename = apppagename.replace(':', ' ')
 278         apppagename = apppagename.replace('[', ' ')
 279         apppagename = apppagename.replace(']', ' ')
 280         # Drop double spaces caused mostly by replacing ':' above
 281         apppagename = apppagename.replace('  ', ' ')
 282         for expagename in site.allpages(prefix=apppagename,
 283                                         filterredir='nonredirects',
 284                                         generator=False):
 285             if expagename == apppagename:
 286                 noclobber = True
 287         # Another reason not to make the redirect page is if the app name
 288         # is the same as it's ID, because that will overwrite the real page
 289         # with an redirect to itself! (Although it seems like an odd
 290         # scenario this happens a lot, e.g. where there is metadata but no
 291         # builds or binaries to extract a name from.
 292         if apppagename == pagename:
 293             noclobber = True
 294         if not noclobber:
 295             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 296
 297     for tcat, genp in [(wikicat, generated_pages),
 298                        (wikiredircat, generated_redirects)]:
 299         catpages = site.Pages['Category:' + tcat]
 300         existingpages = []
 301         for page in catpages:
 302             existingpages.append(page.name)
 303             if page.name in genp:
 304                 pagetxt = page.edit()
 305                 if pagetxt != genp[page.name]:
 306                     logging.debug("Updating modified page " + page.name)
 307                     page.save(genp[page.name], summary='Auto-updated')
 308                 else:
 309                     logging.debug("Page " + page.name + " is unchanged")
 310             else:
 311                 logging.warn("Deleting page " + page.name)
 312                 page.delete('No longer published')
 313         for pagename, text in genp.items():
 314             logging.debug("Checking " + pagename)
 315             if pagename not in existingpages:
 316                 logging.debug("Creating page " + pagename)
 317                 try:
 318                     newpage = site.Pages[pagename]
 319                     newpage.save(text, summary='Auto-created')
 320                 except Exception as e:
 321                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 322
 323     # Purge server cache to ensure counts are up to date
 324     site.pages['Repository Maintenance'].purge()
 325
 326
 327 def delete_disabled_builds(apps, apkcache, repodirs):
 328     """Delete disabled build outputs.
 329
 330     :param apps: list of all applications, as per metadata.read_metadata
 331     :param apkcache: current apk cache information
 332     :param repodirs: the repo directories to process
 333     """
 334     for appid, app in apps.items():
 335         for build in app['builds']:
 336             if not build.disable:
 337                 continue
 338             apkfilename = common.get_release_filename(app, build)
 339             iconfilename = "%s.%s.png" % (
 340                 appid,
 341                 build.versionCode)
 342             for repodir in repodirs:
 343                 files = [
 344                     os.path.join(repodir, apkfilename),
 345                     os.path.join(repodir, apkfilename + '.asc'),
 346                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 347                 ]
 348                 for density in all_screen_densities:
 349                     repo_dir = get_icon_dir(repodir, density)
 350                     files.append(os.path.join(repo_dir, iconfilename))
 351
 352                 for f in files:
 353                     if os.path.exists(f):
 354                         logging.info("Deleting disabled build output " + f)
 355                         os.remove(f)
 356             if apkfilename in apkcache:
 357                 del apkcache[apkfilename]
 358
 359
 360 def resize_icon(iconpath, density):
 361
 362     if not os.path.isfile(iconpath):
 363         return
 364
 365     fp = None
 366     try:
 367         fp = open(iconpath, 'rb')
 368         im = Image.open(fp)
 369         size = dpi_to_px(density)
 370
 371         if any(length > size for length in im.size):
 372             oldsize = im.size
 373             im.thumbnail((size, size), Image.ANTIALIAS)
 374             logging.debug("%s was too large at %s - new size is %s" % (
 375                 iconpath, oldsize, im.size))
 376             im.save(iconpath, "PNG", optimize=True,
 377                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 378
 379     except Exception as e:
 380         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 381
 382     finally:
 383         if fp:
 384             fp.close()
 385
 386
 387 def resize_all_icons(repodirs):
 388     """Resize all icons that exceed the max size
 389
 390     :param repodirs: the repo directories to process
 391     """
 392     for repodir in repodirs:
 393         for density in screen_densities:
 394             icon_dir = get_icon_dir(repodir, density)
 395             icon_glob = os.path.join(icon_dir, '*.png')
 396             for iconpath in glob.glob(icon_glob):
 397                 resize_icon(iconpath, density)
 398
 399
 400 def getsig(apkpath):
 401     """ Get the signing certificate of an apk. To get the same md5 has that
 402     Android gets, we encode the .RSA certificate in a specific format and pass
 403     it hex-encoded to the md5 digest algorithm.
 404
 405     :param apkpath: path to the apk
 406     :returns: A string containing the md5 of the signature of the apk or None
 407               if an error occurred.
 408     """
 409
 410     with zipfile.ZipFile(apkpath, 'r') as apk:
 411         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 412
 413         if len(certs) < 1:
 414             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 415             return None
 416         if len(certs) > 1:
 417             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 418             return None
 419
 420         cert = apk.read(certs[0])
 421
 422     cert_encoded = common.get_certificate(cert)
 423
 424     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 425
 426
 427 def get_cache_file():
 428     return os.path.join('tmp', 'apkcache')
 429
 430
 431 def get_cache():
 432     """Get the cached dict of the APK index
 433
 434     Gather information about all the apk files in the repo directory,
 435     using cached data if possible. Some of the index operations take a
 436     long time, like calculating the SHA-256 and verifying the APK
 437     signature.
 438
 439     The cache is invalidated if the metadata version is different, or
 440     the 'allow_disabled_algorithms' config/option is different.  In
 441     those cases, there is no easy way to know what has changed from
 442     the cache, so just rerun the whole thing.
 443
 444     :return: apkcache
 445
 446     """
 447     apkcachefile = get_cache_file()
 448     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 449     if not options.clean and os.path.exists(apkcachefile):
 450         with open(apkcachefile, 'rb') as cf:
 451             apkcache = pickle.load(cf, encoding='utf-8')
 452         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 453            or apkcache.get('allow_disabled_algorithms') != ada:
 454             apkcache = {}
 455     else:
 456         apkcache = {}
 457
 458     apkcache["METADATA_VERSION"] = METADATA_VERSION
 459     apkcache['allow_disabled_algorithms'] = ada
 460
 461     return apkcache
 462
 463
 464 def write_cache(apkcache):
 465     apkcachefile = get_cache_file()
 466     cache_path = os.path.dirname(apkcachefile)
 467     if not os.path.exists(cache_path):
 468         os.makedirs(cache_path)
 469     with open(apkcachefile, 'wb') as cf:
 470         pickle.dump(apkcache, cf)
 471
 472
 473 def get_icon_bytes(apkzip, iconsrc):
 474     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 475     try:
 476         return apkzip.read(iconsrc)
 477     except KeyError:
 478         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 479
 480
 481 def sha256sum(filename):
 482     '''Calculate the sha256 of the given file'''
 483     sha = hashlib.sha256()
 484     with open(filename, 'rb') as f:
 485         while True:
 486             t = f.read(16384)
 487             if len(t) == 0:
 488                 break
 489             sha.update(t)
 490     return sha.hexdigest()
 491
 492
 493 def has_known_vulnerability(filename):
 494     """checks for known vulnerabilities in the APK
 495
 496     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 497     version.  Google also enforces this:
 498     https://support.google.com/faqs/answer/6376725?hl=en
 499
 500     Checks whether there are more than one classes.dex or AndroidManifest.xml
 501     files, which is invalid and an essential part of the "Master Key" attack.
 502     http://www.saurik.com/id/17
 503
 504     Janus is similar to Master Key but is perhaps easier to scan for.
 505     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 506     """
 507
 508     found_vuln = False
 509
 510     # statically load this pattern
 511     if not hasattr(has_known_vulnerability, "pattern"):
 512         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 513
 514     with open(filename.encode(), 'rb') as fp:
 515         first4 = fp.read(4)
 516     if first4 != b'\x50\x4b\x03\x04':
 517         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 518                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 519                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 520
 521     files_in_apk = set()
 522     with zipfile.ZipFile(filename) as zf:
 523         for name in zf.namelist():
 524             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 525                 lib = zf.open(name)
 526                 while True:
 527                     chunk = lib.read(4096)
 528                     if chunk == b'':
 529                         break
 530                     m = has_known_vulnerability.pattern.search(chunk)
 531                     if m:
 532                         version = m.group(1).decode('ascii')
 533                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 534                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 535                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 536                             logging.debug(_('"{path}" contains recent {name} ({version})')
 537                                           .format(path=filename, name=name, version=version))
 538                         else:
 539                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 540                                             .format(path=filename, name=name, version=version))
 541                             found_vuln = True
 542                         break
 543             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 544                 if name in files_in_apk:
 545                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 546                                     .format(apkfilename=filename, name=name))
 547                     found_vuln = True
 548                 files_in_apk.add(name)
 549     return found_vuln
 550
 551
 552 def insert_obbs(repodir, apps, apks):
 553     """Scans the .obb files in a given repo directory and adds them to the
 554     relevant APK instances.  OBB files have versionCodes like APK
 555     files, and they are loosely associated.  If there is an OBB file
 556     present, then any APK with the same or higher versionCode will use
 557     that OBB file.  There are two OBB types: main and patch, each APK
 558     can only have only have one of each.
 559
 560     https://developer.android.com/google/play/expansion-files.html
 561
 562     :param repodir: repo directory to scan
 563     :param apps: list of current, valid apps
 564     :param apks: current information on all APKs
 565
 566     """
 567
 568     def obbWarnDelete(f, msg):
 569         logging.warning(msg + ' ' + f)
 570         if options.delete_unknown:
 571             logging.error(_("Deleting unknown file: {path}").format(path=f))
 572             os.remove(f)
 573
 574     obbs = []
 575     java_Integer_MIN_VALUE = -pow(2, 31)
 576     currentPackageNames = apps.keys()
 577     for f in glob.glob(os.path.join(repodir, '*.obb')):
 578         obbfile = os.path.basename(f)
 579         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 580         chunks = obbfile.split('.')
 581         if chunks[0] != 'main' and chunks[0] != 'patch':
 582             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 583             continue
 584         if not re.match(r'^-?[0-9]+$', chunks[1]):
 585             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 586                           .format(name=chunks[0]))
 587             continue
 588         versionCode = int(chunks[1])
 589         packagename = ".".join(chunks[2:-1])
 590
 591         highestVersionCode = java_Integer_MIN_VALUE
 592         if packagename not in currentPackageNames:
 593             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 594             continue
 595         for apk in apks:
 596             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 597                 highestVersionCode = apk['versionCode']
 598         if versionCode > highestVersionCode:
 599             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 600                           .format(integer=str(versionCode)))
 601             continue
 602         obbsha256 = sha256sum(f)
 603         obbs.append((packagename, versionCode, obbfile, obbsha256))
 604
 605     for apk in apks:
 606         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 607             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 608                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 609                     apk['obbMainFile'] = obbfile
 610                     apk['obbMainFileSha256'] = obbsha256
 611                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 612                     apk['obbPatchFile'] = obbfile
 613                     apk['obbPatchFileSha256'] = obbsha256
 614             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 615                 break
 616
 617
 618 def translate_per_build_anti_features(apps, apks):
 619     """Grab the anti-features list from the build metadata
 620
 621     For most Anti-Features, they are really most applicable per-APK,
 622     not for an app.  An app can fix a vulnerability, add/remove
 623     tracking, etc.  This reads the 'antifeatures' list from the Build
 624     entries in the fdroiddata metadata file, then transforms it into
 625     the 'antiFeatures' list of unique items for the index.
 626
 627     The field key is all lower case in the metadata file to match the
 628     rest of the Build fields.  It is 'antiFeatures' camel case in the
 629     implementation, index, and fdroidclient since it is translated
 630     from the build 'antifeatures' field, not directly included.
 631
 632     """
 633
 634     antiFeatures = dict()
 635     for packageName, app in apps.items():
 636         d = dict()
 637         for build in app['builds']:
 638             afl = build.get('antifeatures')
 639             if afl:
 640                 d[int(build.versionCode)] = afl
 641         if len(d) > 0:
 642             antiFeatures[packageName] = d
 643
 644     for apk in apks:
 645         d = antiFeatures.get(apk['packageName'])
 646         if d:
 647             afl = d.get(apk['versionCode'])
 648             if afl:
 649                 apk['antiFeatures'].update(afl)
 650
 651
 652 def _get_localized_dict(app, locale):
 653     '''get the dict to add localized store metadata to'''
 654     if 'localized' not in app:
 655         app['localized'] = collections.OrderedDict()
 656     if locale not in app['localized']:
 657         app['localized'][locale] = collections.OrderedDict()
 658     return app['localized'][locale]
 659
 660
 661 def _set_localized_text_entry(app, locale, key, f):
 662     limit = config['char_limits'][key]
 663     localized = _get_localized_dict(app, locale)
 664     with open(f) as fp:
 665         text = fp.read()[:limit]
 666         if len(text) > 0:
 667             localized[key] = text
 668
 669
 670 def _set_author_entry(app, key, f):
 671     limit = config['char_limits']['author']
 672     with open(f) as fp:
 673         text = fp.read()[:limit]
 674         if len(text) > 0:
 675             app[key] = text
 676
 677
 678 def _strip_and_copy_image(inpath, outpath):
 679     """Remove any metadata from image and copy it to new path
 680
 681     Sadly, image metadata like EXIF can be used to exploit devices.
 682     It is not used at all in the F-Droid ecosystem, so its much safer
 683     just to remove it entirely.
 684
 685     """
 686
 687     extension = common.get_extension(inpath)[1]
 688     if os.path.isdir(outpath):
 689         outpath = os.path.join(outpath, os.path.basename(inpath))
 690     if extension == 'png':
 691         with open(inpath, 'rb') as fp:
 692             in_image = Image.open(fp)
 693             in_image.save(outpath, "PNG", optimize=True,
 694                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 695     elif extension == 'jpg' or extension == 'jpeg':
 696         with open(inpath, 'rb') as fp:
 697             in_image = Image.open(fp)
 698             data = list(in_image.getdata())
 699             out_image = Image.new(in_image.mode, in_image.size)
 700         out_image.putdata(data)
 701         out_image.save(outpath, "JPEG", optimize=True)
 702     else:
 703         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 704                               .format(extension=extension))
 705
 706
 707 def copy_triple_t_store_metadata(apps):
 708     """Include store metadata from the app's source repo
 709
 710     The Triple-T Gradle Play Publisher is a plugin that has a standard
 711     file layout for all of the metadata and graphics that the Google
 712     Play Store accepts.  Since F-Droid has the git repo, it can just
 713     pluck those files directly.  This method reads any text files into
 714     the app dict, then copies any graphics into the fdroid repo
 715     directory structure.
 716
 717     This needs to be run before insert_localized_app_metadata() so that
 718     the graphics files that are copied into the fdroid repo get
 719     properly indexed.
 720
 721     https://github.com/Triple-T/gradle-play-publisher#upload-images
 722     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 723
 724     """
 725
 726     if not os.path.isdir('build'):
 727         return  # nothing to do
 728
 729     for packageName, app in apps.items():
 730         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 731             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 732             for root, dirs, files in os.walk(d):
 733                 segments = root.split('/')
 734                 locale = segments[-2]
 735                 for f in files:
 736                     if f == 'fulldescription':
 737                         _set_localized_text_entry(app, locale, 'description',
 738                                                   os.path.join(root, f))
 739                         continue
 740                     elif f == 'shortdescription':
 741                         _set_localized_text_entry(app, locale, 'summary',
 742                                                   os.path.join(root, f))
 743                         continue
 744                     elif f == 'title':
 745                         _set_localized_text_entry(app, locale, 'name',
 746                                                   os.path.join(root, f))
 747                         continue
 748                     elif f == 'video':
 749                         _set_localized_text_entry(app, locale, 'video',
 750                                                   os.path.join(root, f))
 751                         continue
 752                     elif f == 'whatsnew':
 753                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 754                                                   os.path.join(root, f))
 755                         continue
 756                     elif f == 'contactEmail':
 757                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 758                         continue
 759                     elif f == 'contactPhone':
 760                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 761                         continue
 762                     elif f == 'contactWebsite':
 763                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 764                         continue
 765
 766                     base, extension = common.get_extension(f)
 767                     dirname = os.path.basename(root)
 768                     if extension in ALLOWED_EXTENSIONS \
 769                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 770                         if segments[-2] == 'listing':
 771                             locale = segments[-3]
 772                         else:
 773                             locale = segments[-2]
 774                         destdir = os.path.join('repo', packageName, locale, dirname)
 775                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 776                         sourcefile = os.path.join(root, f)
 777                         destfile = os.path.join(destdir, os.path.basename(f))
 778                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 779                         _strip_and_copy_image(sourcefile, destfile)
 780
 781
 782 def insert_localized_app_metadata(apps):
 783     """scans standard locations for graphics and localized text
 784
 785     Scans for localized description files, store graphics, and
 786     screenshot PNG files in statically defined screenshots directory
 787     and adds them to the app metadata.  The screenshots and graphic
 788     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 789     and must be in the following layout:
 790     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 791
 792     repo/packageName/locale/featureGraphic.png
 793     repo/packageName/locale/phoneScreenshots/1.png
 794     repo/packageName/locale/phoneScreenshots/2.png
 795
 796     The changelog files must be text files named with the versionCode
 797     ending with ".txt" and must be in the following layout:
 798     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 799
 800     repo/packageName/locale/changelogs/12345.txt
 801
 802     This will scan the each app's source repo then the metadata/ dir
 803     for these standard locations of changelog files.  If it finds
 804     them, they will be added to the dict of all packages, with the
 805     versions in the metadata/ folder taking precendence over the what
 806     is in the app's source repo.
 807
 808     Where "packageName" is the app's packageName and "locale" is the locale
 809     of the graphics, e.g. what language they are in, using the IETF RFC5646
 810     format (en-US, fr-CA, es-MX, etc).
 811
 812     This will also scan the app's git for a fastlane folder, and the
 813     metadata/ folder and the apps' source repos for standard locations
 814     of graphic and screenshot files.  If it finds them, it will copy
 815     them into the repo.  The fastlane files follow this pattern:
 816     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 817
 818     """
 819
 820     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 821     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 822     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 823     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 824
 825     for srcd in sorted(sourcedirs):
 826         if not os.path.isdir(srcd):
 827             continue
 828         for root, dirs, files in os.walk(srcd):
 829             segments = root.split('/')
 830             packageName = segments[1]
 831             if packageName not in apps:
 832                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 833                 continue
 834             locale = segments[-1]
 835             destdir = os.path.join('repo', packageName, locale)
 836
 837             # flavours specified in build receipt
 838             build_flavours = ""
 839             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 840                     and 'gradle' in apps[packageName].builds[-1]:
 841                 build_flavours = apps[packageName].builds[-1].gradle
 842
 843             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 844                 logging.debug("ignoring due to wrong flavour")
 845                 continue
 846
 847             for f in files:
 848                 if f in ('description.txt', 'full_description.txt'):
 849                     _set_localized_text_entry(apps[packageName], locale, 'description',
 850                                               os.path.join(root, f))
 851                     continue
 852                 elif f in ('summary.txt', 'short_description.txt'):
 853                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 854                                               os.path.join(root, f))
 855                     continue
 856                 elif f in ('name.txt', 'title.txt'):
 857                     _set_localized_text_entry(apps[packageName], locale, 'name',
 858                                               os.path.join(root, f))
 859                     continue
 860                 elif f == 'video.txt':
 861                     _set_localized_text_entry(apps[packageName], locale, 'video',
 862                                               os.path.join(root, f))
 863                     continue
 864                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 865                     locale = segments[-2]
 866                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 867                                               os.path.join(root, f))
 868                     continue
 869
 870                 base, extension = common.get_extension(f)
 871                 if locale == 'images':
 872                     locale = segments[-2]
 873                     destdir = os.path.join('repo', packageName, locale)
 874                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 875                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 876                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 877                     _strip_and_copy_image(os.path.join(root, f), destdir)
 878             for d in dirs:
 879                 if d in SCREENSHOT_DIRS:
 880                     if locale == 'images':
 881                         locale = segments[-2]
 882                         destdir = os.path.join('repo', packageName, locale)
 883                     for f in glob.glob(os.path.join(root, d, '*.*')):
 884                         _ignored, extension = common.get_extension(f)
 885                         if extension in ALLOWED_EXTENSIONS:
 886                             screenshotdestdir = os.path.join(destdir, d)
 887                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 888                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 889                             _strip_and_copy_image(f, screenshotdestdir)
 890
 891     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 892     for d in repofiles:
 893         if not os.path.isdir(d):
 894             continue
 895         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 896             if not os.path.isfile(f):
 897                 continue
 898             segments = f.split('/')
 899             packageName = segments[1]
 900             locale = segments[2]
 901             screenshotdir = segments[3]
 902             filename = os.path.basename(f)
 903             base, extension = common.get_extension(filename)
 904
 905             if packageName not in apps:
 906                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 907                                 .format(path=filename, name=packageName))
 908                 continue
 909             graphics = _get_localized_dict(apps[packageName], locale)
 910
 911             if extension not in ALLOWED_EXTENSIONS:
 912                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 913             elif base in GRAPHIC_NAMES:
 914                 # there can only be zero or one of these per locale
 915                 graphics[base] = filename
 916             elif screenshotdir in SCREENSHOT_DIRS:
 917                 # there can any number of these per locale
 918                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 919                 if screenshotdir not in graphics:
 920                     graphics[screenshotdir] = []
 921                 graphics[screenshotdir].append(filename)
 922             else:
 923                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 924
 925
 926 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 927     """Scan a repo for all files with an extension except APK/OBB
 928
 929     :param apkcache: current cached info about all repo files
 930     :param repodir: repo directory to scan
 931     :param knownapks: list of all known files, as per metadata.read_metadata
 932     :param use_date_from_file: use date from file (instead of current date)
 933                                for newly added files
 934     """
 935
 936     cachechanged = False
 937     repo_files = []
 938     repodir = repodir.encode('utf-8')
 939     for name in os.listdir(repodir):
 940         file_extension = common.get_file_extension(name)
 941         if file_extension == 'apk' or file_extension == 'obb':
 942             continue
 943         filename = os.path.join(repodir, name)
 944         name_utf8 = name.decode('utf-8')
 945         if filename.endswith(b'_src.tar.gz'):
 946             logging.debug(_('skipping source tarball: {path}')
 947                           .format(path=filename.decode('utf-8')))
 948             continue
 949         if not common.is_repo_file(filename):
 950             continue
 951         stat = os.stat(filename)
 952         if stat.st_size == 0:
 953             raise FDroidException(_('{path} is zero size!')
 954                                   .format(path=filename))
 955
 956         shasum = sha256sum(filename)
 957         usecache = False
 958         if name in apkcache:
 959             repo_file = apkcache[name]
 960             # added time is cached as tuple but used here as datetime instance
 961             if 'added' in repo_file:
 962                 a = repo_file['added']
 963                 if isinstance(a, datetime):
 964                     repo_file['added'] = a
 965                 else:
 966                     repo_file['added'] = datetime(*a[:6])
 967             if repo_file.get('hash') == shasum:
 968                 logging.debug(_("Reading {apkfilename} from cache")
 969                               .format(apkfilename=name_utf8))
 970                 usecache = True
 971             else:
 972                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 973                               .format(apkfilename=name_utf8))
 974
 975         if not usecache:
 976             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 977             repo_file = collections.OrderedDict()
 978             repo_file['name'] = os.path.splitext(name_utf8)[0]
 979             # TODO rename apkname globally to something more generic
 980             repo_file['apkName'] = name_utf8
 981             repo_file['hash'] = shasum
 982             repo_file['hashType'] = 'sha256'
 983             repo_file['versionCode'] = 0
 984             repo_file['versionName'] = shasum
 985             # the static ID is the SHA256 unless it is set in the metadata
 986             repo_file['packageName'] = shasum
 987
 988             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 989             if m:
 990                 repo_file['packageName'] = m.group(1)
 991                 repo_file['versionCode'] = int(m.group(2))
 992             srcfilename = name + b'_src.tar.gz'
 993             if os.path.exists(os.path.join(repodir, srcfilename)):
 994                 repo_file['srcname'] = srcfilename.decode('utf-8')
 995             repo_file['size'] = stat.st_size
 996
 997             apkcache[name] = repo_file
 998             cachechanged = True
 999
1000         if use_date_from_file:
1001             timestamp = stat.st_ctime
1002             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1003         else:
1004             default_date_param = None
1005
1006         # Record in knownapks, getting the added date at the same time..
1007         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1008                                     default_date=default_date_param)
1009         if added:
1010             repo_file['added'] = added
1011
1012         repo_files.append(repo_file)
1013
1014     return repo_files, cachechanged
1015
1016
1017 def scan_apk(apk_file):
1018     """
1019     Scans an APK file and returns dictionary with metadata of the APK.
1020
1021     Attention: This does *not* verify that the APK signature is correct.
1022
1023     :param apk_file: The (ideally absolute) path to the APK file
1024     :raises BuildException
1025     :return A dict containing APK metadata
1026     """
1027     apk = {
1028         'hash': sha256sum(apk_file),
1029         'hashType': 'sha256',
1030         'uses-permission': [],
1031         'uses-permission-sdk-23': [],
1032         'features': [],
1033         'icons_src': {},
1034         'icons': {},
1035         'antiFeatures': set(),
1036     }
1037
1038     if SdkToolsPopen(['aapt', 'version'], output=False):
1039         scan_apk_aapt(apk, apk_file)
1040     else:
1041         scan_apk_androguard(apk, apk_file)
1042
1043     # Get the signature, or rather the signing key fingerprints
1044     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1045     apk['sig'] = getsig(apk_file)
1046     if not apk['sig']:
1047         raise BuildException("Failed to get apk signature")
1048     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1049                                                                apk_file))
1050     if not apk.get('signer'):
1051         raise BuildException("Failed to get apk signing key fingerprint")
1052
1053     # Get size of the APK
1054     apk['size'] = os.path.getsize(apk_file)
1055
1056     if 'minSdkVersion' not in apk:
1057         logging.warning("No SDK version information found in {0}".format(apk_file))
1058         apk['minSdkVersion'] = 1
1059
1060     # Check for known vulnerabilities
1061     if has_known_vulnerability(apk_file):
1062         apk['antiFeatures'].add('KnownVuln')
1063
1064     return apk
1065
1066
1067 def scan_apk_aapt(apk, apkfile):
1068     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1069     if p.returncode != 0:
1070         if options.delete_unknown:
1071             if os.path.exists(apkfile):
1072                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1073                 os.remove(apkfile)
1074             else:
1075                 logging.error("Could not find {0} to remove it".format(apkfile))
1076         else:
1077             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1078         raise BuildException(_("Invalid APK"))
1079     for line in p.output.splitlines():
1080         if line.startswith("package:"):
1081             try:
1082                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1083                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1084                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1085             except Exception as e:
1086                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1087         elif line.startswith("application:"):
1088             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1089             # Keep path to non-dpi icon in case we need it
1090             match = re.match(APK_ICON_PAT_NODPI, line)
1091             if match:
1092                 apk['icons_src']['-1'] = match.group(1)
1093         elif line.startswith("launchable-activity:"):
1094             # Only use launchable-activity as fallback to application
1095             if not apk['name']:
1096                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1097             if '-1' not in apk['icons_src']:
1098                 match = re.match(APK_ICON_PAT_NODPI, line)
1099                 if match:
1100                     apk['icons_src']['-1'] = match.group(1)
1101         elif line.startswith("application-icon-"):
1102             match = re.match(APK_ICON_PAT, line)
1103             if match:
1104                 density = match.group(1)
1105                 path = match.group(2)
1106                 apk['icons_src'][density] = path
1107         elif line.startswith("sdkVersion:"):
1108             m = re.match(APK_SDK_VERSION_PAT, line)
1109             if m is None:
1110                 logging.error(line.replace('sdkVersion:', '')
1111                               + ' is not a valid minSdkVersion!')
1112             else:
1113                 apk['minSdkVersion'] = m.group(1)
1114                 # if target not set, default to min
1115                 if 'targetSdkVersion' not in apk:
1116                     apk['targetSdkVersion'] = m.group(1)
1117         elif line.startswith("targetSdkVersion:"):
1118             m = re.match(APK_SDK_VERSION_PAT, line)
1119             if m is None:
1120                 logging.error(line.replace('targetSdkVersion:', '')
1121                               + ' is not a valid targetSdkVersion!')
1122             else:
1123                 apk['targetSdkVersion'] = m.group(1)
1124         elif line.startswith("maxSdkVersion:"):
1125             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1126         elif line.startswith("native-code:"):
1127             apk['nativecode'] = []
1128             for arch in line[13:].split(' '):
1129                 apk['nativecode'].append(arch[1:-1])
1130         elif line.startswith('uses-permission:'):
1131             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1132             if perm_match['maxSdkVersion']:
1133                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1134             permission = UsesPermission(
1135                 perm_match['name'],
1136                 perm_match['maxSdkVersion']
1137             )
1138
1139             apk['uses-permission'].append(permission)
1140         elif line.startswith('uses-permission-sdk-23:'):
1141             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1142             if perm_match['maxSdkVersion']:
1143                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1144             permission_sdk_23 = UsesPermissionSdk23(
1145                 perm_match['name'],
1146                 perm_match['maxSdkVersion']
1147             )
1148
1149             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1150
1151         elif line.startswith('uses-feature:'):
1152             feature = re.match(APK_FEATURE_PAT, line).group(1)
1153             # Filter out this, it's only added with the latest SDK tools and
1154             # causes problems for lots of apps.
1155             if feature != "android.hardware.screen.portrait" \
1156                     and feature != "android.hardware.screen.landscape":
1157                 if feature.startswith("android.feature."):
1158                     feature = feature[16:]
1159                 apk['features'].add(feature)
1160
1161
1162 def scan_apk_androguard(apk, apkfile):
1163     try:
1164         from androguard.core.bytecodes.apk import APK
1165         apkobject = APK(apkfile)
1166         if apkobject.is_valid_APK():
1167             arsc = apkobject.get_android_resources()
1168         else:
1169             if options.delete_unknown:
1170                 if os.path.exists(apkfile):
1171                     logging.error(_("Failed to get apk information, deleting {path}")
1172                                   .format(path=apkfile))
1173                     os.remove(apkfile)
1174                 else:
1175                     logging.error(_("Could not find {path} to remove it")
1176                                   .format(path=apkfile))
1177             else:
1178                 logging.error(_("Failed to get apk information, skipping {path}")
1179                               .format(path=apkfile))
1180             raise BuildException(_("Invalid APK"))
1181     except ImportError:
1182         raise FDroidException("androguard library is not installed and aapt not present")
1183     except FileNotFoundError:
1184         logging.error(_("Could not open apk file for analysis"))
1185         raise BuildException(_("Invalid APK"))
1186
1187     apk['packageName'] = apkobject.get_package()
1188     apk['versionCode'] = int(apkobject.get_androidversion_code())
1189     apk['versionName'] = apkobject.get_androidversion_name()
1190     if apk['versionName'][0] == "@":
1191         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1192         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1193         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1194     apk['name'] = apkobject.get_app_name()
1195
1196     if apkobject.get_max_sdk_version() is not None:
1197         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1198     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1199     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1200
1201     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1202     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1203
1204     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1205
1206     for file in apkobject.get_files():
1207         d_re = density_re.match(file)
1208         if d_re:
1209             folder = d_re.group(1).split('-')
1210             if len(folder) > 1:
1211                 resolution = folder[1]
1212             else:
1213                 resolution = 'mdpi'
1214             density = screen_resolutions[resolution]
1215             apk['icons_src'][density] = d_re.group(0)
1216
1217     if apk['icons_src'].get('-1') is None:
1218         apk['icons_src']['-1'] = apk['icons_src']['160']
1219
1220     arch_re = re.compile("^lib/(.*)/.*$")
1221     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1222     if len(arch) >= 1:
1223         apk['nativecode'] = []
1224         apk['nativecode'].extend(sorted(list(arch)))
1225
1226     xml = apkobject.get_android_manifest_xml()
1227
1228     for item in xml.getElementsByTagName('uses-permission'):
1229         name = str(item.getAttribute("android:name"))
1230         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1231         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1232         permission = UsesPermission(
1233             name,
1234             maxSdkVersion
1235         )
1236         apk['uses-permission'].append(permission)
1237
1238     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1239         name = str(item.getAttribute("android:name"))
1240         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1241         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1242         permission_sdk_23 = UsesPermissionSdk23(
1243             name,
1244             maxSdkVersion
1245         )
1246         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1247
1248     for item in xml.getElementsByTagName('uses-feature'):
1249         feature = str(item.getAttribute("android:name"))
1250         if feature != "android.hardware.screen.portrait" \
1251                 and feature != "android.hardware.screen.landscape":
1252             if feature.startswith("android.feature."):
1253                 feature = feature[16:]
1254         apk['features'].append(feature)
1255
1256
1257 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1258                 allow_disabled_algorithms=False, archive_bad_sig=False):
1259     """Processes the apk with the given filename in the given repo directory.
1260
1261     This also extracts the icons.
1262
1263     :param apkcache: current apk cache information
1264     :param apkfilename: the filename of the apk to scan
1265     :param repodir: repo directory to scan
1266     :param knownapks: known apks info
1267     :param use_date_from_apk: use date from APK (instead of current date)
1268                               for newly added APKs
1269     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1270                                       disabled algorithms in the signature (e.g. MD5)
1271     :param archive_bad_sig: move APKs with a bad signature to the archive
1272     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1273      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1274     """
1275
1276     apk = {}
1277     apkfile = os.path.join(repodir, apkfilename)
1278
1279     cachechanged = False
1280     usecache = False
1281     if apkfilename in apkcache:
1282         apk = apkcache[apkfilename]
1283         if apk.get('hash') == sha256sum(apkfile):
1284             logging.debug(_("Reading {apkfilename} from cache")
1285                           .format(apkfilename=apkfilename))
1286             usecache = True
1287         else:
1288             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1289                           .format(apkfilename=apkfilename))
1290
1291     if not usecache:
1292         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1293
1294         try:
1295             apk = scan_apk(apkfile)
1296         except BuildException:
1297             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1298                             .format(apkfilename=apkfilename))
1299             return True, None, False
1300
1301         # Check for debuggable apks...
1302         if common.isApkAndDebuggable(apkfile):
1303             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1304
1305         if options.rename_apks:
1306             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1307             std_short_name = os.path.join(repodir, n)
1308             if apkfile != std_short_name:
1309                 if os.path.exists(std_short_name):
1310                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1311                     if apkfile != std_long_name:
1312                         if os.path.exists(std_long_name):
1313                             dupdir = os.path.join('duplicates', repodir)
1314                             if not os.path.isdir(dupdir):
1315                                 os.makedirs(dupdir, exist_ok=True)
1316                             dupfile = os.path.join('duplicates', std_long_name)
1317                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1318                             os.rename(apkfile, dupfile)
1319                             return True, None, False
1320                         else:
1321                             os.rename(apkfile, std_long_name)
1322                     apkfile = std_long_name
1323                 else:
1324                     os.rename(apkfile, std_short_name)
1325                     apkfile = std_short_name
1326                 apkfilename = apkfile[len(repodir) + 1:]
1327
1328         apk['apkName'] = apkfilename
1329         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1330         if os.path.exists(os.path.join(repodir, srcfilename)):
1331             apk['srcname'] = srcfilename
1332
1333         # verify the jar signature is correct, allow deprecated
1334         # algorithms only if the APK is in the archive.
1335         skipapk = False
1336         if not common.verify_apk_signature(apkfile):
1337             if repodir == 'archive' or allow_disabled_algorithms:
1338                 if common.verify_old_apk_signature(apkfile):
1339                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1340                 else:
1341                     skipapk = True
1342             else:
1343                 skipapk = True
1344
1345         if skipapk:
1346             if archive_bad_sig:
1347                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1348                                 .format(apkfilename=apkfilename))
1349                 move_apk_between_sections(repodir, 'archive', apk)
1350             else:
1351                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1352                                 .format(apkfilename=apkfilename))
1353             return True, None, False
1354
1355         apkzip = zipfile.ZipFile(apkfile, 'r')
1356
1357         manifest = apkzip.getinfo('AndroidManifest.xml')
1358         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1359         if (1980, 0, 0) != manifest.date_time[0:3]:
1360             try:
1361                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1362             except ValueError as e:
1363                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1364                                 .format(apkfilename=apkfile) + str(e))
1365
1366         # extract icons from APK zip file
1367         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1368         try:
1369             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1370         finally:
1371             apkzip.close()  # ensure that APK zip file gets closed
1372
1373         # resize existing icons for densities missing in the APK
1374         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1375
1376         if use_date_from_apk and manifest.date_time[1] != 0:
1377             default_date_param = datetime(*manifest.date_time)
1378         else:
1379             default_date_param = None
1380
1381         # Record in known apks, getting the added date at the same time..
1382         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1383                                     default_date=default_date_param)
1384         if added:
1385             apk['added'] = added
1386
1387         apkcache[apkfilename] = apk
1388         cachechanged = True
1389
1390     return False, apk, cachechanged
1391
1392
1393 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1394     """Processes the apks in the given repo directory.
1395
1396     This also extracts the icons.
1397
1398     :param apkcache: current apk cache information
1399     :param repodir: repo directory to scan
1400     :param knownapks: known apks info
1401     :param use_date_from_apk: use date from APK (instead of current date)
1402                               for newly added APKs
1403     :returns: (apks, cachechanged) where apks is a list of apk information,
1404               and cachechanged is True if the apkcache got changed.
1405     """
1406
1407     cachechanged = False
1408
1409     for icon_dir in get_all_icon_dirs(repodir):
1410         if os.path.exists(icon_dir):
1411             if options.clean:
1412                 shutil.rmtree(icon_dir)
1413                 os.makedirs(icon_dir)
1414         else:
1415             os.makedirs(icon_dir)
1416
1417     apks = []
1418     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1419         apkfilename = apkfile[len(repodir) + 1:]
1420         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1421         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1422                                              use_date_from_apk, ada, True)
1423         if skip:
1424             continue
1425         apks.append(apk)
1426         cachechanged = cachechanged or cachethis
1427
1428     return apks, cachechanged
1429
1430
1431 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1432     """
1433     Extracts icons from the given APK zip in various densities,
1434     saves them into given repo directory
1435     and stores their names in the APK metadata dictionary.
1436
1437     :param icon_filename: A string representing the icon's file name
1438     :param apk: A populated dictionary containing APK metadata.
1439                 Needs to have 'icons_src' key
1440     :param apkzip: An opened zipfile.ZipFile of the APK file
1441     :param repo_dir: The directory of the APK's repository
1442     :return: A list of icon densities that are missing
1443     """
1444     empty_densities = []
1445     for density in screen_densities:
1446         if density not in apk['icons_src']:
1447             empty_densities.append(density)
1448             continue
1449         icon_src = apk['icons_src'][density]
1450         icon_dir = get_icon_dir(repo_dir, density)
1451         icon_dest = os.path.join(icon_dir, icon_filename)
1452
1453         # Extract the icon files per density
1454         if icon_src.endswith('.xml'):
1455             png = os.path.basename(icon_src)[:-4] + '.png'
1456             for f in apkzip.namelist():
1457                 if f.endswith(png):
1458                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1459                     if m and screen_resolutions[m.group(2)] == density:
1460                         icon_src = f
1461             if icon_src.endswith('.xml'):
1462                 empty_densities.append(density)
1463                 continue
1464         try:
1465             with open(icon_dest, 'wb') as f:
1466                 f.write(get_icon_bytes(apkzip, icon_src))
1467             apk['icons'][density] = icon_filename
1468         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1469             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1470             del apk['icons_src'][density]
1471             empty_densities.append(density)
1472
1473     if '-1' in apk['icons_src']:
1474         icon_src = apk['icons_src']['-1']
1475         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1476         with open(icon_path, 'wb') as f:
1477             f.write(get_icon_bytes(apkzip, icon_src))
1478         im = None
1479         try:
1480             im = Image.open(icon_path)
1481             dpi = px_to_dpi(im.size[0])
1482             for density in screen_densities:
1483                 if density in apk['icons']:
1484                     break
1485                 if density == screen_densities[-1] or dpi >= int(density):
1486                     apk['icons'][density] = icon_filename
1487                     shutil.move(icon_path,
1488                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1489                     empty_densities.remove(density)
1490                     break
1491         except Exception as e:
1492             logging.warning(_("Failed reading {path}: {error}")
1493                             .format(path=icon_path, error=e))
1494         finally:
1495             if im and hasattr(im, 'close'):
1496                 im.close()
1497
1498     if apk['icons']:
1499         apk['icon'] = icon_filename
1500
1501     return empty_densities
1502
1503
1504 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1505     """
1506     Resize existing icons for densities missing in the APK to ensure all densities are available
1507
1508     :param empty_densities: A list of icon densities that are missing
1509     :param icon_filename: A string representing the icon's file name
1510     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1511     :param repo_dir: The directory of the APK's repository
1512     """
1513     # First try resizing down to not lose quality
1514     last_density = None
1515     for density in screen_densities:
1516         if density not in empty_densities:
1517             last_density = density
1518             continue
1519         if last_density is None:
1520             continue
1521         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1522
1523         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1524         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1525         fp = None
1526         try:
1527             fp = open(last_icon_path, 'rb')
1528             im = Image.open(fp)
1529
1530             size = dpi_to_px(density)
1531
1532             im.thumbnail((size, size), Image.ANTIALIAS)
1533             im.save(icon_path, "PNG", optimize=True,
1534                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1535             empty_densities.remove(density)
1536         except Exception as e:
1537             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1538         finally:
1539             if fp:
1540                 fp.close()
1541
1542     # Then just copy from the highest resolution available
1543     last_density = None
1544     for density in reversed(screen_densities):
1545         if density not in empty_densities:
1546             last_density = density
1547             continue
1548
1549         if last_density is None:
1550             continue
1551
1552         shutil.copyfile(
1553             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1554             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1555         )
1556         empty_densities.remove(density)
1557
1558     for density in screen_densities:
1559         icon_dir = get_icon_dir(repo_dir, density)
1560         icon_dest = os.path.join(icon_dir, icon_filename)
1561         resize_icon(icon_dest, density)
1562
1563     # Copy from icons-mdpi to icons since mdpi is the baseline density
1564     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1565     if os.path.isfile(baseline):
1566         apk['icons']['0'] = icon_filename
1567         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1568
1569
1570 def apply_info_from_latest_apk(apps, apks):
1571     """
1572     Some information from the apks needs to be applied up to the application level.
1573     When doing this, we use the info from the most recent version's apk.
1574     We deal with figuring out when the app was added and last updated at the same time.
1575     """
1576     for appid, app in apps.items():
1577         bestver = UNSET_VERSION_CODE
1578         for apk in apks:
1579             if apk['packageName'] == appid:
1580                 if apk['versionCode'] > bestver:
1581                     bestver = apk['versionCode']
1582                     bestapk = apk
1583
1584                 if 'added' in apk:
1585                     if not app.added or apk['added'] < app.added:
1586                         app.added = apk['added']
1587                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1588                         app.lastUpdated = apk['added']
1589
1590         if not app.added:
1591             logging.debug("Don't know when " + appid + " was added")
1592         if not app.lastUpdated:
1593             logging.debug("Don't know when " + appid + " was last updated")
1594
1595         if bestver == UNSET_VERSION_CODE:
1596
1597             if app.Name is None:
1598                 app.Name = app.AutoName or appid
1599             app.icon = None
1600             logging.debug("Application " + appid + " has no packages")
1601         else:
1602             if app.Name is None:
1603                 app.Name = bestapk['name']
1604             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1605             if app.CurrentVersionCode is None:
1606                 app.CurrentVersionCode = str(bestver)
1607
1608
1609 def make_categories_txt(repodir, categories):
1610     '''Write a category list in the repo to allow quick access'''
1611     catdata = ''
1612     for cat in sorted(categories):
1613         catdata += cat + '\n'
1614     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1615         f.write(catdata)
1616
1617
1618 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1619
1620     def filter_apk_list_sorted(apk_list):
1621         res = []
1622         for apk in apk_list:
1623             if apk['packageName'] == appid:
1624                 res.append(apk)
1625
1626         # Sort the apk list by version code. First is highest/newest.
1627         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1628
1629     for appid, app in apps.items():
1630
1631         if app.ArchivePolicy:
1632             keepversions = int(app.ArchivePolicy[:-9])
1633         else:
1634             keepversions = defaultkeepversions
1635
1636         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1637                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1638
1639         current_app_apks = filter_apk_list_sorted(apks)
1640         if len(current_app_apks) > keepversions:
1641             # Move back the ones we don't want.
1642             for apk in current_app_apks[keepversions:]:
1643                 move_apk_between_sections(repodir, archivedir, apk)
1644                 archapks.append(apk)
1645                 apks.remove(apk)
1646
1647         current_app_archapks = filter_apk_list_sorted(archapks)
1648         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1649             kept = 0
1650             # Move forward the ones we want again, except DisableAlgorithm
1651             for apk in current_app_archapks:
1652                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1653                     move_apk_between_sections(archivedir, repodir, apk)
1654                     archapks.remove(apk)
1655                     apks.append(apk)
1656                     kept += 1
1657                 if kept == keepversions:
1658                     break
1659
1660
1661 def move_apk_between_sections(from_dir, to_dir, apk):
1662     """move an APK from repo to archive or vice versa"""
1663
1664     def _move_file(from_dir, to_dir, filename, ignore_missing):
1665         from_path = os.path.join(from_dir, filename)
1666         if ignore_missing and not os.path.exists(from_path):
1667             return
1668         to_path = os.path.join(to_dir, filename)
1669         if not os.path.exists(to_dir):
1670             os.mkdir(to_dir)
1671         shutil.move(from_path, to_path)
1672
1673     if from_dir == to_dir:
1674         return
1675
1676     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1677     _move_file(from_dir, to_dir, apk['apkName'], False)
1678     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1679     for density in all_screen_densities:
1680         from_icon_dir = get_icon_dir(from_dir, density)
1681         to_icon_dir = get_icon_dir(to_dir, density)
1682         if density not in apk.get('icons', []):
1683             continue
1684         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1685     if 'srcname' in apk:
1686         _move_file(from_dir, to_dir, apk['srcname'], False)
1687
1688
1689 def add_apks_to_per_app_repos(repodir, apks):
1690     apks_per_app = dict()
1691     for apk in apks:
1692         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1693         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1694         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1695         apks_per_app[apk['packageName']] = apk
1696
1697         if not os.path.exists(apk['per_app_icons']):
1698             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1699             os.makedirs(apk['per_app_icons'])
1700
1701         apkpath = os.path.join(repodir, apk['apkName'])
1702         shutil.copy(apkpath, apk['per_app_repo'])
1703         apksigpath = apkpath + '.sig'
1704         if os.path.exists(apksigpath):
1705             shutil.copy(apksigpath, apk['per_app_repo'])
1706         apkascpath = apkpath + '.asc'
1707         if os.path.exists(apkascpath):
1708             shutil.copy(apkascpath, apk['per_app_repo'])
1709
1710
1711 def create_metadata_from_template(apk):
1712     '''create a new metadata file using internal or external template
1713
1714     Generate warnings for apk's with no metadata (or create skeleton
1715     metadata files, if requested on the command line).  Though the
1716     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1717     since those impose things on the metadata file made from the
1718     template: field sort order, empty field value, formatting, etc.
1719     '''
1720
1721     import yaml
1722     if os.path.exists('template.yml'):
1723         with open('template.yml') as f:
1724             metatxt = f.read()
1725         if 'name' in apk and apk['name'] != '':
1726             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1727                              r'\1 ' + apk['name'],
1728                              metatxt,
1729                              flags=re.IGNORECASE | re.MULTILINE)
1730         else:
1731             logging.warning(_('{appid} does not have a name! Using package name instead.')
1732                             .format(appid=apk['packageName']))
1733             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1734                              r'\1 ' + apk['packageName'],
1735                              metatxt,
1736                              flags=re.IGNORECASE | re.MULTILINE)
1737         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1738             f.write(metatxt)
1739     else:
1740         app = dict()
1741         app['Categories'] = [os.path.basename(os.getcwd())]
1742         # include some blanks as part of the template
1743         app['AuthorName'] = ''
1744         app['Summary'] = ''
1745         app['WebSite'] = ''
1746         app['IssueTracker'] = ''
1747         app['SourceCode'] = ''
1748         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1749         if 'name' in apk and apk['name'] != '':
1750             app['Name'] = apk['name']
1751         else:
1752             logging.warning(_('{appid} does not have a name! Using package name instead.')
1753                             .format(appid=apk['packageName']))
1754             app['Name'] = apk['packageName']
1755         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1756             yaml.dump(app, f, default_flow_style=False)
1757     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1758
1759
1760 config = None
1761 options = None
1762
1763
1764 def main():
1765
1766     global config, options
1767
1768     # Parse command line...
1769     parser = ArgumentParser()
1770     common.setup_global_opts(parser)
1771     parser.add_argument("--create-key", action="store_true", default=False,
1772                         help=_("Add a repo signing key to an unsigned repo"))
1773     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1774                         help=_("Add skeleton metadata files for APKs that are missing them"))
1775     parser.add_argument("--delete-unknown", action="store_true", default=False,
1776                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1777     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1778                         help=_("Report on build data status"))
1779     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1780                         help=_("Interactively ask about things that need updating."))
1781     parser.add_argument("-I", "--icons", action="store_true", default=False,
1782                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1783     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1784                         help=_("Specify editor to use in interactive mode. Default " +
1785                                "is {path}").format(path='/etc/alternatives/editor'))
1786     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1787                         help=_("Update the wiki"))
1788     parser.add_argument("--pretty", action="store_true", default=False,
1789                         help=_("Produce human-readable XML/JSON for index files"))
1790     parser.add_argument("--clean", action="store_true", default=False,
1791                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1792     parser.add_argument("--nosign", action="store_true", default=False,
1793                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1794     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1795                         help=_("Use date from APK instead of current time for newly added APKs"))
1796     parser.add_argument("--rename-apks", action="store_true", default=False,
1797                         help=_("Rename APK files that do not match package.name_123.apk"))
1798     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1799                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1800     metadata.add_metadata_arguments(parser)
1801     options = parser.parse_args()
1802     metadata.warnings_action = options.W
1803
1804     config = common.read_config(options)
1805
1806     if not ('jarsigner' in config and 'keytool' in config):
1807         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1808
1809     repodirs = ['repo']
1810     if config['archive_older'] != 0:
1811         repodirs.append('archive')
1812         if not os.path.exists('archive'):
1813             os.mkdir('archive')
1814
1815     if options.icons:
1816         resize_all_icons(repodirs)
1817         sys.exit(0)
1818
1819     if options.rename_apks:
1820         options.clean = True
1821
1822     # check that icons exist now, rather than fail at the end of `fdroid update`
1823     for k in ['repo_icon', 'archive_icon']:
1824         if k in config:
1825             if not os.path.exists(config[k]):
1826                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1827                                  .format(name=k, path=config[k]))
1828                 sys.exit(1)
1829
1830     # if the user asks to create a keystore, do it now, reusing whatever it can
1831     if options.create_key:
1832         if os.path.exists(config['keystore']):
1833             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1834             logging.critical("\t'" + config['keystore'] + "'")
1835             sys.exit(1)
1836
1837         if 'repo_keyalias' not in config:
1838             config['repo_keyalias'] = socket.getfqdn()
1839             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1840         if 'keydname' not in config:
1841             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1842             common.write_to_config(config, 'keydname', config['keydname'])
1843         if 'keystore' not in config:
1844             config['keystore'] = common.default_config['keystore']
1845             common.write_to_config(config, 'keystore', config['keystore'])
1846
1847         password = common.genpassword()
1848         if 'keystorepass' not in config:
1849             config['keystorepass'] = password
1850             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1851         if 'keypass' not in config:
1852             config['keypass'] = password
1853             common.write_to_config(config, 'keypass', config['keypass'])
1854         common.genkeystore(config)
1855
1856     # Get all apps...
1857     apps = metadata.read_metadata()
1858
1859     # Generate a list of categories...
1860     categories = set()
1861     for app in apps.values():
1862         categories.update(app.Categories)
1863
1864     # Read known apks data (will be updated and written back when we've finished)
1865     knownapks = common.KnownApks()
1866
1867     # Get APK cache
1868     apkcache = get_cache()
1869
1870     # Delete builds for disabled apps
1871     delete_disabled_builds(apps, apkcache, repodirs)
1872
1873     # Scan all apks in the main repo
1874     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1875
1876     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1877                                            options.use_date_from_apk)
1878     cachechanged = cachechanged or fcachechanged
1879     apks += files
1880     for apk in apks:
1881         if apk['packageName'] not in apps:
1882             if options.create_metadata:
1883                 create_metadata_from_template(apk)
1884                 apps = metadata.read_metadata()
1885             else:
1886                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1887                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1888                 if options.delete_unknown:
1889                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1890                                  .format(apkfilename=apk['apkName']))
1891                     rmf = os.path.join(repodirs[0], apk['apkName'])
1892                     if not os.path.exists(rmf):
1893                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1894                     else:
1895                         os.remove(rmf)
1896                 else:
1897                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1898
1899     copy_triple_t_store_metadata(apps)
1900     insert_obbs(repodirs[0], apps, apks)
1901     insert_localized_app_metadata(apps)
1902     translate_per_build_anti_features(apps, apks)
1903
1904     # Scan the archive repo for apks as well
1905     if len(repodirs) > 1:
1906         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1907         if cc:
1908             cachechanged = True
1909     else:
1910         archapks = []
1911
1912     # Apply information from latest apks to the application and update dates
1913     apply_info_from_latest_apk(apps, apks + archapks)
1914
1915     # Sort the app list by name, then the web site doesn't have to by default.
1916     # (we had to wait until we'd scanned the apks to do this, because mostly the
1917     # name comes from there!)
1918     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1919
1920     # APKs are placed into multiple repos based on the app package, providing
1921     # per-app subscription feeds for nightly builds and things like it
1922     if config['per_app_repos']:
1923         add_apks_to_per_app_repos(repodirs[0], apks)
1924         for appid, app in apps.items():
1925             repodir = os.path.join(appid, 'fdroid', 'repo')
1926             appdict = dict()
1927             appdict[appid] = app
1928             if os.path.isdir(repodir):
1929                 index.make(appdict, [appid], apks, repodir, False)
1930             else:
1931                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1932         return
1933
1934     if len(repodirs) > 1:
1935         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1936
1937     # Make the index for the main repo...
1938     index.make(apps, sortedids, apks, repodirs[0], False)
1939     make_categories_txt(repodirs[0], categories)
1940
1941     # If there's an archive repo,  make the index for it. We already scanned it
1942     # earlier on.
1943     if len(repodirs) > 1:
1944         index.make(apps, sortedids, archapks, repodirs[1], True)
1945
1946     git_remote = config.get('binary_transparency_remote')
1947     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1948         from . import btlog
1949         btlog.make_binary_transparency_log(repodirs)
1950
1951     if config['update_stats']:
1952         # Update known apks info...
1953         knownapks.writeifchanged()
1954
1955         # Generate latest apps data for widget
1956         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1957             data = ''
1958             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1959                 for line in f:
1960                     appid = line.rstrip()
1961                     data += appid + "\t"
1962                     app = apps[appid]
1963                     data += app.Name + "\t"
1964                     if app.icon is not None:
1965                         data += app.icon + "\t"
1966                     data += app.License + "\n"
1967             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1968                 f.write(data)
1969
1970     if cachechanged:
1971         write_cache(apkcache)
1972
1973     # Update the wiki...
1974     if options.wiki:
1975         update_wiki(apps, sortedids, apks + archapks)
1976
1977     logging.info(_("Finished"))
1978
1979
1980 if __name__ == "__main__":
1981     main()