fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*)'")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0' or density == '65534':
  98         return os.path.join(repodir, "icons")
  99     else:
 100         return os.path.join(repodir, "icons-%s" % density)
 101
 102
 103 def get_icon_dirs(repodir):
 104     for density in screen_densities:
 105         yield get_icon_dir(repodir, density)
 106
 107
 108 def get_all_icon_dirs(repodir):
 109     for density in all_screen_densities:
 110         yield get_icon_dir(repodir, density)
 111
 112
 113 def update_wiki(apps, sortedids, apks):
 114     """Update the wiki
 115
 116     :param apps: fully populated list of all applications
 117     :param apks: all apks, except...
 118     """
 119     logging.info("Updating wiki")
 120     wikicat = 'Apps'
 121     wikiredircat = 'App Redirects'
 122     import mwclient
 123     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 124                          path=config['wiki_path'])
 125     site.login(config['wiki_user'], config['wiki_password'])
 126     generated_pages = {}
 127     generated_redirects = {}
 128
 129     for appid in sortedids:
 130         app = metadata.App(apps[appid])
 131
 132         wikidata = ''
 133         if app.Disabled:
 134             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 135         if app.AntiFeatures:
 136             for af in sorted(app.AntiFeatures):
 137                 wikidata += '{{AntiFeature|' + af + '}}\n'
 138         if app.RequiresRoot:
 139             requiresroot = 'Yes'
 140         else:
 141             requiresroot = 'No'
 142         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 143             appid,
 144             app.Name,
 145             app.added.strftime('%Y-%m-%d') if app.added else '',
 146             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 147             app.SourceCode,
 148             app.IssueTracker,
 149             app.WebSite,
 150             app.Changelog,
 151             app.Donate,
 152             app.FlattrID,
 153             app.LiberapayID,
 154             app.Bitcoin,
 155             app.Litecoin,
 156             app.License,
 157             requiresroot,
 158             app.AuthorName,
 159             app.AuthorEmail)
 160
 161         if app.Provides:
 162             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 163
 164         wikidata += app.Summary
 165         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 166
 167         wikidata += "=Description=\n"
 168         wikidata += metadata.description_wiki(app.Description) + "\n"
 169
 170         wikidata += "=Maintainer Notes=\n"
 171         if app.MaintainerNotes:
 172             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 173         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 174
 175         # Get a list of all packages for this application...
 176         apklist = []
 177         gotcurrentver = False
 178         cantupdate = False
 179         buildfails = False
 180         for apk in apks:
 181             if apk['packageName'] == appid:
 182                 if str(apk['versionCode']) == app.CurrentVersionCode:
 183                     gotcurrentver = True
 184                 apklist.append(apk)
 185         # Include ones we can't build, as a special case...
 186         for build in app.builds:
 187             if build.disable:
 188                 if build.versionCode == app.CurrentVersionCode:
 189                     cantupdate = True
 190                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 191                 apklist.append({'versionCode': int(build.versionCode),
 192                                 'versionName': build.versionName,
 193                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 194                                 })
 195             else:
 196                 builtit = False
 197                 for apk in apklist:
 198                     if apk['versionCode'] == int(build.versionCode):
 199                         builtit = True
 200                         break
 201                 if not builtit:
 202                     buildfails = True
 203                     apklist.append({'versionCode': int(build.versionCode),
 204                                     'versionName': build.versionName,
 205                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 206                                     })
 207         if app.CurrentVersionCode == '0':
 208             cantupdate = True
 209         # Sort with most recent first...
 210         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 211
 212         wikidata += "=Versions=\n"
 213         if len(apklist) == 0:
 214             wikidata += "We currently have no versions of this app available."
 215         elif not gotcurrentver:
 216             wikidata += "We don't have the current version of this app."
 217         else:
 218             wikidata += "We have the current version of this app."
 219         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 220         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 221         if len(app.NoSourceSince) > 0:
 222             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 223         if len(app.CurrentVersion) > 0:
 224             wikidata += "The current (recommended) version is " + app.CurrentVersion
 225             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 226         validapks = 0
 227         for apk in apklist:
 228             wikidata += "==" + apk['versionName'] + "==\n"
 229
 230             if 'buildproblem' in apk:
 231                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 232             else:
 233                 validapks += 1
 234                 wikidata += "This version is built and signed by "
 235                 if 'srcname' in apk:
 236                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 237                 else:
 238                     wikidata += "the original developer.\n\n"
 239             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 240
 241         wikidata += '\n[[Category:' + wikicat + ']]\n'
 242         if len(app.NoSourceSince) > 0:
 243             wikidata += '\n[[Category:Apps missing source code]]\n'
 244         if validapks == 0 and not app.Disabled:
 245             wikidata += '\n[[Category:Apps with no packages]]\n'
 246         if cantupdate and not app.Disabled:
 247             wikidata += "\n[[Category:Apps we cannot update]]\n"
 248         if buildfails and not app.Disabled:
 249             wikidata += "\n[[Category:Apps with failing builds]]\n"
 250         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 251             wikidata += '\n[[Category:Apps to Update]]\n'
 252         if app.Disabled:
 253             wikidata += '\n[[Category:Apps that are disabled]]\n'
 254         if app.UpdateCheckMode == 'None' and not app.Disabled:
 255             wikidata += '\n[[Category:Apps with no update check]]\n'
 256         for appcat in app.Categories:
 257             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 258
 259         # We can't have underscores in the page name, even if they're in
 260         # the package ID, because MediaWiki messes with them...
 261         pagename = appid.replace('_', ' ')
 262
 263         # Drop a trailing newline, because mediawiki is going to drop it anyway
 264         # and it we don't we'll think the page has changed when it hasn't...
 265         if wikidata.endswith('\n'):
 266             wikidata = wikidata[:-1]
 267
 268         generated_pages[pagename] = wikidata
 269
 270         # Make a redirect from the name to the ID too, unless there's
 271         # already an existing page with the name and it isn't a redirect.
 272         noclobber = False
 273         apppagename = app.Name.replace('_', ' ')
 274         apppagename = apppagename.replace('{', '')
 275         apppagename = apppagename.replace('}', ' ')
 276         apppagename = apppagename.replace(':', ' ')
 277         apppagename = apppagename.replace('[', ' ')
 278         apppagename = apppagename.replace(']', ' ')
 279         # Drop double spaces caused mostly by replacing ':' above
 280         apppagename = apppagename.replace('  ', ' ')
 281         for expagename in site.allpages(prefix=apppagename,
 282                                         filterredir='nonredirects',
 283                                         generator=False):
 284             if expagename == apppagename:
 285                 noclobber = True
 286         # Another reason not to make the redirect page is if the app name
 287         # is the same as it's ID, because that will overwrite the real page
 288         # with an redirect to itself! (Although it seems like an odd
 289         # scenario this happens a lot, e.g. where there is metadata but no
 290         # builds or binaries to extract a name from.
 291         if apppagename == pagename:
 292             noclobber = True
 293         if not noclobber:
 294             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 295
 296     for tcat, genp in [(wikicat, generated_pages),
 297                        (wikiredircat, generated_redirects)]:
 298         catpages = site.Pages['Category:' + tcat]
 299         existingpages = []
 300         for page in catpages:
 301             existingpages.append(page.name)
 302             if page.name in genp:
 303                 pagetxt = page.edit()
 304                 if pagetxt != genp[page.name]:
 305                     logging.debug("Updating modified page " + page.name)
 306                     page.save(genp[page.name], summary='Auto-updated')
 307                 else:
 308                     logging.debug("Page " + page.name + " is unchanged")
 309             else:
 310                 logging.warn("Deleting page " + page.name)
 311                 page.delete('No longer published')
 312         for pagename, text in genp.items():
 313             logging.debug("Checking " + pagename)
 314             if pagename not in existingpages:
 315                 logging.debug("Creating page " + pagename)
 316                 try:
 317                     newpage = site.Pages[pagename]
 318                     newpage.save(text, summary='Auto-created')
 319                 except Exception as e:
 320                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 321
 322     # Purge server cache to ensure counts are up to date
 323     site.Pages['Repository Maintenance'].purge()
 324
 325     # Write a page with the last build log for this version code
 326     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 327     newpage = site.Pages[wiki_page_path]
 328     txt = ''
 329     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 330     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 331     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 332     txt += common.get_git_describe_link()
 333     txt += "\n\n"
 334     txt += common.get_android_tools_version_log()
 335     newpage.save(txt, summary='Run log')
 336     newpage = site.Pages['update']
 337     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 338
 339
 340 def delete_disabled_builds(apps, apkcache, repodirs):
 341     """Delete disabled build outputs.
 342
 343     :param apps: list of all applications, as per metadata.read_metadata
 344     :param apkcache: current apk cache information
 345     :param repodirs: the repo directories to process
 346     """
 347     for appid, app in apps.items():
 348         for build in app['builds']:
 349             if not build.disable:
 350                 continue
 351             apkfilename = common.get_release_filename(app, build)
 352             iconfilename = "%s.%s.png" % (
 353                 appid,
 354                 build.versionCode)
 355             for repodir in repodirs:
 356                 files = [
 357                     os.path.join(repodir, apkfilename),
 358                     os.path.join(repodir, apkfilename + '.asc'),
 359                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 360                 ]
 361                 for density in all_screen_densities:
 362                     repo_dir = get_icon_dir(repodir, density)
 363                     files.append(os.path.join(repo_dir, iconfilename))
 364
 365                 for f in files:
 366                     if os.path.exists(f):
 367                         logging.info("Deleting disabled build output " + f)
 368                         os.remove(f)
 369             if apkfilename in apkcache:
 370                 del apkcache[apkfilename]
 371
 372
 373 def resize_icon(iconpath, density):
 374
 375     if not os.path.isfile(iconpath):
 376         return
 377
 378     fp = None
 379     try:
 380         fp = open(iconpath, 'rb')
 381         im = Image.open(fp)
 382         size = dpi_to_px(density)
 383
 384         if any(length > size for length in im.size):
 385             oldsize = im.size
 386             im.thumbnail((size, size), Image.ANTIALIAS)
 387             logging.debug("%s was too large at %s - new size is %s" % (
 388                 iconpath, oldsize, im.size))
 389             im.save(iconpath, "PNG", optimize=True,
 390                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 391
 392     except Exception as e:
 393         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 394
 395     finally:
 396         if fp:
 397             fp.close()
 398
 399
 400 def resize_all_icons(repodirs):
 401     """Resize all icons that exceed the max size
 402
 403     :param repodirs: the repo directories to process
 404     """
 405     for repodir in repodirs:
 406         for density in screen_densities:
 407             icon_dir = get_icon_dir(repodir, density)
 408             icon_glob = os.path.join(icon_dir, '*.png')
 409             for iconpath in glob.glob(icon_glob):
 410                 resize_icon(iconpath, density)
 411
 412
 413 def getsig(apkpath):
 414     """ Get the signing certificate of an apk. To get the same md5 has that
 415     Android gets, we encode the .RSA certificate in a specific format and pass
 416     it hex-encoded to the md5 digest algorithm.
 417
 418     :param apkpath: path to the apk
 419     :returns: A string containing the md5 of the signature of the apk or None
 420               if an error occurred.
 421     """
 422
 423     with zipfile.ZipFile(apkpath, 'r') as apk:
 424         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 425
 426         if len(certs) < 1:
 427             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 428             return None
 429         if len(certs) > 1:
 430             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 431             return None
 432
 433         cert = apk.read(certs[0])
 434
 435     cert_encoded = common.get_certificate(cert)
 436
 437     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 438
 439
 440 def get_cache_file():
 441     return os.path.join('tmp', 'apkcache')
 442
 443
 444 def get_cache():
 445     """Get the cached dict of the APK index
 446
 447     Gather information about all the apk files in the repo directory,
 448     using cached data if possible. Some of the index operations take a
 449     long time, like calculating the SHA-256 and verifying the APK
 450     signature.
 451
 452     The cache is invalidated if the metadata version is different, or
 453     the 'allow_disabled_algorithms' config/option is different.  In
 454     those cases, there is no easy way to know what has changed from
 455     the cache, so just rerun the whole thing.
 456
 457     :return: apkcache
 458
 459     """
 460     apkcachefile = get_cache_file()
 461     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 462     if not options.clean and os.path.exists(apkcachefile):
 463         with open(apkcachefile, 'rb') as cf:
 464             apkcache = pickle.load(cf, encoding='utf-8')
 465         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 466            or apkcache.get('allow_disabled_algorithms') != ada:
 467             apkcache = {}
 468     else:
 469         apkcache = {}
 470
 471     apkcache["METADATA_VERSION"] = METADATA_VERSION
 472     apkcache['allow_disabled_algorithms'] = ada
 473
 474     return apkcache
 475
 476
 477 def write_cache(apkcache):
 478     apkcachefile = get_cache_file()
 479     cache_path = os.path.dirname(apkcachefile)
 480     if not os.path.exists(cache_path):
 481         os.makedirs(cache_path)
 482     with open(apkcachefile, 'wb') as cf:
 483         pickle.dump(apkcache, cf)
 484
 485
 486 def get_icon_bytes(apkzip, iconsrc):
 487     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 488     try:
 489         return apkzip.read(iconsrc)
 490     except KeyError:
 491         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 492
 493
 494 def sha256sum(filename):
 495     '''Calculate the sha256 of the given file'''
 496     sha = hashlib.sha256()
 497     with open(filename, 'rb') as f:
 498         while True:
 499             t = f.read(16384)
 500             if len(t) == 0:
 501                 break
 502             sha.update(t)
 503     return sha.hexdigest()
 504
 505
 506 def has_known_vulnerability(filename):
 507     """checks for known vulnerabilities in the APK
 508
 509     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 510     version.  Google also enforces this:
 511     https://support.google.com/faqs/answer/6376725?hl=en
 512
 513     Checks whether there are more than one classes.dex or AndroidManifest.xml
 514     files, which is invalid and an essential part of the "Master Key" attack.
 515     http://www.saurik.com/id/17
 516
 517     Janus is similar to Master Key but is perhaps easier to scan for.
 518     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 519     """
 520
 521     found_vuln = False
 522
 523     # statically load this pattern
 524     if not hasattr(has_known_vulnerability, "pattern"):
 525         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 526
 527     with open(filename.encode(), 'rb') as fp:
 528         first4 = fp.read(4)
 529     if first4 != b'\x50\x4b\x03\x04':
 530         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 531                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 532                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 533
 534     files_in_apk = set()
 535     with zipfile.ZipFile(filename) as zf:
 536         for name in zf.namelist():
 537             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 538                 lib = zf.open(name)
 539                 while True:
 540                     chunk = lib.read(4096)
 541                     if chunk == b'':
 542                         break
 543                     m = has_known_vulnerability.pattern.search(chunk)
 544                     if m:
 545                         version = m.group(1).decode('ascii')
 546                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 547                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 548                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 549                             logging.debug(_('"{path}" contains recent {name} ({version})')
 550                                           .format(path=filename, name=name, version=version))
 551                         else:
 552                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 553                                             .format(path=filename, name=name, version=version))
 554                             found_vuln = True
 555                         break
 556             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 557                 if name in files_in_apk:
 558                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 559                                     .format(apkfilename=filename, name=name))
 560                     found_vuln = True
 561                 files_in_apk.add(name)
 562     return found_vuln
 563
 564
 565 def insert_obbs(repodir, apps, apks):
 566     """Scans the .obb files in a given repo directory and adds them to the
 567     relevant APK instances.  OBB files have versionCodes like APK
 568     files, and they are loosely associated.  If there is an OBB file
 569     present, then any APK with the same or higher versionCode will use
 570     that OBB file.  There are two OBB types: main and patch, each APK
 571     can only have only have one of each.
 572
 573     https://developer.android.com/google/play/expansion-files.html
 574
 575     :param repodir: repo directory to scan
 576     :param apps: list of current, valid apps
 577     :param apks: current information on all APKs
 578
 579     """
 580
 581     def obbWarnDelete(f, msg):
 582         logging.warning(msg + ' ' + f)
 583         if options.delete_unknown:
 584             logging.error(_("Deleting unknown file: {path}").format(path=f))
 585             os.remove(f)
 586
 587     obbs = []
 588     java_Integer_MIN_VALUE = -pow(2, 31)
 589     currentPackageNames = apps.keys()
 590     for f in glob.glob(os.path.join(repodir, '*.obb')):
 591         obbfile = os.path.basename(f)
 592         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 593         chunks = obbfile.split('.')
 594         if chunks[0] != 'main' and chunks[0] != 'patch':
 595             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 596             continue
 597         if not re.match(r'^-?[0-9]+$', chunks[1]):
 598             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 599                           .format(name=chunks[0]))
 600             continue
 601         versionCode = int(chunks[1])
 602         packagename = ".".join(chunks[2:-1])
 603
 604         highestVersionCode = java_Integer_MIN_VALUE
 605         if packagename not in currentPackageNames:
 606             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 607             continue
 608         for apk in apks:
 609             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 610                 highestVersionCode = apk['versionCode']
 611         if versionCode > highestVersionCode:
 612             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 613                           .format(integer=str(versionCode)))
 614             continue
 615         obbsha256 = sha256sum(f)
 616         obbs.append((packagename, versionCode, obbfile, obbsha256))
 617
 618     for apk in apks:
 619         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 620             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 621                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 622                     apk['obbMainFile'] = obbfile
 623                     apk['obbMainFileSha256'] = obbsha256
 624                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 625                     apk['obbPatchFile'] = obbfile
 626                     apk['obbPatchFileSha256'] = obbsha256
 627             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 628                 break
 629
 630
 631 def translate_per_build_anti_features(apps, apks):
 632     """Grab the anti-features list from the build metadata
 633
 634     For most Anti-Features, they are really most applicable per-APK,
 635     not for an app.  An app can fix a vulnerability, add/remove
 636     tracking, etc.  This reads the 'antifeatures' list from the Build
 637     entries in the fdroiddata metadata file, then transforms it into
 638     the 'antiFeatures' list of unique items for the index.
 639
 640     The field key is all lower case in the metadata file to match the
 641     rest of the Build fields.  It is 'antiFeatures' camel case in the
 642     implementation, index, and fdroidclient since it is translated
 643     from the build 'antifeatures' field, not directly included.
 644
 645     """
 646
 647     antiFeatures = dict()
 648     for packageName, app in apps.items():
 649         d = dict()
 650         for build in app['builds']:
 651             afl = build.get('antifeatures')
 652             if afl:
 653                 d[int(build.versionCode)] = afl
 654         if len(d) > 0:
 655             antiFeatures[packageName] = d
 656
 657     for apk in apks:
 658         d = antiFeatures.get(apk['packageName'])
 659         if d:
 660             afl = d.get(apk['versionCode'])
 661             if afl:
 662                 apk['antiFeatures'].update(afl)
 663
 664
 665 def _get_localized_dict(app, locale):
 666     '''get the dict to add localized store metadata to'''
 667     if 'localized' not in app:
 668         app['localized'] = collections.OrderedDict()
 669     if locale not in app['localized']:
 670         app['localized'][locale] = collections.OrderedDict()
 671     return app['localized'][locale]
 672
 673
 674 def _set_localized_text_entry(app, locale, key, f):
 675     limit = config['char_limits'][key]
 676     localized = _get_localized_dict(app, locale)
 677     with open(f) as fp:
 678         text = fp.read()[:limit]
 679         if len(text) > 0:
 680             localized[key] = text
 681
 682
 683 def _set_author_entry(app, key, f):
 684     limit = config['char_limits']['author']
 685     with open(f) as fp:
 686         text = fp.read()[:limit]
 687         if len(text) > 0:
 688             app[key] = text
 689
 690
 691 def _strip_and_copy_image(inpath, outpath):
 692     """Remove any metadata from image and copy it to new path
 693
 694     Sadly, image metadata like EXIF can be used to exploit devices.
 695     It is not used at all in the F-Droid ecosystem, so its much safer
 696     just to remove it entirely.
 697
 698     """
 699
 700     extension = common.get_extension(inpath)[1]
 701     if os.path.isdir(outpath):
 702         outpath = os.path.join(outpath, os.path.basename(inpath))
 703     if extension == 'png':
 704         with open(inpath, 'rb') as fp:
 705             in_image = Image.open(fp)
 706             in_image.save(outpath, "PNG", optimize=True,
 707                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 708     elif extension == 'jpg' or extension == 'jpeg':
 709         with open(inpath, 'rb') as fp:
 710             in_image = Image.open(fp)
 711             data = list(in_image.getdata())
 712             out_image = Image.new(in_image.mode, in_image.size)
 713         out_image.putdata(data)
 714         out_image.save(outpath, "JPEG", optimize=True)
 715     else:
 716         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 717                               .format(extension=extension))
 718
 719
 720 def copy_triple_t_store_metadata(apps):
 721     """Include store metadata from the app's source repo
 722
 723     The Triple-T Gradle Play Publisher is a plugin that has a standard
 724     file layout for all of the metadata and graphics that the Google
 725     Play Store accepts.  Since F-Droid has the git repo, it can just
 726     pluck those files directly.  This method reads any text files into
 727     the app dict, then copies any graphics into the fdroid repo
 728     directory structure.
 729
 730     This needs to be run before insert_localized_app_metadata() so that
 731     the graphics files that are copied into the fdroid repo get
 732     properly indexed.
 733
 734     https://github.com/Triple-T/gradle-play-publisher#upload-images
 735     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 736
 737     """
 738
 739     if not os.path.isdir('build'):
 740         return  # nothing to do
 741
 742     for packageName, app in apps.items():
 743         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 744             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 745             for root, dirs, files in os.walk(d):
 746                 segments = root.split('/')
 747                 locale = segments[-2]
 748                 for f in files:
 749                     if f == 'fulldescription':
 750                         _set_localized_text_entry(app, locale, 'description',
 751                                                   os.path.join(root, f))
 752                         continue
 753                     elif f == 'shortdescription':
 754                         _set_localized_text_entry(app, locale, 'summary',
 755                                                   os.path.join(root, f))
 756                         continue
 757                     elif f == 'title':
 758                         _set_localized_text_entry(app, locale, 'name',
 759                                                   os.path.join(root, f))
 760                         continue
 761                     elif f == 'video':
 762                         _set_localized_text_entry(app, locale, 'video',
 763                                                   os.path.join(root, f))
 764                         continue
 765                     elif f == 'whatsnew':
 766                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 767                                                   os.path.join(root, f))
 768                         continue
 769                     elif f == 'contactEmail':
 770                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 771                         continue
 772                     elif f == 'contactPhone':
 773                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 774                         continue
 775                     elif f == 'contactWebsite':
 776                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 777                         continue
 778
 779                     base, extension = common.get_extension(f)
 780                     dirname = os.path.basename(root)
 781                     if extension in ALLOWED_EXTENSIONS \
 782                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 783                         if segments[-2] == 'listing':
 784                             locale = segments[-3]
 785                         else:
 786                             locale = segments[-2]
 787                         destdir = os.path.join('repo', packageName, locale, dirname)
 788                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 789                         sourcefile = os.path.join(root, f)
 790                         destfile = os.path.join(destdir, os.path.basename(f))
 791                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 792                         _strip_and_copy_image(sourcefile, destfile)
 793
 794
 795 def insert_localized_app_metadata(apps):
 796     """scans standard locations for graphics and localized text
 797
 798     Scans for localized description files, store graphics, and
 799     screenshot PNG files in statically defined screenshots directory
 800     and adds them to the app metadata.  The screenshots and graphic
 801     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 802     and must be in the following layout:
 803     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 804
 805     repo/packageName/locale/featureGraphic.png
 806     repo/packageName/locale/phoneScreenshots/1.png
 807     repo/packageName/locale/phoneScreenshots/2.png
 808
 809     The changelog files must be text files named with the versionCode
 810     ending with ".txt" and must be in the following layout:
 811     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 812
 813     repo/packageName/locale/changelogs/12345.txt
 814
 815     This will scan the each app's source repo then the metadata/ dir
 816     for these standard locations of changelog files.  If it finds
 817     them, they will be added to the dict of all packages, with the
 818     versions in the metadata/ folder taking precendence over the what
 819     is in the app's source repo.
 820
 821     Where "packageName" is the app's packageName and "locale" is the locale
 822     of the graphics, e.g. what language they are in, using the IETF RFC5646
 823     format (en-US, fr-CA, es-MX, etc).
 824
 825     This will also scan the app's git for a fastlane folder, and the
 826     metadata/ folder and the apps' source repos for standard locations
 827     of graphic and screenshot files.  If it finds them, it will copy
 828     them into the repo.  The fastlane files follow this pattern:
 829     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 830
 831     """
 832
 833     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 834     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 837
 838     for srcd in sorted(sourcedirs):
 839         if not os.path.isdir(srcd):
 840             continue
 841         for root, dirs, files in os.walk(srcd):
 842             segments = root.split('/')
 843             packageName = segments[1]
 844             if packageName not in apps:
 845                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 846                 continue
 847             locale = segments[-1]
 848             destdir = os.path.join('repo', packageName, locale)
 849
 850             # flavours specified in build receipt
 851             build_flavours = ""
 852             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 853                     and 'gradle' in apps[packageName].builds[-1]:
 854                 build_flavours = apps[packageName].builds[-1].gradle
 855
 856             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 857                 logging.debug("ignoring due to wrong flavour")
 858                 continue
 859
 860             for f in files:
 861                 if f in ('description.txt', 'full_description.txt'):
 862                     _set_localized_text_entry(apps[packageName], locale, 'description',
 863                                               os.path.join(root, f))
 864                     continue
 865                 elif f in ('summary.txt', 'short_description.txt'):
 866                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 867                                               os.path.join(root, f))
 868                     continue
 869                 elif f in ('name.txt', 'title.txt'):
 870                     _set_localized_text_entry(apps[packageName], locale, 'name',
 871                                               os.path.join(root, f))
 872                     continue
 873                 elif f == 'video.txt':
 874                     _set_localized_text_entry(apps[packageName], locale, 'video',
 875                                               os.path.join(root, f))
 876                     continue
 877                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 878                     locale = segments[-2]
 879                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 880                                               os.path.join(root, f))
 881                     continue
 882
 883                 base, extension = common.get_extension(f)
 884                 if locale == 'images':
 885                     locale = segments[-2]
 886                     destdir = os.path.join('repo', packageName, locale)
 887                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 888                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 889                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 890                     _strip_and_copy_image(os.path.join(root, f), destdir)
 891             for d in dirs:
 892                 if d in SCREENSHOT_DIRS:
 893                     if locale == 'images':
 894                         locale = segments[-2]
 895                         destdir = os.path.join('repo', packageName, locale)
 896                     for f in glob.glob(os.path.join(root, d, '*.*')):
 897                         _ignored, extension = common.get_extension(f)
 898                         if extension in ALLOWED_EXTENSIONS:
 899                             screenshotdestdir = os.path.join(destdir, d)
 900                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 901                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 902                             _strip_and_copy_image(f, screenshotdestdir)
 903
 904     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 905     for d in repofiles:
 906         if not os.path.isdir(d):
 907             continue
 908         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 909             if not os.path.isfile(f):
 910                 continue
 911             segments = f.split('/')
 912             packageName = segments[1]
 913             locale = segments[2]
 914             screenshotdir = segments[3]
 915             filename = os.path.basename(f)
 916             base, extension = common.get_extension(filename)
 917
 918             if packageName not in apps:
 919                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 920                                 .format(path=filename, name=packageName))
 921                 continue
 922             graphics = _get_localized_dict(apps[packageName], locale)
 923
 924             if extension not in ALLOWED_EXTENSIONS:
 925                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 926             elif base in GRAPHIC_NAMES:
 927                 # there can only be zero or one of these per locale
 928                 graphics[base] = filename
 929             elif screenshotdir in SCREENSHOT_DIRS:
 930                 # there can any number of these per locale
 931                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 932                 if screenshotdir not in graphics:
 933                     graphics[screenshotdir] = []
 934                 graphics[screenshotdir].append(filename)
 935             else:
 936                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 937
 938
 939 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 940     """Scan a repo for all files with an extension except APK/OBB
 941
 942     :param apkcache: current cached info about all repo files
 943     :param repodir: repo directory to scan
 944     :param knownapks: list of all known files, as per metadata.read_metadata
 945     :param use_date_from_file: use date from file (instead of current date)
 946                                for newly added files
 947     """
 948
 949     cachechanged = False
 950     repo_files = []
 951     repodir = repodir.encode('utf-8')
 952     for name in os.listdir(repodir):
 953         file_extension = common.get_file_extension(name)
 954         if file_extension == 'apk' or file_extension == 'obb':
 955             continue
 956         filename = os.path.join(repodir, name)
 957         name_utf8 = name.decode('utf-8')
 958         if filename.endswith(b'_src.tar.gz'):
 959             logging.debug(_('skipping source tarball: {path}')
 960                           .format(path=filename.decode('utf-8')))
 961             continue
 962         if not common.is_repo_file(filename):
 963             continue
 964         stat = os.stat(filename)
 965         if stat.st_size == 0:
 966             raise FDroidException(_('{path} is zero size!')
 967                                   .format(path=filename))
 968
 969         shasum = sha256sum(filename)
 970         usecache = False
 971         if name in apkcache:
 972             repo_file = apkcache[name]
 973             # added time is cached as tuple but used here as datetime instance
 974             if 'added' in repo_file:
 975                 a = repo_file['added']
 976                 if isinstance(a, datetime):
 977                     repo_file['added'] = a
 978                 else:
 979                     repo_file['added'] = datetime(*a[:6])
 980             if repo_file.get('hash') == shasum:
 981                 logging.debug(_("Reading {apkfilename} from cache")
 982                               .format(apkfilename=name_utf8))
 983                 usecache = True
 984             else:
 985                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 986                               .format(apkfilename=name_utf8))
 987
 988         if not usecache:
 989             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 990             repo_file = collections.OrderedDict()
 991             repo_file['name'] = os.path.splitext(name_utf8)[0]
 992             # TODO rename apkname globally to something more generic
 993             repo_file['apkName'] = name_utf8
 994             repo_file['hash'] = shasum
 995             repo_file['hashType'] = 'sha256'
 996             repo_file['versionCode'] = 0
 997             repo_file['versionName'] = shasum
 998             # the static ID is the SHA256 unless it is set in the metadata
 999             repo_file['packageName'] = shasum
1000
1001             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1002             if m:
1003                 repo_file['packageName'] = m.group(1)
1004                 repo_file['versionCode'] = int(m.group(2))
1005             srcfilename = name + b'_src.tar.gz'
1006             if os.path.exists(os.path.join(repodir, srcfilename)):
1007                 repo_file['srcname'] = srcfilename.decode('utf-8')
1008             repo_file['size'] = stat.st_size
1009
1010             apkcache[name] = repo_file
1011             cachechanged = True
1012
1013         if use_date_from_file:
1014             timestamp = stat.st_ctime
1015             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1016         else:
1017             default_date_param = None
1018
1019         # Record in knownapks, getting the added date at the same time..
1020         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1021                                     default_date=default_date_param)
1022         if added:
1023             repo_file['added'] = added
1024
1025         repo_files.append(repo_file)
1026
1027     return repo_files, cachechanged
1028
1029
1030 def scan_apk(apk_file):
1031     """
1032     Scans an APK file and returns dictionary with metadata of the APK.
1033
1034     Attention: This does *not* verify that the APK signature is correct.
1035
1036     :param apk_file: The (ideally absolute) path to the APK file
1037     :raises BuildException
1038     :return A dict containing APK metadata
1039     """
1040     apk = {
1041         'hash': sha256sum(apk_file),
1042         'hashType': 'sha256',
1043         'uses-permission': [],
1044         'uses-permission-sdk-23': [],
1045         'features': [],
1046         'icons_src': {},
1047         'icons': {},
1048         'antiFeatures': set(),
1049     }
1050
1051     if common.use_androguard():
1052         scan_apk_androguard(apk, apk_file)
1053     else:
1054         scan_apk_aapt(apk, apk_file)
1055
1056     # Get the signature, or rather the signing key fingerprints
1057     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1058     apk['sig'] = getsig(apk_file)
1059     if not apk['sig']:
1060         raise BuildException("Failed to get apk signature")
1061     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1062                                                                apk_file))
1063     if not apk.get('signer'):
1064         raise BuildException("Failed to get apk signing key fingerprint")
1065
1066     # Get size of the APK
1067     apk['size'] = os.path.getsize(apk_file)
1068
1069     if 'minSdkVersion' not in apk:
1070         logging.warning("No SDK version information found in {0}".format(apk_file))
1071         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
1072     if 'targetSdkVersion' not in apk:
1073         apk['targetSdkVersion'] = apk['minSdkVersion']
1074
1075     # Check for known vulnerabilities
1076     if has_known_vulnerability(apk_file):
1077         apk['antiFeatures'].add('KnownVuln')
1078
1079     return apk
1080
1081
1082 def _get_apk_icons_src(apkfile, icon_name):
1083     """Extract the paths to the app icon in all available densities
1084
1085     """
1086     icons_src = dict()
1087     density_re = re.compile('^res/(.*)/' + icon_name + '\.(png|xml)$')
1088     with zipfile.ZipFile(apkfile) as zf:
1089         for filename in zf.namelist():
1090             m = density_re.match(filename)
1091             if m:
1092                 folder = m.group(1).split('-')
1093                 if len(folder) > 1:
1094                     density = screen_resolutions[folder[1]]
1095                 else:
1096                     density = '160'
1097                 icons_src[density] = m.group(0)
1098     if icons_src.get('-1') is None:
1099         icons_src['-1'] = icons_src['160']
1100     return icons_src
1101
1102
1103 def scan_apk_aapt(apk, apkfile):
1104     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1105     if p.returncode != 0:
1106         if options.delete_unknown:
1107             if os.path.exists(apkfile):
1108                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1109                 os.remove(apkfile)
1110             else:
1111                 logging.error("Could not find {0} to remove it".format(apkfile))
1112         else:
1113             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1114         raise BuildException(_("Invalid APK"))
1115     icon_name = None
1116     for line in p.output.splitlines():
1117         if line.startswith("package:"):
1118             try:
1119                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1120                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1121                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1122             except Exception as e:
1123                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1124         elif line.startswith("application:"):
1125             m = re.match(APK_LABEL_ICON_PAT, line)
1126             if m:
1127                 apk['name'] = m.group(1)
1128                 icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
1129         elif not apk.get('name') and line.startswith("launchable-activity:"):
1130             # Only use launchable-activity as fallback to application
1131             apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)
1132         elif line.startswith("sdkVersion:"):
1133             m = re.match(APK_SDK_VERSION_PAT, line)
1134             if m is None:
1135                 logging.error(line.replace('sdkVersion:', '')
1136                               + ' is not a valid minSdkVersion!')
1137             else:
1138                 apk['minSdkVersion'] = m.group(1)
1139         elif line.startswith("targetSdkVersion:"):
1140             m = re.match(APK_SDK_VERSION_PAT, line)
1141             if m is None:
1142                 logging.error(line.replace('targetSdkVersion:', '')
1143                               + ' is not a valid targetSdkVersion!')
1144             else:
1145                 apk['targetSdkVersion'] = m.group(1)
1146         elif line.startswith("maxSdkVersion:"):
1147             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1148         elif line.startswith("native-code:"):
1149             apk['nativecode'] = []
1150             for arch in line[13:].split(' '):
1151                 apk['nativecode'].append(arch[1:-1])
1152         elif line.startswith('uses-permission:'):
1153             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1154             if perm_match['maxSdkVersion']:
1155                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1156             permission = UsesPermission(
1157                 perm_match['name'],
1158                 perm_match['maxSdkVersion']
1159             )
1160
1161             apk['uses-permission'].append(permission)
1162         elif line.startswith('uses-permission-sdk-23:'):
1163             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1164             if perm_match['maxSdkVersion']:
1165                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1166             permission_sdk_23 = UsesPermissionSdk23(
1167                 perm_match['name'],
1168                 perm_match['maxSdkVersion']
1169             )
1170
1171             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1172
1173         elif line.startswith('uses-feature:'):
1174             feature = re.match(APK_FEATURE_PAT, line).group(1)
1175             # Filter out this, it's only added with the latest SDK tools and
1176             # causes problems for lots of apps.
1177             if feature != "android.hardware.screen.portrait" \
1178                     and feature != "android.hardware.screen.landscape":
1179                 if feature.startswith("android.feature."):
1180                     feature = feature[16:]
1181                 apk['features'].add(feature)
1182     apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1183
1184
1185 def scan_apk_androguard(apk, apkfile):
1186     try:
1187         from androguard.core.bytecodes.apk import APK
1188         apkobject = APK(apkfile)
1189         if apkobject.is_valid_APK():
1190             arsc = apkobject.get_android_resources()
1191         else:
1192             if options.delete_unknown:
1193                 if os.path.exists(apkfile):
1194                     logging.error(_("Failed to get apk information, deleting {path}")
1195                                   .format(path=apkfile))
1196                     os.remove(apkfile)
1197                 else:
1198                     logging.error(_("Could not find {path} to remove it")
1199                                   .format(path=apkfile))
1200             else:
1201                 logging.error(_("Failed to get apk information, skipping {path}")
1202                               .format(path=apkfile))
1203             raise BuildException(_("Invalid APK"))
1204     except ImportError:
1205         raise FDroidException("androguard library is not installed and aapt not present")
1206     except FileNotFoundError:
1207         logging.error(_("Could not open apk file for analysis"))
1208         raise BuildException(_("Invalid APK"))
1209
1210     apk['packageName'] = apkobject.get_package()
1211     apk['versionCode'] = int(apkobject.get_androidversion_code())
1212     apk['versionName'] = apkobject.get_androidversion_name()
1213     if apk['versionName'][0] == "@":
1214         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1215         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1216         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1217     apk['name'] = apkobject.get_app_name()
1218
1219     if apkobject.get_max_sdk_version() is not None:
1220         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1221     if apkobject.get_min_sdk_version() is not None:
1222         apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1223     if apkobject.get_target_sdk_version() is not None:
1224         apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1225
1226     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1227     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1228     apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1229
1230     arch_re = re.compile("^lib/(.*)/.*$")
1231     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1232     if len(arch) >= 1:
1233         apk['nativecode'] = []
1234         apk['nativecode'].extend(sorted(list(arch)))
1235
1236     xml = apkobject.get_android_manifest_xml()
1237
1238     for item in xml.findall('uses-permission'):
1239         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1240         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1241         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1242         permission = UsesPermission(
1243             name,
1244             maxSdkVersion
1245         )
1246         apk['uses-permission'].append(permission)
1247     for name, maxSdkVersion in apkobject.get_uses_implied_permission_list():
1248         permission = UsesPermission(
1249             name,
1250             maxSdkVersion
1251         )
1252         apk['uses-permission'].append(permission)
1253
1254     for item in xml.findall('uses-permission-sdk-23'):
1255         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1256         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1257         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1258         permission_sdk_23 = UsesPermissionSdk23(
1259             name,
1260             maxSdkVersion
1261         )
1262         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1263
1264     for item in xml.findall('uses-feature'):
1265         feature = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1266         if feature != "android.hardware.screen.portrait" \
1267                 and feature != "android.hardware.screen.landscape":
1268             if feature.startswith("android.feature."):
1269                 feature = feature[16:]
1270         required = item.attrib.get('{' + xml.nsmap['android'] + '}required')
1271         if required is None or required == 'true':
1272             apk['features'].append(feature)
1273
1274
1275 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1276                 allow_disabled_algorithms=False, archive_bad_sig=False):
1277     """Processes the apk with the given filename in the given repo directory.
1278
1279     This also extracts the icons.
1280
1281     :param apkcache: current apk cache information
1282     :param apkfilename: the filename of the apk to scan
1283     :param repodir: repo directory to scan
1284     :param knownapks: known apks info
1285     :param use_date_from_apk: use date from APK (instead of current date)
1286                               for newly added APKs
1287     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1288                                       disabled algorithms in the signature (e.g. MD5)
1289     :param archive_bad_sig: move APKs with a bad signature to the archive
1290     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1291      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1292     """
1293
1294     apk = {}
1295     apkfile = os.path.join(repodir, apkfilename)
1296
1297     cachechanged = False
1298     usecache = False
1299     if apkfilename in apkcache:
1300         apk = apkcache[apkfilename]
1301         if apk.get('hash') == sha256sum(apkfile):
1302             logging.debug(_("Reading {apkfilename} from cache")
1303                           .format(apkfilename=apkfilename))
1304             usecache = True
1305         else:
1306             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1307                           .format(apkfilename=apkfilename))
1308
1309     if not usecache:
1310         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1311
1312         try:
1313             apk = scan_apk(apkfile)
1314         except BuildException:
1315             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1316                             .format(apkfilename=apkfilename))
1317             return True, None, False
1318
1319         # Check for debuggable apks...
1320         if common.is_apk_and_debuggable(apkfile):
1321             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1322
1323         if options.rename_apks:
1324             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1325             std_short_name = os.path.join(repodir, n)
1326             if apkfile != std_short_name:
1327                 if os.path.exists(std_short_name):
1328                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1329                     if apkfile != std_long_name:
1330                         if os.path.exists(std_long_name):
1331                             dupdir = os.path.join('duplicates', repodir)
1332                             if not os.path.isdir(dupdir):
1333                                 os.makedirs(dupdir, exist_ok=True)
1334                             dupfile = os.path.join('duplicates', std_long_name)
1335                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1336                             os.rename(apkfile, dupfile)
1337                             return True, None, False
1338                         else:
1339                             os.rename(apkfile, std_long_name)
1340                     apkfile = std_long_name
1341                 else:
1342                     os.rename(apkfile, std_short_name)
1343                     apkfile = std_short_name
1344                 apkfilename = apkfile[len(repodir) + 1:]
1345
1346         apk['apkName'] = apkfilename
1347         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1348         if os.path.exists(os.path.join(repodir, srcfilename)):
1349             apk['srcname'] = srcfilename
1350
1351         # verify the jar signature is correct, allow deprecated
1352         # algorithms only if the APK is in the archive.
1353         skipapk = False
1354         if not common.verify_apk_signature(apkfile):
1355             if repodir == 'archive' or allow_disabled_algorithms:
1356                 if common.verify_old_apk_signature(apkfile):
1357                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1358                 else:
1359                     skipapk = True
1360             else:
1361                 skipapk = True
1362
1363         if skipapk:
1364             if archive_bad_sig:
1365                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1366                                 .format(apkfilename=apkfilename))
1367                 move_apk_between_sections(repodir, 'archive', apk)
1368             else:
1369                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1370                                 .format(apkfilename=apkfilename))
1371             return True, None, False
1372
1373         apkzip = zipfile.ZipFile(apkfile, 'r')
1374
1375         manifest = apkzip.getinfo('AndroidManifest.xml')
1376         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1377         if (1980, 0, 0) != manifest.date_time[0:3]:
1378             try:
1379                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1380             except ValueError as e:
1381                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1382                                 .format(apkfilename=apkfile) + str(e))
1383
1384         # extract icons from APK zip file
1385         iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
1386         try:
1387             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1388         finally:
1389             apkzip.close()  # ensure that APK zip file gets closed
1390
1391         # resize existing icons for densities missing in the APK
1392         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1393
1394         if use_date_from_apk and manifest.date_time[1] != 0:
1395             default_date_param = datetime(*manifest.date_time)
1396         else:
1397             default_date_param = None
1398
1399         # Record in known apks, getting the added date at the same time..
1400         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1401                                     default_date=default_date_param)
1402         if added:
1403             apk['added'] = added
1404
1405         apkcache[apkfilename] = apk
1406         cachechanged = True
1407
1408     return False, apk, cachechanged
1409
1410
1411 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1412     """Processes the apks in the given repo directory.
1413
1414     This also extracts the icons.
1415
1416     :param apkcache: current apk cache information
1417     :param repodir: repo directory to scan
1418     :param knownapks: known apks info
1419     :param use_date_from_apk: use date from APK (instead of current date)
1420                               for newly added APKs
1421     :returns: (apks, cachechanged) where apks is a list of apk information,
1422               and cachechanged is True if the apkcache got changed.
1423     """
1424
1425     cachechanged = False
1426
1427     for icon_dir in get_all_icon_dirs(repodir):
1428         if os.path.exists(icon_dir):
1429             if options.clean:
1430                 shutil.rmtree(icon_dir)
1431                 os.makedirs(icon_dir)
1432         else:
1433             os.makedirs(icon_dir)
1434
1435     apks = []
1436     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1437         apkfilename = apkfile[len(repodir) + 1:]
1438         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1439         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1440                                              use_date_from_apk, ada, True)
1441         if skip:
1442             continue
1443         apks.append(apk)
1444         cachechanged = cachechanged or cachethis
1445
1446     return apks, cachechanged
1447
1448
1449 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1450     """Extracts PNG icons from an APK with the supported pixel densities
1451
1452     Extracts icons from the given APK zip in various densities, saves
1453     them into given repo directory and stores their names in the APK
1454     metadata dictionary.  If the icon is an XML icon, then this tries
1455     to find PNG icon that can replace it.
1456
1457     :param icon_filename: A string representing the icon's file name
1458     :param apk: A populated dictionary containing APK metadata.
1459                 Needs to have 'icons_src' key
1460     :param apkzip: An opened zipfile.ZipFile of the APK file
1461     :param repo_dir: The directory of the APK's repository
1462     :return: A list of icon densities that are missing
1463
1464     """
1465     res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1466     pngs = dict()
1467     for f in apkzip.namelist():
1468         m = res_name_re.match(f)
1469         if m and m.group(4) == 'png':
1470             density = screen_resolutions[m.group(2)]
1471             pngs[m.group(3) + '/' + density] = m.group(0)
1472
1473     icon_type = None
1474     empty_densities = []
1475     for density in screen_densities:
1476         if density not in apk['icons_src']:
1477             empty_densities.append(density)
1478             continue
1479         icon_src = apk['icons_src'][density]
1480         icon_dir = get_icon_dir(repo_dir, density)
1481         icon_type = '.png'
1482
1483         # Extract the icon files per density
1484         if icon_src.endswith('.xml'):
1485             m = res_name_re.match(icon_src)
1486             if m:
1487                 name = pngs.get(m.group(3) + '/' + str(density))
1488                 if name:
1489                     icon_src = name
1490             if icon_src.endswith('.xml'):
1491                 empty_densities.append(density)
1492                 icon_type = '.xml'
1493         icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
1494
1495         try:
1496             with open(icon_dest, 'wb') as f:
1497                 f.write(get_icon_bytes(apkzip, icon_src))
1498             apk['icons'][density] = icon_filename + icon_type
1499         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1500             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1501             del apk['icons_src'][density]
1502             empty_densities.append(density)
1503
1504     # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
1505     if '-1' in apk['icons_src']:
1506         icon_src = apk['icons_src']['-1']
1507         icon_type = icon_src[-4:]
1508         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
1509         with open(icon_path, 'wb') as f:
1510             f.write(get_icon_bytes(apkzip, icon_src))
1511         if icon_type == '.png':
1512             im = None
1513             try:
1514                 im = Image.open(icon_path)
1515                 dpi = px_to_dpi(im.size[0])
1516                 for density in screen_densities:
1517                     if density in apk['icons']:
1518                         break
1519                     if density == screen_densities[-1] or dpi >= int(density):
1520                         apk['icons'][density] = icon_filename + icon_type
1521                         shutil.move(icon_path,
1522                                     os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
1523                         empty_densities.remove(density)
1524                         break
1525             except Exception as e:
1526                 logging.warning(_("Failed reading {path}: {error}")
1527                                 .format(path=icon_path, error=e))
1528             finally:
1529                 if im and hasattr(im, 'close'):
1530                     im.close()
1531
1532     if apk['icons']:
1533         apk['icon'] = icon_filename + icon_type
1534
1535     return empty_densities
1536
1537
1538 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1539     """
1540     Resize existing PNG icons for densities missing in the APK to ensure all densities are available
1541
1542     :param empty_densities: A list of icon densities that are missing
1543     :param icon_filename: A string representing the icon's file name
1544     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1545     :param repo_dir: The directory of the APK's repository
1546
1547     """
1548     icon_filename += '.png'
1549     # First try resizing down to not lose quality
1550     last_density = None
1551     for density in screen_densities:
1552         if density == '65534':  # not possible to generate 'anydpi' from other densities
1553             continue
1554         if density not in empty_densities:
1555             last_density = density
1556             continue
1557         if last_density is None:
1558             continue
1559         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1560
1561         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1562         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1563         fp = None
1564         try:
1565             fp = open(last_icon_path, 'rb')
1566             im = Image.open(fp)
1567
1568             size = dpi_to_px(density)
1569
1570             im.thumbnail((size, size), Image.ANTIALIAS)
1571             im.save(icon_path, "PNG", optimize=True,
1572                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1573             empty_densities.remove(density)
1574         except Exception as e:
1575             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1576         finally:
1577             if fp:
1578                 fp.close()
1579
1580     # Then just copy from the highest resolution available
1581     last_density = None
1582     for density in reversed(screen_densities):
1583         if density not in empty_densities:
1584             last_density = density
1585             continue
1586
1587         if last_density is None:
1588             continue
1589
1590         shutil.copyfile(
1591             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1592             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1593         )
1594         empty_densities.remove(density)
1595
1596     for density in screen_densities:
1597         icon_dir = get_icon_dir(repo_dir, density)
1598         icon_dest = os.path.join(icon_dir, icon_filename)
1599         resize_icon(icon_dest, density)
1600
1601     # Copy from icons-mdpi to icons since mdpi is the baseline density
1602     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1603     if os.path.isfile(baseline):
1604         apk['icons']['0'] = icon_filename
1605         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1606
1607
1608 def apply_info_from_latest_apk(apps, apks):
1609     """
1610     Some information from the apks needs to be applied up to the application level.
1611     When doing this, we use the info from the most recent version's apk.
1612     We deal with figuring out when the app was added and last updated at the same time.
1613     """
1614     for appid, app in apps.items():
1615         bestver = UNSET_VERSION_CODE
1616         for apk in apks:
1617             if apk['packageName'] == appid:
1618                 if apk['versionCode'] > bestver:
1619                     bestver = apk['versionCode']
1620                     bestapk = apk
1621
1622                 if 'added' in apk:
1623                     if not app.added or apk['added'] < app.added:
1624                         app.added = apk['added']
1625                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1626                         app.lastUpdated = apk['added']
1627
1628         if not app.added:
1629             logging.debug("Don't know when " + appid + " was added")
1630         if not app.lastUpdated:
1631             logging.debug("Don't know when " + appid + " was last updated")
1632
1633         if bestver == UNSET_VERSION_CODE:
1634
1635             if app.Name is None:
1636                 app.Name = app.AutoName or appid
1637             app.icon = None
1638             logging.debug("Application " + appid + " has no packages")
1639         else:
1640             if app.Name is None:
1641                 app.Name = bestapk['name']
1642             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1643             if app.CurrentVersionCode is None:
1644                 app.CurrentVersionCode = str(bestver)
1645
1646
1647 def make_categories_txt(repodir, categories):
1648     '''Write a category list in the repo to allow quick access'''
1649     catdata = ''
1650     for cat in sorted(categories):
1651         catdata += cat + '\n'
1652     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1653         f.write(catdata)
1654
1655
1656 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1657
1658     def filter_apk_list_sorted(apk_list):
1659         res = []
1660         for apk in apk_list:
1661             if apk['packageName'] == appid:
1662                 res.append(apk)
1663
1664         # Sort the apk list by version code. First is highest/newest.
1665         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1666
1667     for appid, app in apps.items():
1668
1669         if app.ArchivePolicy:
1670             keepversions = int(app.ArchivePolicy[:-9])
1671         else:
1672             keepversions = defaultkeepversions
1673
1674         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1675                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1676
1677         current_app_apks = filter_apk_list_sorted(apks)
1678         if len(current_app_apks) > keepversions:
1679             # Move back the ones we don't want.
1680             for apk in current_app_apks[keepversions:]:
1681                 move_apk_between_sections(repodir, archivedir, apk)
1682                 archapks.append(apk)
1683                 apks.remove(apk)
1684
1685         current_app_archapks = filter_apk_list_sorted(archapks)
1686         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1687             kept = 0
1688             # Move forward the ones we want again, except DisableAlgorithm
1689             for apk in current_app_archapks:
1690                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1691                     move_apk_between_sections(archivedir, repodir, apk)
1692                     archapks.remove(apk)
1693                     apks.append(apk)
1694                     kept += 1
1695                 if kept == keepversions:
1696                     break
1697
1698
1699 def move_apk_between_sections(from_dir, to_dir, apk):
1700     """move an APK from repo to archive or vice versa"""
1701
1702     def _move_file(from_dir, to_dir, filename, ignore_missing):
1703         from_path = os.path.join(from_dir, filename)
1704         if ignore_missing and not os.path.exists(from_path):
1705             return
1706         to_path = os.path.join(to_dir, filename)
1707         if not os.path.exists(to_dir):
1708             os.mkdir(to_dir)
1709         shutil.move(from_path, to_path)
1710
1711     if from_dir == to_dir:
1712         return
1713
1714     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1715     _move_file(from_dir, to_dir, apk['apkName'], False)
1716     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1717     for density in all_screen_densities:
1718         from_icon_dir = get_icon_dir(from_dir, density)
1719         to_icon_dir = get_icon_dir(to_dir, density)
1720         if density not in apk.get('icons', []):
1721             continue
1722         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1723     if 'srcname' in apk:
1724         _move_file(from_dir, to_dir, apk['srcname'], False)
1725
1726
1727 def add_apks_to_per_app_repos(repodir, apks):
1728     apks_per_app = dict()
1729     for apk in apks:
1730         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1731         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1732         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1733         apks_per_app[apk['packageName']] = apk
1734
1735         if not os.path.exists(apk['per_app_icons']):
1736             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1737             os.makedirs(apk['per_app_icons'])
1738
1739         apkpath = os.path.join(repodir, apk['apkName'])
1740         shutil.copy(apkpath, apk['per_app_repo'])
1741         apksigpath = apkpath + '.sig'
1742         if os.path.exists(apksigpath):
1743             shutil.copy(apksigpath, apk['per_app_repo'])
1744         apkascpath = apkpath + '.asc'
1745         if os.path.exists(apkascpath):
1746             shutil.copy(apkascpath, apk['per_app_repo'])
1747
1748
1749 def create_metadata_from_template(apk):
1750     '''create a new metadata file using internal or external template
1751
1752     Generate warnings for apk's with no metadata (or create skeleton
1753     metadata files, if requested on the command line).  Though the
1754     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1755     since those impose things on the metadata file made from the
1756     template: field sort order, empty field value, formatting, etc.
1757     '''
1758
1759     import yaml
1760     if os.path.exists('template.yml'):
1761         with open('template.yml') as f:
1762             metatxt = f.read()
1763         if 'name' in apk and apk['name'] != '':
1764             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1765                              r'\1 ' + apk['name'],
1766                              metatxt,
1767                              flags=re.IGNORECASE | re.MULTILINE)
1768         else:
1769             logging.warning(_('{appid} does not have a name! Using package name instead.')
1770                             .format(appid=apk['packageName']))
1771             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1772                              r'\1 ' + apk['packageName'],
1773                              metatxt,
1774                              flags=re.IGNORECASE | re.MULTILINE)
1775         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1776             f.write(metatxt)
1777     else:
1778         app = dict()
1779         app['Categories'] = [os.path.basename(os.getcwd())]
1780         # include some blanks as part of the template
1781         app['AuthorName'] = ''
1782         app['Summary'] = ''
1783         app['WebSite'] = ''
1784         app['IssueTracker'] = ''
1785         app['SourceCode'] = ''
1786         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1787         if 'name' in apk and apk['name'] != '':
1788             app['Name'] = apk['name']
1789         else:
1790             logging.warning(_('{appid} does not have a name! Using package name instead.')
1791                             .format(appid=apk['packageName']))
1792             app['Name'] = apk['packageName']
1793         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1794             yaml.dump(app, f, default_flow_style=False)
1795     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1796
1797
1798 config = None
1799 options = None
1800 start_timestamp = time.gmtime()
1801
1802
1803 def main():
1804
1805     global config, options
1806
1807     # Parse command line...
1808     parser = ArgumentParser()
1809     common.setup_global_opts(parser)
1810     parser.add_argument("--create-key", action="store_true", default=False,
1811                         help=_("Add a repo signing key to an unsigned repo"))
1812     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1813                         help=_("Add skeleton metadata files for APKs that are missing them"))
1814     parser.add_argument("--delete-unknown", action="store_true", default=False,
1815                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1816     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1817                         help=_("Report on build data status"))
1818     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1819                         help=_("Interactively ask about things that need updating."))
1820     parser.add_argument("-I", "--icons", action="store_true", default=False,
1821                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1822     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1823                         help=_("Specify editor to use in interactive mode. Default " +
1824                                "is {path}").format(path='/etc/alternatives/editor'))
1825     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1826                         help=_("Update the wiki"))
1827     parser.add_argument("--pretty", action="store_true", default=False,
1828                         help=_("Produce human-readable XML/JSON for index files"))
1829     parser.add_argument("--clean", action="store_true", default=False,
1830                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1831     parser.add_argument("--nosign", action="store_true", default=False,
1832                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1833     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1834                         help=_("Use date from APK instead of current time for newly added APKs"))
1835     parser.add_argument("--rename-apks", action="store_true", default=False,
1836                         help=_("Rename APK files that do not match package.name_123.apk"))
1837     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1838                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1839     metadata.add_metadata_arguments(parser)
1840     options = parser.parse_args()
1841     metadata.warnings_action = options.W
1842
1843     config = common.read_config(options)
1844
1845     if not ('jarsigner' in config and 'keytool' in config):
1846         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1847
1848     repodirs = ['repo']
1849     if config['archive_older'] != 0:
1850         repodirs.append('archive')
1851         if not os.path.exists('archive'):
1852             os.mkdir('archive')
1853
1854     if options.icons:
1855         resize_all_icons(repodirs)
1856         sys.exit(0)
1857
1858     if options.rename_apks:
1859         options.clean = True
1860
1861     # check that icons exist now, rather than fail at the end of `fdroid update`
1862     for k in ['repo_icon', 'archive_icon']:
1863         if k in config:
1864             if not os.path.exists(config[k]):
1865                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1866                                  .format(name=k, path=config[k]))
1867                 sys.exit(1)
1868
1869     # if the user asks to create a keystore, do it now, reusing whatever it can
1870     if options.create_key:
1871         if os.path.exists(config['keystore']):
1872             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1873             logging.critical("\t'" + config['keystore'] + "'")
1874             sys.exit(1)
1875
1876         if 'repo_keyalias' not in config:
1877             config['repo_keyalias'] = socket.getfqdn()
1878             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1879         if 'keydname' not in config:
1880             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1881             common.write_to_config(config, 'keydname', config['keydname'])
1882         if 'keystore' not in config:
1883             config['keystore'] = common.default_config['keystore']
1884             common.write_to_config(config, 'keystore', config['keystore'])
1885
1886         password = common.genpassword()
1887         if 'keystorepass' not in config:
1888             config['keystorepass'] = password
1889             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1890         if 'keypass' not in config:
1891             config['keypass'] = password
1892             common.write_to_config(config, 'keypass', config['keypass'])
1893         common.genkeystore(config)
1894
1895     # Get all apps...
1896     apps = metadata.read_metadata()
1897
1898     # Generate a list of categories...
1899     categories = set()
1900     for app in apps.values():
1901         categories.update(app.Categories)
1902
1903     # Read known apks data (will be updated and written back when we've finished)
1904     knownapks = common.KnownApks()
1905
1906     # Get APK cache
1907     apkcache = get_cache()
1908
1909     # Delete builds for disabled apps
1910     delete_disabled_builds(apps, apkcache, repodirs)
1911
1912     # Scan all apks in the main repo
1913     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1914
1915     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1916                                            options.use_date_from_apk)
1917     cachechanged = cachechanged or fcachechanged
1918     apks += files
1919     for apk in apks:
1920         if apk['packageName'] not in apps:
1921             if options.create_metadata:
1922                 create_metadata_from_template(apk)
1923                 apps = metadata.read_metadata()
1924             else:
1925                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1926                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1927                 if options.delete_unknown:
1928                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1929                                  .format(apkfilename=apk['apkName']))
1930                     rmf = os.path.join(repodirs[0], apk['apkName'])
1931                     if not os.path.exists(rmf):
1932                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1933                     else:
1934                         os.remove(rmf)
1935                 else:
1936                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1937
1938     copy_triple_t_store_metadata(apps)
1939     insert_obbs(repodirs[0], apps, apks)
1940     insert_localized_app_metadata(apps)
1941     translate_per_build_anti_features(apps, apks)
1942
1943     # Scan the archive repo for apks as well
1944     if len(repodirs) > 1:
1945         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1946         if cc:
1947             cachechanged = True
1948     else:
1949         archapks = []
1950
1951     # Apply information from latest apks to the application and update dates
1952     apply_info_from_latest_apk(apps, apks + archapks)
1953
1954     # Sort the app list by name, then the web site doesn't have to by default.
1955     # (we had to wait until we'd scanned the apks to do this, because mostly the
1956     # name comes from there!)
1957     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1958
1959     # APKs are placed into multiple repos based on the app package, providing
1960     # per-app subscription feeds for nightly builds and things like it
1961     if config['per_app_repos']:
1962         add_apks_to_per_app_repos(repodirs[0], apks)
1963         for appid, app in apps.items():
1964             repodir = os.path.join(appid, 'fdroid', 'repo')
1965             appdict = dict()
1966             appdict[appid] = app
1967             if os.path.isdir(repodir):
1968                 index.make(appdict, [appid], apks, repodir, False)
1969             else:
1970                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1971         return
1972
1973     if len(repodirs) > 1:
1974         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1975
1976     # Make the index for the main repo...
1977     index.make(apps, sortedids, apks, repodirs[0], False)
1978     make_categories_txt(repodirs[0], categories)
1979
1980     # If there's an archive repo,  make the index for it. We already scanned it
1981     # earlier on.
1982     if len(repodirs) > 1:
1983         index.make(apps, sortedids, archapks, repodirs[1], True)
1984
1985     git_remote = config.get('binary_transparency_remote')
1986     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1987         from . import btlog
1988         btlog.make_binary_transparency_log(repodirs)
1989
1990     if config['update_stats']:
1991         # Update known apks info...
1992         knownapks.writeifchanged()
1993
1994         # Generate latest apps data for widget
1995         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1996             data = ''
1997             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1998                 for line in f:
1999                     appid = line.rstrip()
2000                     data += appid + "\t"
2001                     app = apps[appid]
2002                     data += app.Name + "\t"
2003                     if app.icon is not None:
2004                         data += app.icon + "\t"
2005                     data += app.License + "\n"
2006             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
2007                 f.write(data)
2008
2009     if cachechanged:
2010         write_cache(apkcache)
2011
2012     # Update the wiki...
2013     if options.wiki:
2014         update_wiki(apps, sortedids, apks + archapks)
2015
2016     logging.info(_("Finished"))
2017
2018
2019 if __name__ == "__main__":
2020     main()