fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*)'")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0' or density == '65534':
  98         return os.path.join(repodir, "icons")
  99     else:
 100         return os.path.join(repodir, "icons-%s" % density)
 101
 102
 103 def get_icon_dirs(repodir):
 104     for density in screen_densities:
 105         yield get_icon_dir(repodir, density)
 106
 107
 108 def get_all_icon_dirs(repodir):
 109     for density in all_screen_densities:
 110         yield get_icon_dir(repodir, density)
 111
 112
 113 def update_wiki(apps, sortedids, apks):
 114     """Update the wiki
 115
 116     :param apps: fully populated list of all applications
 117     :param apks: all apks, except...
 118     """
 119     logging.info("Updating wiki")
 120     wikicat = 'Apps'
 121     wikiredircat = 'App Redirects'
 122     import mwclient
 123     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 124                          path=config['wiki_path'])
 125     site.login(config['wiki_user'], config['wiki_password'])
 126     generated_pages = {}
 127     generated_redirects = {}
 128
 129     for appid in sortedids:
 130         app = metadata.App(apps[appid])
 131
 132         wikidata = ''
 133         if app.Disabled:
 134             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 135         if app.AntiFeatures:
 136             for af in sorted(app.AntiFeatures):
 137                 wikidata += '{{AntiFeature|' + af + '}}\n'
 138         if app.RequiresRoot:
 139             requiresroot = 'Yes'
 140         else:
 141             requiresroot = 'No'
 142         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 143             appid,
 144             app.Name,
 145             app.added.strftime('%Y-%m-%d') if app.added else '',
 146             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 147             app.SourceCode,
 148             app.IssueTracker,
 149             app.WebSite,
 150             app.Changelog,
 151             app.Donate,
 152             app.FlattrID,
 153             app.LiberapayID,
 154             app.Bitcoin,
 155             app.Litecoin,
 156             app.License,
 157             requiresroot,
 158             app.AuthorName,
 159             app.AuthorEmail)
 160
 161         if app.Provides:
 162             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 163
 164         wikidata += app.Summary
 165         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 166
 167         wikidata += "=Description=\n"
 168         wikidata += metadata.description_wiki(app.Description) + "\n"
 169
 170         wikidata += "=Maintainer Notes=\n"
 171         if app.MaintainerNotes:
 172             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 173         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 174
 175         # Get a list of all packages for this application...
 176         apklist = []
 177         gotcurrentver = False
 178         cantupdate = False
 179         buildfails = False
 180         for apk in apks:
 181             if apk['packageName'] == appid:
 182                 if str(apk['versionCode']) == app.CurrentVersionCode:
 183                     gotcurrentver = True
 184                 apklist.append(apk)
 185         # Include ones we can't build, as a special case...
 186         for build in app.builds:
 187             if build.disable:
 188                 if build.versionCode == app.CurrentVersionCode:
 189                     cantupdate = True
 190                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 191                 apklist.append({'versionCode': int(build.versionCode),
 192                                 'versionName': build.versionName,
 193                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 194                                 })
 195             else:
 196                 builtit = False
 197                 for apk in apklist:
 198                     if apk['versionCode'] == int(build.versionCode):
 199                         builtit = True
 200                         break
 201                 if not builtit:
 202                     buildfails = True
 203                     apklist.append({'versionCode': int(build.versionCode),
 204                                     'versionName': build.versionName,
 205                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 206                                     })
 207         if app.CurrentVersionCode == '0':
 208             cantupdate = True
 209         # Sort with most recent first...
 210         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 211
 212         wikidata += "=Versions=\n"
 213         if len(apklist) == 0:
 214             wikidata += "We currently have no versions of this app available."
 215         elif not gotcurrentver:
 216             wikidata += "We don't have the current version of this app."
 217         else:
 218             wikidata += "We have the current version of this app."
 219         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 220         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 221         if len(app.NoSourceSince) > 0:
 222             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 223         if len(app.CurrentVersion) > 0:
 224             wikidata += "The current (recommended) version is " + app.CurrentVersion
 225             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 226         validapks = 0
 227         for apk in apklist:
 228             wikidata += "==" + apk['versionName'] + "==\n"
 229
 230             if 'buildproblem' in apk:
 231                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 232             else:
 233                 validapks += 1
 234                 wikidata += "This version is built and signed by "
 235                 if 'srcname' in apk:
 236                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 237                 else:
 238                     wikidata += "the original developer.\n\n"
 239             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 240
 241         wikidata += '\n[[Category:' + wikicat + ']]\n'
 242         if len(app.NoSourceSince) > 0:
 243             wikidata += '\n[[Category:Apps missing source code]]\n'
 244         if validapks == 0 and not app.Disabled:
 245             wikidata += '\n[[Category:Apps with no packages]]\n'
 246         if cantupdate and not app.Disabled:
 247             wikidata += "\n[[Category:Apps we cannot update]]\n"
 248         if buildfails and not app.Disabled:
 249             wikidata += "\n[[Category:Apps with failing builds]]\n"
 250         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 251             wikidata += '\n[[Category:Apps to Update]]\n'
 252         if app.Disabled:
 253             wikidata += '\n[[Category:Apps that are disabled]]\n'
 254         if app.UpdateCheckMode == 'None' and not app.Disabled:
 255             wikidata += '\n[[Category:Apps with no update check]]\n'
 256         for appcat in app.Categories:
 257             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 258
 259         # We can't have underscores in the page name, even if they're in
 260         # the package ID, because MediaWiki messes with them...
 261         pagename = appid.replace('_', ' ')
 262
 263         # Drop a trailing newline, because mediawiki is going to drop it anyway
 264         # and it we don't we'll think the page has changed when it hasn't...
 265         if wikidata.endswith('\n'):
 266             wikidata = wikidata[:-1]
 267
 268         generated_pages[pagename] = wikidata
 269
 270         # Make a redirect from the name to the ID too, unless there's
 271         # already an existing page with the name and it isn't a redirect.
 272         noclobber = False
 273         apppagename = app.Name.replace('_', ' ')
 274         apppagename = apppagename.replace('{', '')
 275         apppagename = apppagename.replace('}', ' ')
 276         apppagename = apppagename.replace(':', ' ')
 277         apppagename = apppagename.replace('[', ' ')
 278         apppagename = apppagename.replace(']', ' ')
 279         # Drop double spaces caused mostly by replacing ':' above
 280         apppagename = apppagename.replace('  ', ' ')
 281         for expagename in site.allpages(prefix=apppagename,
 282                                         filterredir='nonredirects',
 283                                         generator=False):
 284             if expagename == apppagename:
 285                 noclobber = True
 286         # Another reason not to make the redirect page is if the app name
 287         # is the same as it's ID, because that will overwrite the real page
 288         # with an redirect to itself! (Although it seems like an odd
 289         # scenario this happens a lot, e.g. where there is metadata but no
 290         # builds or binaries to extract a name from.
 291         if apppagename == pagename:
 292             noclobber = True
 293         if not noclobber:
 294             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 295
 296     for tcat, genp in [(wikicat, generated_pages),
 297                        (wikiredircat, generated_redirects)]:
 298         catpages = site.Pages['Category:' + tcat]
 299         existingpages = []
 300         for page in catpages:
 301             existingpages.append(page.name)
 302             if page.name in genp:
 303                 pagetxt = page.edit()
 304                 if pagetxt != genp[page.name]:
 305                     logging.debug("Updating modified page " + page.name)
 306                     page.save(genp[page.name], summary='Auto-updated')
 307                 else:
 308                     logging.debug("Page " + page.name + " is unchanged")
 309             else:
 310                 logging.warn("Deleting page " + page.name)
 311                 page.delete('No longer published')
 312         for pagename, text in genp.items():
 313             logging.debug("Checking " + pagename)
 314             if pagename not in existingpages:
 315                 logging.debug("Creating page " + pagename)
 316                 try:
 317                     newpage = site.Pages[pagename]
 318                     newpage.save(text, summary='Auto-created')
 319                 except Exception as e:
 320                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 321
 322     # Purge server cache to ensure counts are up to date
 323     site.Pages['Repository Maintenance'].purge()
 324
 325     # Write a page with the last build log for this version code
 326     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 327     newpage = site.Pages[wiki_page_path]
 328     txt = ''
 329     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 330     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 331     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 332     txt += common.get_git_describe_link()
 333     txt += "\n\n"
 334     txt += common.get_android_tools_version_log()
 335     newpage.save(txt, summary='Run log')
 336     newpage = site.Pages['update']
 337     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 338
 339
 340 def delete_disabled_builds(apps, apkcache, repodirs):
 341     """Delete disabled build outputs.
 342
 343     :param apps: list of all applications, as per metadata.read_metadata
 344     :param apkcache: current apk cache information
 345     :param repodirs: the repo directories to process
 346     """
 347     for appid, app in apps.items():
 348         for build in app['builds']:
 349             if not build.disable:
 350                 continue
 351             apkfilename = common.get_release_filename(app, build)
 352             iconfilename = "%s.%s.png" % (
 353                 appid,
 354                 build.versionCode)
 355             for repodir in repodirs:
 356                 files = [
 357                     os.path.join(repodir, apkfilename),
 358                     os.path.join(repodir, apkfilename + '.asc'),
 359                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 360                 ]
 361                 for density in all_screen_densities:
 362                     repo_dir = get_icon_dir(repodir, density)
 363                     files.append(os.path.join(repo_dir, iconfilename))
 364
 365                 for f in files:
 366                     if os.path.exists(f):
 367                         logging.info("Deleting disabled build output " + f)
 368                         os.remove(f)
 369             if apkfilename in apkcache:
 370                 del apkcache[apkfilename]
 371
 372
 373 def resize_icon(iconpath, density):
 374
 375     if not os.path.isfile(iconpath):
 376         return
 377
 378     fp = None
 379     try:
 380         fp = open(iconpath, 'rb')
 381         im = Image.open(fp)
 382         size = dpi_to_px(density)
 383
 384         if any(length > size for length in im.size):
 385             oldsize = im.size
 386             im.thumbnail((size, size), Image.ANTIALIAS)
 387             logging.debug("%s was too large at %s - new size is %s" % (
 388                 iconpath, oldsize, im.size))
 389             im.save(iconpath, "PNG", optimize=True,
 390                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 391
 392     except Exception as e:
 393         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 394
 395     finally:
 396         if fp:
 397             fp.close()
 398
 399
 400 def resize_all_icons(repodirs):
 401     """Resize all icons that exceed the max size
 402
 403     :param repodirs: the repo directories to process
 404     """
 405     for repodir in repodirs:
 406         for density in screen_densities:
 407             icon_dir = get_icon_dir(repodir, density)
 408             icon_glob = os.path.join(icon_dir, '*.png')
 409             for iconpath in glob.glob(icon_glob):
 410                 resize_icon(iconpath, density)
 411
 412
 413 def getsig(apkpath):
 414     """ Get the signing certificate of an apk. To get the same md5 has that
 415     Android gets, we encode the .RSA certificate in a specific format and pass
 416     it hex-encoded to the md5 digest algorithm.
 417
 418     :param apkpath: path to the apk
 419     :returns: A string containing the md5 of the signature of the apk or None
 420               if an error occurred.
 421     """
 422
 423     with zipfile.ZipFile(apkpath, 'r') as apk:
 424         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 425
 426         if len(certs) < 1:
 427             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 428             return None
 429         if len(certs) > 1:
 430             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 431             return None
 432
 433         cert = apk.read(certs[0])
 434
 435     cert_encoded = common.get_certificate(cert)
 436
 437     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 438
 439
 440 def get_cache_file():
 441     return os.path.join('tmp', 'apkcache')
 442
 443
 444 def get_cache():
 445     """Get the cached dict of the APK index
 446
 447     Gather information about all the apk files in the repo directory,
 448     using cached data if possible. Some of the index operations take a
 449     long time, like calculating the SHA-256 and verifying the APK
 450     signature.
 451
 452     The cache is invalidated if the metadata version is different, or
 453     the 'allow_disabled_algorithms' config/option is different.  In
 454     those cases, there is no easy way to know what has changed from
 455     the cache, so just rerun the whole thing.
 456
 457     :return: apkcache
 458
 459     """
 460     apkcachefile = get_cache_file()
 461     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 462     if not options.clean and os.path.exists(apkcachefile):
 463         with open(apkcachefile, 'rb') as cf:
 464             apkcache = pickle.load(cf, encoding='utf-8')
 465         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 466            or apkcache.get('allow_disabled_algorithms') != ada:
 467             apkcache = {}
 468     else:
 469         apkcache = {}
 470
 471     apkcache["METADATA_VERSION"] = METADATA_VERSION
 472     apkcache['allow_disabled_algorithms'] = ada
 473
 474     return apkcache
 475
 476
 477 def write_cache(apkcache):
 478     apkcachefile = get_cache_file()
 479     cache_path = os.path.dirname(apkcachefile)
 480     if not os.path.exists(cache_path):
 481         os.makedirs(cache_path)
 482     with open(apkcachefile, 'wb') as cf:
 483         pickle.dump(apkcache, cf)
 484
 485
 486 def get_icon_bytes(apkzip, iconsrc):
 487     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 488     try:
 489         return apkzip.read(iconsrc)
 490     except KeyError:
 491         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 492
 493
 494 def sha256sum(filename):
 495     '''Calculate the sha256 of the given file'''
 496     sha = hashlib.sha256()
 497     with open(filename, 'rb') as f:
 498         while True:
 499             t = f.read(16384)
 500             if len(t) == 0:
 501                 break
 502             sha.update(t)
 503     return sha.hexdigest()
 504
 505
 506 def has_known_vulnerability(filename):
 507     """checks for known vulnerabilities in the APK
 508
 509     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 510     version.  Google also enforces this:
 511     https://support.google.com/faqs/answer/6376725?hl=en
 512
 513     Checks whether there are more than one classes.dex or AndroidManifest.xml
 514     files, which is invalid and an essential part of the "Master Key" attack.
 515     http://www.saurik.com/id/17
 516
 517     Janus is similar to Master Key but is perhaps easier to scan for.
 518     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 519     """
 520
 521     found_vuln = False
 522
 523     # statically load this pattern
 524     if not hasattr(has_known_vulnerability, "pattern"):
 525         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 526
 527     with open(filename.encode(), 'rb') as fp:
 528         first4 = fp.read(4)
 529     if first4 != b'\x50\x4b\x03\x04':
 530         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 531                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 532                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 533
 534     files_in_apk = set()
 535     with zipfile.ZipFile(filename) as zf:
 536         for name in zf.namelist():
 537             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 538                 lib = zf.open(name)
 539                 while True:
 540                     chunk = lib.read(4096)
 541                     if chunk == b'':
 542                         break
 543                     m = has_known_vulnerability.pattern.search(chunk)
 544                     if m:
 545                         version = m.group(1).decode('ascii')
 546                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 547                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 548                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 549                             logging.debug(_('"{path}" contains recent {name} ({version})')
 550                                           .format(path=filename, name=name, version=version))
 551                         else:
 552                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 553                                             .format(path=filename, name=name, version=version))
 554                             found_vuln = True
 555                         break
 556             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 557                 if name in files_in_apk:
 558                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 559                                     .format(apkfilename=filename, name=name))
 560                     found_vuln = True
 561                 files_in_apk.add(name)
 562     return found_vuln
 563
 564
 565 def insert_obbs(repodir, apps, apks):
 566     """Scans the .obb files in a given repo directory and adds them to the
 567     relevant APK instances.  OBB files have versionCodes like APK
 568     files, and they are loosely associated.  If there is an OBB file
 569     present, then any APK with the same or higher versionCode will use
 570     that OBB file.  There are two OBB types: main and patch, each APK
 571     can only have only have one of each.
 572
 573     https://developer.android.com/google/play/expansion-files.html
 574
 575     :param repodir: repo directory to scan
 576     :param apps: list of current, valid apps
 577     :param apks: current information on all APKs
 578
 579     """
 580
 581     def obbWarnDelete(f, msg):
 582         logging.warning(msg + ' ' + f)
 583         if options.delete_unknown:
 584             logging.error(_("Deleting unknown file: {path}").format(path=f))
 585             os.remove(f)
 586
 587     obbs = []
 588     java_Integer_MIN_VALUE = -pow(2, 31)
 589     currentPackageNames = apps.keys()
 590     for f in glob.glob(os.path.join(repodir, '*.obb')):
 591         obbfile = os.path.basename(f)
 592         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 593         chunks = obbfile.split('.')
 594         if chunks[0] != 'main' and chunks[0] != 'patch':
 595             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 596             continue
 597         if not re.match(r'^-?[0-9]+$', chunks[1]):
 598             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 599                           .format(name=chunks[0]))
 600             continue
 601         versionCode = int(chunks[1])
 602         packagename = ".".join(chunks[2:-1])
 603
 604         highestVersionCode = java_Integer_MIN_VALUE
 605         if packagename not in currentPackageNames:
 606             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 607             continue
 608         for apk in apks:
 609             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 610                 highestVersionCode = apk['versionCode']
 611         if versionCode > highestVersionCode:
 612             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 613                           .format(integer=str(versionCode)))
 614             continue
 615         obbsha256 = sha256sum(f)
 616         obbs.append((packagename, versionCode, obbfile, obbsha256))
 617
 618     for apk in apks:
 619         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 620             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 621                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 622                     apk['obbMainFile'] = obbfile
 623                     apk['obbMainFileSha256'] = obbsha256
 624                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 625                     apk['obbPatchFile'] = obbfile
 626                     apk['obbPatchFileSha256'] = obbsha256
 627             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 628                 break
 629
 630
 631 def translate_per_build_anti_features(apps, apks):
 632     """Grab the anti-features list from the build metadata
 633
 634     For most Anti-Features, they are really most applicable per-APK,
 635     not for an app.  An app can fix a vulnerability, add/remove
 636     tracking, etc.  This reads the 'antifeatures' list from the Build
 637     entries in the fdroiddata metadata file, then transforms it into
 638     the 'antiFeatures' list of unique items for the index.
 639
 640     The field key is all lower case in the metadata file to match the
 641     rest of the Build fields.  It is 'antiFeatures' camel case in the
 642     implementation, index, and fdroidclient since it is translated
 643     from the build 'antifeatures' field, not directly included.
 644
 645     """
 646
 647     antiFeatures = dict()
 648     for packageName, app in apps.items():
 649         d = dict()
 650         for build in app['builds']:
 651             afl = build.get('antifeatures')
 652             if afl:
 653                 d[int(build.versionCode)] = afl
 654         if len(d) > 0:
 655             antiFeatures[packageName] = d
 656
 657     for apk in apks:
 658         d = antiFeatures.get(apk['packageName'])
 659         if d:
 660             afl = d.get(apk['versionCode'])
 661             if afl:
 662                 apk['antiFeatures'].update(afl)
 663
 664
 665 def _get_localized_dict(app, locale):
 666     '''get the dict to add localized store metadata to'''
 667     if 'localized' not in app:
 668         app['localized'] = collections.OrderedDict()
 669     if locale not in app['localized']:
 670         app['localized'][locale] = collections.OrderedDict()
 671     return app['localized'][locale]
 672
 673
 674 def _set_localized_text_entry(app, locale, key, f):
 675     limit = config['char_limits'][key]
 676     localized = _get_localized_dict(app, locale)
 677     with open(f) as fp:
 678         text = fp.read()[:limit]
 679         if len(text) > 0:
 680             localized[key] = text
 681
 682
 683 def _set_author_entry(app, key, f):
 684     limit = config['char_limits']['author']
 685     with open(f) as fp:
 686         text = fp.read()[:limit]
 687         if len(text) > 0:
 688             app[key] = text
 689
 690
 691 def _strip_and_copy_image(inpath, outpath):
 692     """Remove any metadata from image and copy it to new path
 693
 694     Sadly, image metadata like EXIF can be used to exploit devices.
 695     It is not used at all in the F-Droid ecosystem, so its much safer
 696     just to remove it entirely.
 697
 698     """
 699
 700     extension = common.get_extension(inpath)[1]
 701     if os.path.isdir(outpath):
 702         outpath = os.path.join(outpath, os.path.basename(inpath))
 703     if extension == 'png':
 704         with open(inpath, 'rb') as fp:
 705             in_image = Image.open(fp)
 706             in_image.save(outpath, "PNG", optimize=True,
 707                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 708     elif extension == 'jpg' or extension == 'jpeg':
 709         with open(inpath, 'rb') as fp:
 710             in_image = Image.open(fp)
 711             data = list(in_image.getdata())
 712             out_image = Image.new(in_image.mode, in_image.size)
 713         out_image.putdata(data)
 714         out_image.save(outpath, "JPEG", optimize=True)
 715     else:
 716         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 717                               .format(extension=extension))
 718
 719
 720 def copy_triple_t_store_metadata(apps):
 721     """Include store metadata from the app's source repo
 722
 723     The Triple-T Gradle Play Publisher is a plugin that has a standard
 724     file layout for all of the metadata and graphics that the Google
 725     Play Store accepts.  Since F-Droid has the git repo, it can just
 726     pluck those files directly.  This method reads any text files into
 727     the app dict, then copies any graphics into the fdroid repo
 728     directory structure.
 729
 730     This needs to be run before insert_localized_app_metadata() so that
 731     the graphics files that are copied into the fdroid repo get
 732     properly indexed.
 733
 734     https://github.com/Triple-T/gradle-play-publisher#upload-images
 735     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 736
 737     """
 738
 739     if not os.path.isdir('build'):
 740         return  # nothing to do
 741
 742     for packageName, app in apps.items():
 743         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 744             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 745             for root, dirs, files in os.walk(d):
 746                 segments = root.split('/')
 747                 locale = segments[-2]
 748                 for f in files:
 749                     if f == 'fulldescription':
 750                         _set_localized_text_entry(app, locale, 'description',
 751                                                   os.path.join(root, f))
 752                         continue
 753                     elif f == 'shortdescription':
 754                         _set_localized_text_entry(app, locale, 'summary',
 755                                                   os.path.join(root, f))
 756                         continue
 757                     elif f == 'title':
 758                         _set_localized_text_entry(app, locale, 'name',
 759                                                   os.path.join(root, f))
 760                         continue
 761                     elif f == 'video':
 762                         _set_localized_text_entry(app, locale, 'video',
 763                                                   os.path.join(root, f))
 764                         continue
 765                     elif f == 'whatsnew':
 766                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 767                                                   os.path.join(root, f))
 768                         continue
 769                     elif f == 'contactEmail':
 770                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 771                         continue
 772                     elif f == 'contactPhone':
 773                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 774                         continue
 775                     elif f == 'contactWebsite':
 776                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 777                         continue
 778
 779                     base, extension = common.get_extension(f)
 780                     dirname = os.path.basename(root)
 781                     if extension in ALLOWED_EXTENSIONS \
 782                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 783                         if segments[-2] == 'listing':
 784                             locale = segments[-3]
 785                         else:
 786                             locale = segments[-2]
 787                         destdir = os.path.join('repo', packageName, locale, dirname)
 788                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 789                         sourcefile = os.path.join(root, f)
 790                         destfile = os.path.join(destdir, os.path.basename(f))
 791                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 792                         _strip_and_copy_image(sourcefile, destfile)
 793
 794
 795 def insert_localized_app_metadata(apps):
 796     """scans standard locations for graphics and localized text
 797
 798     Scans for localized description files, store graphics, and
 799     screenshot PNG files in statically defined screenshots directory
 800     and adds them to the app metadata.  The screenshots and graphic
 801     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 802     and must be in the following layout:
 803     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 804
 805     repo/packageName/locale/featureGraphic.png
 806     repo/packageName/locale/phoneScreenshots/1.png
 807     repo/packageName/locale/phoneScreenshots/2.png
 808
 809     The changelog files must be text files named with the versionCode
 810     ending with ".txt" and must be in the following layout:
 811     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 812
 813     repo/packageName/locale/changelogs/12345.txt
 814
 815     This will scan the each app's source repo then the metadata/ dir
 816     for these standard locations of changelog files.  If it finds
 817     them, they will be added to the dict of all packages, with the
 818     versions in the metadata/ folder taking precendence over the what
 819     is in the app's source repo.
 820
 821     Where "packageName" is the app's packageName and "locale" is the locale
 822     of the graphics, e.g. what language they are in, using the IETF RFC5646
 823     format (en-US, fr-CA, es-MX, etc).
 824
 825     This will also scan the app's git for a fastlane folder, and the
 826     metadata/ folder and the apps' source repos for standard locations
 827     of graphic and screenshot files.  If it finds them, it will copy
 828     them into the repo.  The fastlane files follow this pattern:
 829     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 830
 831     """
 832
 833     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 834     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 837
 838     for srcd in sorted(sourcedirs):
 839         if not os.path.isdir(srcd):
 840             continue
 841         for root, dirs, files in os.walk(srcd):
 842             segments = root.split('/')
 843             packageName = segments[1]
 844             if packageName not in apps:
 845                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 846                 continue
 847             locale = segments[-1]
 848             destdir = os.path.join('repo', packageName, locale)
 849
 850             # flavours specified in build receipt
 851             build_flavours = ""
 852             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 853                     and 'gradle' in apps[packageName].builds[-1]:
 854                 build_flavours = apps[packageName].builds[-1].gradle
 855
 856             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 857                 logging.debug("ignoring due to wrong flavour")
 858                 continue
 859
 860             for f in files:
 861                 if f in ('description.txt', 'full_description.txt'):
 862                     _set_localized_text_entry(apps[packageName], locale, 'description',
 863                                               os.path.join(root, f))
 864                     continue
 865                 elif f in ('summary.txt', 'short_description.txt'):
 866                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 867                                               os.path.join(root, f))
 868                     continue
 869                 elif f in ('name.txt', 'title.txt'):
 870                     _set_localized_text_entry(apps[packageName], locale, 'name',
 871                                               os.path.join(root, f))
 872                     continue
 873                 elif f == 'video.txt':
 874                     _set_localized_text_entry(apps[packageName], locale, 'video',
 875                                               os.path.join(root, f))
 876                     continue
 877                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 878                     locale = segments[-2]
 879                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 880                                               os.path.join(root, f))
 881                     continue
 882
 883                 base, extension = common.get_extension(f)
 884                 if locale == 'images':
 885                     locale = segments[-2]
 886                     destdir = os.path.join('repo', packageName, locale)
 887                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 888                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 889                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 890                     _strip_and_copy_image(os.path.join(root, f), destdir)
 891             for d in dirs:
 892                 if d in SCREENSHOT_DIRS:
 893                     if locale == 'images':
 894                         locale = segments[-2]
 895                         destdir = os.path.join('repo', packageName, locale)
 896                     for f in glob.glob(os.path.join(root, d, '*.*')):
 897                         _ignored, extension = common.get_extension(f)
 898                         if extension in ALLOWED_EXTENSIONS:
 899                             screenshotdestdir = os.path.join(destdir, d)
 900                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 901                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 902                             _strip_and_copy_image(f, screenshotdestdir)
 903
 904     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 905     for d in repofiles:
 906         if not os.path.isdir(d):
 907             continue
 908         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 909             if not os.path.isfile(f):
 910                 continue
 911             segments = f.split('/')
 912             packageName = segments[1]
 913             locale = segments[2]
 914             screenshotdir = segments[3]
 915             filename = os.path.basename(f)
 916             base, extension = common.get_extension(filename)
 917
 918             if packageName not in apps:
 919                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 920                                 .format(path=filename, name=packageName))
 921                 continue
 922             graphics = _get_localized_dict(apps[packageName], locale)
 923
 924             if extension not in ALLOWED_EXTENSIONS:
 925                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 926             elif base in GRAPHIC_NAMES:
 927                 # there can only be zero or one of these per locale
 928                 graphics[base] = filename
 929             elif screenshotdir in SCREENSHOT_DIRS:
 930                 # there can any number of these per locale
 931                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 932                 if screenshotdir not in graphics:
 933                     graphics[screenshotdir] = []
 934                 graphics[screenshotdir].append(filename)
 935             else:
 936                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 937
 938
 939 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 940     """Scan a repo for all files with an extension except APK/OBB
 941
 942     :param apkcache: current cached info about all repo files
 943     :param repodir: repo directory to scan
 944     :param knownapks: list of all known files, as per metadata.read_metadata
 945     :param use_date_from_file: use date from file (instead of current date)
 946                                for newly added files
 947     """
 948
 949     cachechanged = False
 950     repo_files = []
 951     repodir = repodir.encode('utf-8')
 952     for name in os.listdir(repodir):
 953         file_extension = common.get_file_extension(name)
 954         if file_extension == 'apk' or file_extension == 'obb':
 955             continue
 956         filename = os.path.join(repodir, name)
 957         name_utf8 = name.decode('utf-8')
 958         if filename.endswith(b'_src.tar.gz'):
 959             logging.debug(_('skipping source tarball: {path}')
 960                           .format(path=filename.decode('utf-8')))
 961             continue
 962         if not common.is_repo_file(filename):
 963             continue
 964         stat = os.stat(filename)
 965         if stat.st_size == 0:
 966             raise FDroidException(_('{path} is zero size!')
 967                                   .format(path=filename))
 968
 969         shasum = sha256sum(filename)
 970         usecache = False
 971         if name in apkcache:
 972             repo_file = apkcache[name]
 973             # added time is cached as tuple but used here as datetime instance
 974             if 'added' in repo_file:
 975                 a = repo_file['added']
 976                 if isinstance(a, datetime):
 977                     repo_file['added'] = a
 978                 else:
 979                     repo_file['added'] = datetime(*a[:6])
 980             if repo_file.get('hash') == shasum:
 981                 logging.debug(_("Reading {apkfilename} from cache")
 982                               .format(apkfilename=name_utf8))
 983                 usecache = True
 984             else:
 985                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 986                               .format(apkfilename=name_utf8))
 987
 988         if not usecache:
 989             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 990             repo_file = collections.OrderedDict()
 991             repo_file['name'] = os.path.splitext(name_utf8)[0]
 992             # TODO rename apkname globally to something more generic
 993             repo_file['apkName'] = name_utf8
 994             repo_file['hash'] = shasum
 995             repo_file['hashType'] = 'sha256'
 996             repo_file['versionCode'] = 0
 997             repo_file['versionName'] = shasum
 998             # the static ID is the SHA256 unless it is set in the metadata
 999             repo_file['packageName'] = shasum
1000
1001             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1002             if m:
1003                 repo_file['packageName'] = m.group(1)
1004                 repo_file['versionCode'] = int(m.group(2))
1005             srcfilename = name + b'_src.tar.gz'
1006             if os.path.exists(os.path.join(repodir, srcfilename)):
1007                 repo_file['srcname'] = srcfilename.decode('utf-8')
1008             repo_file['size'] = stat.st_size
1009
1010             apkcache[name] = repo_file
1011             cachechanged = True
1012
1013         if use_date_from_file:
1014             timestamp = stat.st_ctime
1015             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1016         else:
1017             default_date_param = None
1018
1019         # Record in knownapks, getting the added date at the same time..
1020         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1021                                     default_date=default_date_param)
1022         if added:
1023             repo_file['added'] = added
1024
1025         repo_files.append(repo_file)
1026
1027     return repo_files, cachechanged
1028
1029
1030 def scan_apk(apk_file):
1031     """
1032     Scans an APK file and returns dictionary with metadata of the APK.
1033
1034     Attention: This does *not* verify that the APK signature is correct.
1035
1036     :param apk_file: The (ideally absolute) path to the APK file
1037     :raises BuildException
1038     :return A dict containing APK metadata
1039     """
1040     apk = {
1041         'hash': sha256sum(apk_file),
1042         'hashType': 'sha256',
1043         'uses-permission': [],
1044         'uses-permission-sdk-23': [],
1045         'features': [],
1046         'icons_src': {},
1047         'icons': {},
1048         'antiFeatures': set(),
1049     }
1050
1051     if common.use_androguard():
1052         scan_apk_androguard(apk, apk_file)
1053     else:
1054         scan_apk_aapt(apk, apk_file)
1055
1056     # Get the signature, or rather the signing key fingerprints
1057     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1058     apk['sig'] = getsig(apk_file)
1059     if not apk['sig']:
1060         raise BuildException("Failed to get apk signature")
1061     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1062                                                                apk_file))
1063     if not apk.get('signer'):
1064         raise BuildException("Failed to get apk signing key fingerprint")
1065
1066     # Get size of the APK
1067     apk['size'] = os.path.getsize(apk_file)
1068
1069     if 'minSdkVersion' not in apk:
1070         logging.warning("No SDK version information found in {0}".format(apk_file))
1071         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
1072     if 'targetSdkVersion' not in apk:
1073         apk['targetSdkVersion'] = apk['minSdkVersion']
1074
1075     # Check for known vulnerabilities
1076     if has_known_vulnerability(apk_file):
1077         apk['antiFeatures'].add('KnownVuln')
1078
1079     return apk
1080
1081
1082 def _get_apk_icons_src(apkfile, icon_name):
1083     """Extract the paths to the app icon in all available densities
1084
1085     """
1086     icons_src = dict()
1087     density_re = re.compile('^res/(.*)/' + icon_name + '\.(png|xml)$')
1088     with zipfile.ZipFile(apkfile) as zf:
1089         for filename in zf.namelist():
1090             m = density_re.match(filename)
1091             if m:
1092                 folder = m.group(1).split('-')
1093                 if len(folder) > 1:
1094                     density = screen_resolutions[folder[1]]
1095                 else:
1096                     density = '160'
1097                 icons_src[density] = m.group(0)
1098     if icons_src.get('-1') is None:
1099         icons_src['-1'] = icons_src['160']
1100     return icons_src
1101
1102
1103 def scan_apk_aapt(apk, apkfile):
1104     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1105     if p.returncode != 0:
1106         if options.delete_unknown:
1107             if os.path.exists(apkfile):
1108                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1109                 os.remove(apkfile)
1110             else:
1111                 logging.error("Could not find {0} to remove it".format(apkfile))
1112         else:
1113             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1114         raise BuildException(_("Invalid APK"))
1115     icon_name = None
1116     for line in p.output.splitlines():
1117         if line.startswith("package:"):
1118             try:
1119                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1120                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1121                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1122             except Exception as e:
1123                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1124         elif line.startswith("application:"):
1125             m = re.match(APK_LABEL_ICON_PAT, line)
1126             if m:
1127                 apk['name'] = m.group(1)
1128                 icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
1129         elif not apk.get('name') and line.startswith("launchable-activity:"):
1130             # Only use launchable-activity as fallback to application
1131             apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)
1132         elif line.startswith("sdkVersion:"):
1133             m = re.match(APK_SDK_VERSION_PAT, line)
1134             if m is None:
1135                 logging.error(line.replace('sdkVersion:', '')
1136                               + ' is not a valid minSdkVersion!')
1137             else:
1138                 apk['minSdkVersion'] = m.group(1)
1139         elif line.startswith("targetSdkVersion:"):
1140             m = re.match(APK_SDK_VERSION_PAT, line)
1141             if m is None:
1142                 logging.error(line.replace('targetSdkVersion:', '')
1143                               + ' is not a valid targetSdkVersion!')
1144             else:
1145                 apk['targetSdkVersion'] = m.group(1)
1146         elif line.startswith("maxSdkVersion:"):
1147             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1148         elif line.startswith("native-code:"):
1149             apk['nativecode'] = []
1150             for arch in line[13:].split(' '):
1151                 apk['nativecode'].append(arch[1:-1])
1152         elif line.startswith('uses-permission:'):
1153             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1154             if perm_match['maxSdkVersion']:
1155                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1156             permission = UsesPermission(
1157                 perm_match['name'],
1158                 perm_match['maxSdkVersion']
1159             )
1160
1161             apk['uses-permission'].append(permission)
1162         elif line.startswith('uses-permission-sdk-23:'):
1163             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1164             if perm_match['maxSdkVersion']:
1165                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1166             permission_sdk_23 = UsesPermissionSdk23(
1167                 perm_match['name'],
1168                 perm_match['maxSdkVersion']
1169             )
1170
1171             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1172
1173         elif line.startswith('uses-feature:'):
1174             feature = re.match(APK_FEATURE_PAT, line).group(1)
1175             # Filter out this, it's only added with the latest SDK tools and
1176             # causes problems for lots of apps.
1177             if feature != "android.hardware.screen.portrait" \
1178                     and feature != "android.hardware.screen.landscape":
1179                 if feature.startswith("android.feature."):
1180                     feature = feature[16:]
1181                 apk['features'].add(feature)
1182     apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1183
1184
1185 def scan_apk_androguard(apk, apkfile):
1186     try:
1187         from androguard.core.bytecodes.apk import APK
1188         apkobject = APK(apkfile)
1189         if apkobject.is_valid_APK():
1190             arsc = apkobject.get_android_resources()
1191         else:
1192             if options.delete_unknown:
1193                 if os.path.exists(apkfile):
1194                     logging.error(_("Failed to get apk information, deleting {path}")
1195                                   .format(path=apkfile))
1196                     os.remove(apkfile)
1197                 else:
1198                     logging.error(_("Could not find {path} to remove it")
1199                                   .format(path=apkfile))
1200             else:
1201                 logging.error(_("Failed to get apk information, skipping {path}")
1202                               .format(path=apkfile))
1203             raise BuildException(_("Invalid APK"))
1204     except ImportError:
1205         raise FDroidException("androguard library is not installed and aapt not present")
1206     except FileNotFoundError:
1207         logging.error(_("Could not open apk file for analysis"))
1208         raise BuildException(_("Invalid APK"))
1209
1210     apk['packageName'] = apkobject.get_package()
1211     apk['versionCode'] = int(apkobject.get_androidversion_code())
1212     apk['versionName'] = apkobject.get_androidversion_name()
1213     if apk['versionName'][0] == "@":
1214         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1215         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1216         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1217     apk['name'] = apkobject.get_app_name()
1218
1219     if apkobject.get_max_sdk_version() is not None:
1220         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1221     if apkobject.get_min_sdk_version() is not None:
1222         apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1223     if apkobject.get_target_sdk_version() is not None:
1224         apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1225
1226     icon_id_str = apkobject.get_element("application", "icon")
1227     if icon_id_str:
1228         icon_id = int(icon_id_str.replace("@", "0x"), 16)
1229         icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1230         apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1231
1232     arch_re = re.compile("^lib/(.*)/.*$")
1233     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1234     if len(arch) >= 1:
1235         apk['nativecode'] = []
1236         apk['nativecode'].extend(sorted(list(arch)))
1237
1238     xml = apkobject.get_android_manifest_xml()
1239
1240     for item in xml.findall('uses-permission'):
1241         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1242         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1243         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1244         permission = UsesPermission(
1245             name,
1246             maxSdkVersion
1247         )
1248         apk['uses-permission'].append(permission)
1249     for name, maxSdkVersion in apkobject.get_uses_implied_permission_list():
1250         permission = UsesPermission(
1251             name,
1252             maxSdkVersion
1253         )
1254         apk['uses-permission'].append(permission)
1255
1256     for item in xml.findall('uses-permission-sdk-23'):
1257         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1258         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1259         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1260         permission_sdk_23 = UsesPermissionSdk23(
1261             name,
1262             maxSdkVersion
1263         )
1264         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1265
1266     for item in xml.findall('uses-feature'):
1267         feature = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1268         if feature != "android.hardware.screen.portrait" \
1269                 and feature != "android.hardware.screen.landscape":
1270             if feature.startswith("android.feature."):
1271                 feature = feature[16:]
1272         required = item.attrib.get('{' + xml.nsmap['android'] + '}required')
1273         if required is None or required == 'true':
1274             apk['features'].append(feature)
1275
1276
1277 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1278                 allow_disabled_algorithms=False, archive_bad_sig=False):
1279     """Processes the apk with the given filename in the given repo directory.
1280
1281     This also extracts the icons.
1282
1283     :param apkcache: current apk cache information
1284     :param apkfilename: the filename of the apk to scan
1285     :param repodir: repo directory to scan
1286     :param knownapks: known apks info
1287     :param use_date_from_apk: use date from APK (instead of current date)
1288                               for newly added APKs
1289     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1290                                       disabled algorithms in the signature (e.g. MD5)
1291     :param archive_bad_sig: move APKs with a bad signature to the archive
1292     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1293      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1294     """
1295
1296     apk = {}
1297     apkfile = os.path.join(repodir, apkfilename)
1298
1299     cachechanged = False
1300     usecache = False
1301     if apkfilename in apkcache:
1302         apk = apkcache[apkfilename]
1303         if apk.get('hash') == sha256sum(apkfile):
1304             logging.debug(_("Reading {apkfilename} from cache")
1305                           .format(apkfilename=apkfilename))
1306             usecache = True
1307         else:
1308             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1309                           .format(apkfilename=apkfilename))
1310
1311     if not usecache:
1312         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1313
1314         try:
1315             apk = scan_apk(apkfile)
1316         except BuildException:
1317             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1318                             .format(apkfilename=apkfilename))
1319             return True, None, False
1320
1321         # Check for debuggable apks...
1322         if common.is_apk_and_debuggable(apkfile):
1323             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1324
1325         if options.rename_apks:
1326             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1327             std_short_name = os.path.join(repodir, n)
1328             if apkfile != std_short_name:
1329                 if os.path.exists(std_short_name):
1330                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1331                     if apkfile != std_long_name:
1332                         if os.path.exists(std_long_name):
1333                             dupdir = os.path.join('duplicates', repodir)
1334                             if not os.path.isdir(dupdir):
1335                                 os.makedirs(dupdir, exist_ok=True)
1336                             dupfile = os.path.join('duplicates', std_long_name)
1337                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1338                             os.rename(apkfile, dupfile)
1339                             return True, None, False
1340                         else:
1341                             os.rename(apkfile, std_long_name)
1342                     apkfile = std_long_name
1343                 else:
1344                     os.rename(apkfile, std_short_name)
1345                     apkfile = std_short_name
1346                 apkfilename = apkfile[len(repodir) + 1:]
1347
1348         apk['apkName'] = apkfilename
1349         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1350         if os.path.exists(os.path.join(repodir, srcfilename)):
1351             apk['srcname'] = srcfilename
1352
1353         # verify the jar signature is correct, allow deprecated
1354         # algorithms only if the APK is in the archive.
1355         skipapk = False
1356         if not common.verify_apk_signature(apkfile):
1357             if repodir == 'archive' or allow_disabled_algorithms:
1358                 if common.verify_old_apk_signature(apkfile):
1359                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1360                 else:
1361                     skipapk = True
1362             else:
1363                 skipapk = True
1364
1365         if skipapk:
1366             if archive_bad_sig:
1367                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1368                                 .format(apkfilename=apkfilename))
1369                 move_apk_between_sections(repodir, 'archive', apk)
1370             else:
1371                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1372                                 .format(apkfilename=apkfilename))
1373             return True, None, False
1374
1375         apkzip = zipfile.ZipFile(apkfile, 'r')
1376
1377         manifest = apkzip.getinfo('AndroidManifest.xml')
1378         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1379         if (1980, 0, 0) != manifest.date_time[0:3]:
1380             try:
1381                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1382             except ValueError as e:
1383                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1384                                 .format(apkfilename=apkfile) + str(e))
1385
1386         # extract icons from APK zip file
1387         iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
1388         try:
1389             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1390         finally:
1391             apkzip.close()  # ensure that APK zip file gets closed
1392
1393         # resize existing icons for densities missing in the APK
1394         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1395
1396         if use_date_from_apk and manifest.date_time[1] != 0:
1397             default_date_param = datetime(*manifest.date_time)
1398         else:
1399             default_date_param = None
1400
1401         # Record in known apks, getting the added date at the same time..
1402         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1403                                     default_date=default_date_param)
1404         if added:
1405             apk['added'] = added
1406
1407         apkcache[apkfilename] = apk
1408         cachechanged = True
1409
1410     return False, apk, cachechanged
1411
1412
1413 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1414     """Processes the apks in the given repo directory.
1415
1416     This also extracts the icons.
1417
1418     :param apkcache: current apk cache information
1419     :param repodir: repo directory to scan
1420     :param knownapks: known apks info
1421     :param use_date_from_apk: use date from APK (instead of current date)
1422                               for newly added APKs
1423     :returns: (apks, cachechanged) where apks is a list of apk information,
1424               and cachechanged is True if the apkcache got changed.
1425     """
1426
1427     cachechanged = False
1428
1429     for icon_dir in get_all_icon_dirs(repodir):
1430         if os.path.exists(icon_dir):
1431             if options.clean:
1432                 shutil.rmtree(icon_dir)
1433                 os.makedirs(icon_dir)
1434         else:
1435             os.makedirs(icon_dir)
1436
1437     apks = []
1438     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1439         apkfilename = apkfile[len(repodir) + 1:]
1440         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1441         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1442                                              use_date_from_apk, ada, True)
1443         if skip:
1444             continue
1445         apks.append(apk)
1446         cachechanged = cachechanged or cachethis
1447
1448     return apks, cachechanged
1449
1450
1451 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1452     """Extracts PNG icons from an APK with the supported pixel densities
1453
1454     Extracts icons from the given APK zip in various densities, saves
1455     them into given repo directory and stores their names in the APK
1456     metadata dictionary.  If the icon is an XML icon, then this tries
1457     to find PNG icon that can replace it.
1458
1459     :param icon_filename: A string representing the icon's file name
1460     :param apk: A populated dictionary containing APK metadata.
1461                 Needs to have 'icons_src' key
1462     :param apkzip: An opened zipfile.ZipFile of the APK file
1463     :param repo_dir: The directory of the APK's repository
1464     :return: A list of icon densities that are missing
1465
1466     """
1467     res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1468     pngs = dict()
1469     for f in apkzip.namelist():
1470         m = res_name_re.match(f)
1471         if m and m.group(4) == 'png':
1472             density = screen_resolutions[m.group(2)]
1473             pngs[m.group(3) + '/' + density] = m.group(0)
1474
1475     icon_type = None
1476     empty_densities = []
1477     for density in screen_densities:
1478         if density not in apk['icons_src']:
1479             empty_densities.append(density)
1480             continue
1481         icon_src = apk['icons_src'][density]
1482         icon_dir = get_icon_dir(repo_dir, density)
1483         icon_type = '.png'
1484
1485         # Extract the icon files per density
1486         if icon_src.endswith('.xml'):
1487             m = res_name_re.match(icon_src)
1488             if m:
1489                 name = pngs.get(m.group(3) + '/' + str(density))
1490                 if name:
1491                     icon_src = name
1492             if icon_src.endswith('.xml'):
1493                 empty_densities.append(density)
1494                 icon_type = '.xml'
1495         icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
1496
1497         try:
1498             with open(icon_dest, 'wb') as f:
1499                 f.write(get_icon_bytes(apkzip, icon_src))
1500             apk['icons'][density] = icon_filename + icon_type
1501         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1502             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1503             del apk['icons_src'][density]
1504             empty_densities.append(density)
1505
1506     # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
1507     if '-1' in apk['icons_src']:
1508         icon_src = apk['icons_src']['-1']
1509         icon_type = icon_src[-4:]
1510         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
1511         with open(icon_path, 'wb') as f:
1512             f.write(get_icon_bytes(apkzip, icon_src))
1513         if icon_type == '.png':
1514             im = None
1515             try:
1516                 im = Image.open(icon_path)
1517                 dpi = px_to_dpi(im.size[0])
1518                 for density in screen_densities:
1519                     if density in apk['icons']:
1520                         break
1521                     if density == screen_densities[-1] or dpi >= int(density):
1522                         apk['icons'][density] = icon_filename + icon_type
1523                         shutil.move(icon_path,
1524                                     os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
1525                         empty_densities.remove(density)
1526                         break
1527             except Exception as e:
1528                 logging.warning(_("Failed reading {path}: {error}")
1529                                 .format(path=icon_path, error=e))
1530             finally:
1531                 if im and hasattr(im, 'close'):
1532                     im.close()
1533
1534     if apk['icons']:
1535         apk['icon'] = icon_filename + icon_type
1536
1537     return empty_densities
1538
1539
1540 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1541     """
1542     Resize existing PNG icons for densities missing in the APK to ensure all densities are available
1543
1544     :param empty_densities: A list of icon densities that are missing
1545     :param icon_filename: A string representing the icon's file name
1546     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1547     :param repo_dir: The directory of the APK's repository
1548
1549     """
1550     icon_filename += '.png'
1551     # First try resizing down to not lose quality
1552     last_density = None
1553     for density in screen_densities:
1554         if density == '65534':  # not possible to generate 'anydpi' from other densities
1555             continue
1556         if density not in empty_densities:
1557             last_density = density
1558             continue
1559         if last_density is None:
1560             continue
1561         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1562
1563         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1564         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1565         fp = None
1566         try:
1567             fp = open(last_icon_path, 'rb')
1568             im = Image.open(fp)
1569
1570             size = dpi_to_px(density)
1571
1572             im.thumbnail((size, size), Image.ANTIALIAS)
1573             im.save(icon_path, "PNG", optimize=True,
1574                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1575             empty_densities.remove(density)
1576         except Exception as e:
1577             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1578         finally:
1579             if fp:
1580                 fp.close()
1581
1582     # Then just copy from the highest resolution available
1583     last_density = None
1584     for density in reversed(screen_densities):
1585         if density not in empty_densities:
1586             last_density = density
1587             continue
1588
1589         if last_density is None:
1590             continue
1591
1592         shutil.copyfile(
1593             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1594             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1595         )
1596         empty_densities.remove(density)
1597
1598     for density in screen_densities:
1599         icon_dir = get_icon_dir(repo_dir, density)
1600         icon_dest = os.path.join(icon_dir, icon_filename)
1601         resize_icon(icon_dest, density)
1602
1603     # Copy from icons-mdpi to icons since mdpi is the baseline density
1604     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1605     if os.path.isfile(baseline):
1606         apk['icons']['0'] = icon_filename
1607         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1608
1609
1610 def apply_info_from_latest_apk(apps, apks):
1611     """
1612     Some information from the apks needs to be applied up to the application level.
1613     When doing this, we use the info from the most recent version's apk.
1614     We deal with figuring out when the app was added and last updated at the same time.
1615     """
1616     for appid, app in apps.items():
1617         bestver = UNSET_VERSION_CODE
1618         for apk in apks:
1619             if apk['packageName'] == appid:
1620                 if apk['versionCode'] > bestver:
1621                     bestver = apk['versionCode']
1622                     bestapk = apk
1623
1624                 if 'added' in apk:
1625                     if not app.added or apk['added'] < app.added:
1626                         app.added = apk['added']
1627                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1628                         app.lastUpdated = apk['added']
1629
1630         if not app.added:
1631             logging.debug("Don't know when " + appid + " was added")
1632         if not app.lastUpdated:
1633             logging.debug("Don't know when " + appid + " was last updated")
1634
1635         if bestver == UNSET_VERSION_CODE:
1636
1637             if app.Name is None:
1638                 app.Name = app.AutoName or appid
1639             app.icon = None
1640             logging.debug("Application " + appid + " has no packages")
1641         else:
1642             if app.Name is None:
1643                 app.Name = bestapk['name']
1644             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1645             if app.CurrentVersionCode is None:
1646                 app.CurrentVersionCode = str(bestver)
1647
1648
1649 def make_categories_txt(repodir, categories):
1650     '''Write a category list in the repo to allow quick access'''
1651     catdata = ''
1652     for cat in sorted(categories):
1653         catdata += cat + '\n'
1654     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1655         f.write(catdata)
1656
1657
1658 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1659
1660     def filter_apk_list_sorted(apk_list):
1661         res = []
1662         for apk in apk_list:
1663             if apk['packageName'] == appid:
1664                 res.append(apk)
1665
1666         # Sort the apk list by version code. First is highest/newest.
1667         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1668
1669     for appid, app in apps.items():
1670
1671         if app.ArchivePolicy:
1672             keepversions = int(app.ArchivePolicy[:-9])
1673         else:
1674             keepversions = defaultkeepversions
1675
1676         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1677                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1678
1679         current_app_apks = filter_apk_list_sorted(apks)
1680         if len(current_app_apks) > keepversions:
1681             # Move back the ones we don't want.
1682             for apk in current_app_apks[keepversions:]:
1683                 move_apk_between_sections(repodir, archivedir, apk)
1684                 archapks.append(apk)
1685                 apks.remove(apk)
1686
1687         current_app_archapks = filter_apk_list_sorted(archapks)
1688         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1689             kept = 0
1690             # Move forward the ones we want again, except DisableAlgorithm
1691             for apk in current_app_archapks:
1692                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1693                     move_apk_between_sections(archivedir, repodir, apk)
1694                     archapks.remove(apk)
1695                     apks.append(apk)
1696                     kept += 1
1697                 if kept == keepversions:
1698                     break
1699
1700
1701 def move_apk_between_sections(from_dir, to_dir, apk):
1702     """move an APK from repo to archive or vice versa"""
1703
1704     def _move_file(from_dir, to_dir, filename, ignore_missing):
1705         from_path = os.path.join(from_dir, filename)
1706         if ignore_missing and not os.path.exists(from_path):
1707             return
1708         to_path = os.path.join(to_dir, filename)
1709         if not os.path.exists(to_dir):
1710             os.mkdir(to_dir)
1711         shutil.move(from_path, to_path)
1712
1713     if from_dir == to_dir:
1714         return
1715
1716     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1717     _move_file(from_dir, to_dir, apk['apkName'], False)
1718     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1719     for density in all_screen_densities:
1720         from_icon_dir = get_icon_dir(from_dir, density)
1721         to_icon_dir = get_icon_dir(to_dir, density)
1722         if density not in apk.get('icons', []):
1723             continue
1724         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1725     if 'srcname' in apk:
1726         _move_file(from_dir, to_dir, apk['srcname'], False)
1727
1728
1729 def add_apks_to_per_app_repos(repodir, apks):
1730     apks_per_app = dict()
1731     for apk in apks:
1732         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1733         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1734         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1735         apks_per_app[apk['packageName']] = apk
1736
1737         if not os.path.exists(apk['per_app_icons']):
1738             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1739             os.makedirs(apk['per_app_icons'])
1740
1741         apkpath = os.path.join(repodir, apk['apkName'])
1742         shutil.copy(apkpath, apk['per_app_repo'])
1743         apksigpath = apkpath + '.sig'
1744         if os.path.exists(apksigpath):
1745             shutil.copy(apksigpath, apk['per_app_repo'])
1746         apkascpath = apkpath + '.asc'
1747         if os.path.exists(apkascpath):
1748             shutil.copy(apkascpath, apk['per_app_repo'])
1749
1750
1751 def create_metadata_from_template(apk):
1752     '''create a new metadata file using internal or external template
1753
1754     Generate warnings for apk's with no metadata (or create skeleton
1755     metadata files, if requested on the command line).  Though the
1756     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1757     since those impose things on the metadata file made from the
1758     template: field sort order, empty field value, formatting, etc.
1759     '''
1760
1761     import yaml
1762     if os.path.exists('template.yml'):
1763         with open('template.yml') as f:
1764             metatxt = f.read()
1765         if 'name' in apk and apk['name'] != '':
1766             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1767                              r'\1 ' + apk['name'],
1768                              metatxt,
1769                              flags=re.IGNORECASE | re.MULTILINE)
1770         else:
1771             logging.warning(_('{appid} does not have a name! Using package name instead.')
1772                             .format(appid=apk['packageName']))
1773             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1774                              r'\1 ' + apk['packageName'],
1775                              metatxt,
1776                              flags=re.IGNORECASE | re.MULTILINE)
1777         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1778             f.write(metatxt)
1779     else:
1780         app = dict()
1781         app['Categories'] = [os.path.basename(os.getcwd())]
1782         # include some blanks as part of the template
1783         app['AuthorName'] = ''
1784         app['Summary'] = ''
1785         app['WebSite'] = ''
1786         app['IssueTracker'] = ''
1787         app['SourceCode'] = ''
1788         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1789         if 'name' in apk and apk['name'] != '':
1790             app['Name'] = apk['name']
1791         else:
1792             logging.warning(_('{appid} does not have a name! Using package name instead.')
1793                             .format(appid=apk['packageName']))
1794             app['Name'] = apk['packageName']
1795         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1796             yaml.dump(app, f, default_flow_style=False)
1797     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1798
1799
1800 config = None
1801 options = None
1802 start_timestamp = time.gmtime()
1803
1804
1805 def main():
1806
1807     global config, options
1808
1809     # Parse command line...
1810     parser = ArgumentParser()
1811     common.setup_global_opts(parser)
1812     parser.add_argument("--create-key", action="store_true", default=False,
1813                         help=_("Add a repo signing key to an unsigned repo"))
1814     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1815                         help=_("Add skeleton metadata files for APKs that are missing them"))
1816     parser.add_argument("--delete-unknown", action="store_true", default=False,
1817                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1818     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1819                         help=_("Report on build data status"))
1820     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1821                         help=_("Interactively ask about things that need updating."))
1822     parser.add_argument("-I", "--icons", action="store_true", default=False,
1823                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1824     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1825                         help=_("Specify editor to use in interactive mode. Default " +
1826                                "is {path}").format(path='/etc/alternatives/editor'))
1827     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1828                         help=_("Update the wiki"))
1829     parser.add_argument("--pretty", action="store_true", default=False,
1830                         help=_("Produce human-readable XML/JSON for index files"))
1831     parser.add_argument("--clean", action="store_true", default=False,
1832                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1833     parser.add_argument("--nosign", action="store_true", default=False,
1834                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1835     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1836                         help=_("Use date from APK instead of current time for newly added APKs"))
1837     parser.add_argument("--rename-apks", action="store_true", default=False,
1838                         help=_("Rename APK files that do not match package.name_123.apk"))
1839     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1840                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1841     metadata.add_metadata_arguments(parser)
1842     options = parser.parse_args()
1843     metadata.warnings_action = options.W
1844
1845     config = common.read_config(options)
1846
1847     if not ('jarsigner' in config and 'keytool' in config):
1848         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1849
1850     repodirs = ['repo']
1851     if config['archive_older'] != 0:
1852         repodirs.append('archive')
1853         if not os.path.exists('archive'):
1854             os.mkdir('archive')
1855
1856     if options.icons:
1857         resize_all_icons(repodirs)
1858         sys.exit(0)
1859
1860     if options.rename_apks:
1861         options.clean = True
1862
1863     # check that icons exist now, rather than fail at the end of `fdroid update`
1864     for k in ['repo_icon', 'archive_icon']:
1865         if k in config:
1866             if not os.path.exists(config[k]):
1867                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1868                                  .format(name=k, path=config[k]))
1869                 sys.exit(1)
1870
1871     # if the user asks to create a keystore, do it now, reusing whatever it can
1872     if options.create_key:
1873         if os.path.exists(config['keystore']):
1874             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1875             logging.critical("\t'" + config['keystore'] + "'")
1876             sys.exit(1)
1877
1878         if 'repo_keyalias' not in config:
1879             config['repo_keyalias'] = socket.getfqdn()
1880             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1881         if 'keydname' not in config:
1882             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1883             common.write_to_config(config, 'keydname', config['keydname'])
1884         if 'keystore' not in config:
1885             config['keystore'] = common.default_config['keystore']
1886             common.write_to_config(config, 'keystore', config['keystore'])
1887
1888         password = common.genpassword()
1889         if 'keystorepass' not in config:
1890             config['keystorepass'] = password
1891             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1892         if 'keypass' not in config:
1893             config['keypass'] = password
1894             common.write_to_config(config, 'keypass', config['keypass'])
1895         common.genkeystore(config)
1896
1897     # Get all apps...
1898     apps = metadata.read_metadata()
1899
1900     # Generate a list of categories...
1901     categories = set()
1902     for app in apps.values():
1903         categories.update(app.Categories)
1904
1905     # Read known apks data (will be updated and written back when we've finished)
1906     knownapks = common.KnownApks()
1907
1908     # Get APK cache
1909     apkcache = get_cache()
1910
1911     # Delete builds for disabled apps
1912     delete_disabled_builds(apps, apkcache, repodirs)
1913
1914     # Scan all apks in the main repo
1915     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1916
1917     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1918                                            options.use_date_from_apk)
1919     cachechanged = cachechanged or fcachechanged
1920     apks += files
1921     for apk in apks:
1922         if apk['packageName'] not in apps:
1923             if options.create_metadata:
1924                 create_metadata_from_template(apk)
1925                 apps = metadata.read_metadata()
1926             else:
1927                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1928                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1929                 if options.delete_unknown:
1930                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1931                                  .format(apkfilename=apk['apkName']))
1932                     rmf = os.path.join(repodirs[0], apk['apkName'])
1933                     if not os.path.exists(rmf):
1934                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1935                     else:
1936                         os.remove(rmf)
1937                 else:
1938                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1939
1940     copy_triple_t_store_metadata(apps)
1941     insert_obbs(repodirs[0], apps, apks)
1942     insert_localized_app_metadata(apps)
1943     translate_per_build_anti_features(apps, apks)
1944
1945     # Scan the archive repo for apks as well
1946     if len(repodirs) > 1:
1947         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1948         if cc:
1949             cachechanged = True
1950     else:
1951         archapks = []
1952
1953     # Apply information from latest apks to the application and update dates
1954     apply_info_from_latest_apk(apps, apks + archapks)
1955
1956     # Sort the app list by name, then the web site doesn't have to by default.
1957     # (we had to wait until we'd scanned the apks to do this, because mostly the
1958     # name comes from there!)
1959     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1960
1961     # APKs are placed into multiple repos based on the app package, providing
1962     # per-app subscription feeds for nightly builds and things like it
1963     if config['per_app_repos']:
1964         add_apks_to_per_app_repos(repodirs[0], apks)
1965         for appid, app in apps.items():
1966             repodir = os.path.join(appid, 'fdroid', 'repo')
1967             appdict = dict()
1968             appdict[appid] = app
1969             if os.path.isdir(repodir):
1970                 index.make(appdict, [appid], apks, repodir, False)
1971             else:
1972                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1973         return
1974
1975     if len(repodirs) > 1:
1976         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1977
1978     # Make the index for the main repo...
1979     index.make(apps, sortedids, apks, repodirs[0], False)
1980     make_categories_txt(repodirs[0], categories)
1981
1982     # If there's an archive repo,  make the index for it. We already scanned it
1983     # earlier on.
1984     if len(repodirs) > 1:
1985         index.make(apps, sortedids, archapks, repodirs[1], True)
1986
1987     git_remote = config.get('binary_transparency_remote')
1988     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1989         from . import btlog
1990         btlog.make_binary_transparency_log(repodirs)
1991
1992     if config['update_stats']:
1993         # Update known apks info...
1994         knownapks.writeifchanged()
1995
1996         # Generate latest apps data for widget
1997         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1998             data = ''
1999             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
2000                 for line in f:
2001                     appid = line.rstrip()
2002                     data += appid + "\t"
2003                     app = apps[appid]
2004                     data += app.Name + "\t"
2005                     if app.icon is not None:
2006                         data += app.icon + "\t"
2007                     data += app.License + "\n"
2008             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
2009                 f.write(data)
2010
2011     if cachechanged:
2012         write_cache(apkcache)
2013
2014     # Update the wiki...
2015     if options.wiki:
2016         update_wiki(apps, sortedids, apks + archapks)
2017
2018     logging.info(_("Finished"))
2019
2020
2021 if __name__ == "__main__":
2022     main()