fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*)'")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
  63 screen_resolutions = {
  64     "xxxhdpi": '640',
  65     "xxhdpi": '480',
  66     "xhdpi": '320',
  67     "hdpi": '240',
  68     "mdpi": '160',
  69     "ldpi": '120',
  70     "undefined": '-1',
  71     "anydpi": '65534',
  72     "nodpi": '65535'
  73 }
  74
  75 all_screen_densities = ['0'] + screen_densities
  76
  77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  79
  80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  84
  85 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  86
  87
  88 def dpi_to_px(density):
  89     return (int(density) * 48) / 160
  90
  91
  92 def px_to_dpi(px):
  93     return (int(px) * 160) / 48
  94
  95
  96 def get_icon_dir(repodir, density):
  97     if density == '0' or density == '65534':
  98         return os.path.join(repodir, "icons")
  99     else:
 100         return os.path.join(repodir, "icons-%s" % density)
 101
 102
 103 def get_icon_dirs(repodir):
 104     for density in screen_densities:
 105         yield get_icon_dir(repodir, density)
 106
 107
 108 def get_all_icon_dirs(repodir):
 109     for density in all_screen_densities:
 110         yield get_icon_dir(repodir, density)
 111
 112
 113 def update_wiki(apps, sortedids, apks):
 114     """Update the wiki
 115
 116     :param apps: fully populated list of all applications
 117     :param apks: all apks, except...
 118     """
 119     logging.info("Updating wiki")
 120     wikicat = 'Apps'
 121     wikiredircat = 'App Redirects'
 122     import mwclient
 123     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 124                          path=config['wiki_path'])
 125     site.login(config['wiki_user'], config['wiki_password'])
 126     generated_pages = {}
 127     generated_redirects = {}
 128
 129     for appid in sortedids:
 130         app = metadata.App(apps[appid])
 131
 132         wikidata = ''
 133         if app.Disabled:
 134             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 135         if app.AntiFeatures:
 136             for af in sorted(app.AntiFeatures):
 137                 wikidata += '{{AntiFeature|' + af + '}}\n'
 138         if app.RequiresRoot:
 139             requiresroot = 'Yes'
 140         else:
 141             requiresroot = 'No'
 142         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 143             appid,
 144             app.Name,
 145             app.added.strftime('%Y-%m-%d') if app.added else '',
 146             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 147             app.SourceCode,
 148             app.IssueTracker,
 149             app.WebSite,
 150             app.Changelog,
 151             app.Donate,
 152             app.FlattrID,
 153             app.LiberapayID,
 154             app.Bitcoin,
 155             app.Litecoin,
 156             app.License,
 157             requiresroot,
 158             app.AuthorName,
 159             app.AuthorEmail)
 160
 161         if app.Provides:
 162             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 163
 164         wikidata += app.Summary
 165         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 166
 167         wikidata += "=Description=\n"
 168         wikidata += metadata.description_wiki(app.Description) + "\n"
 169
 170         wikidata += "=Maintainer Notes=\n"
 171         if app.MaintainerNotes:
 172             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 173         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 174
 175         # Get a list of all packages for this application...
 176         apklist = []
 177         gotcurrentver = False
 178         cantupdate = False
 179         buildfails = False
 180         for apk in apks:
 181             if apk['packageName'] == appid:
 182                 if str(apk['versionCode']) == app.CurrentVersionCode:
 183                     gotcurrentver = True
 184                 apklist.append(apk)
 185         # Include ones we can't build, as a special case...
 186         for build in app.builds:
 187             if build.disable:
 188                 if build.versionCode == app.CurrentVersionCode:
 189                     cantupdate = True
 190                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 191                 apklist.append({'versionCode': int(build.versionCode),
 192                                 'versionName': build.versionName,
 193                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 194                                 })
 195             else:
 196                 builtit = False
 197                 for apk in apklist:
 198                     if apk['versionCode'] == int(build.versionCode):
 199                         builtit = True
 200                         break
 201                 if not builtit:
 202                     buildfails = True
 203                     apklist.append({'versionCode': int(build.versionCode),
 204                                     'versionName': build.versionName,
 205                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 206                                     })
 207         if app.CurrentVersionCode == '0':
 208             cantupdate = True
 209         # Sort with most recent first...
 210         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 211
 212         wikidata += "=Versions=\n"
 213         if len(apklist) == 0:
 214             wikidata += "We currently have no versions of this app available."
 215         elif not gotcurrentver:
 216             wikidata += "We don't have the current version of this app."
 217         else:
 218             wikidata += "We have the current version of this app."
 219         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 220         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 221         if len(app.NoSourceSince) > 0:
 222             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 223         if len(app.CurrentVersion) > 0:
 224             wikidata += "The current (recommended) version is " + app.CurrentVersion
 225             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 226         validapks = 0
 227         for apk in apklist:
 228             wikidata += "==" + apk['versionName'] + "==\n"
 229
 230             if 'buildproblem' in apk:
 231                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 232             else:
 233                 validapks += 1
 234                 wikidata += "This version is built and signed by "
 235                 if 'srcname' in apk:
 236                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 237                 else:
 238                     wikidata += "the original developer.\n\n"
 239             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 240
 241         wikidata += '\n[[Category:' + wikicat + ']]\n'
 242         if len(app.NoSourceSince) > 0:
 243             wikidata += '\n[[Category:Apps missing source code]]\n'
 244         if validapks == 0 and not app.Disabled:
 245             wikidata += '\n[[Category:Apps with no packages]]\n'
 246         if cantupdate and not app.Disabled:
 247             wikidata += "\n[[Category:Apps we cannot update]]\n"
 248         if buildfails and not app.Disabled:
 249             wikidata += "\n[[Category:Apps with failing builds]]\n"
 250         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 251             wikidata += '\n[[Category:Apps to Update]]\n'
 252         if app.Disabled:
 253             wikidata += '\n[[Category:Apps that are disabled]]\n'
 254         if app.UpdateCheckMode == 'None' and not app.Disabled:
 255             wikidata += '\n[[Category:Apps with no update check]]\n'
 256         for appcat in app.Categories:
 257             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 258
 259         # We can't have underscores in the page name, even if they're in
 260         # the package ID, because MediaWiki messes with them...
 261         pagename = appid.replace('_', ' ')
 262
 263         # Drop a trailing newline, because mediawiki is going to drop it anyway
 264         # and it we don't we'll think the page has changed when it hasn't...
 265         if wikidata.endswith('\n'):
 266             wikidata = wikidata[:-1]
 267
 268         generated_pages[pagename] = wikidata
 269
 270         # Make a redirect from the name to the ID too, unless there's
 271         # already an existing page with the name and it isn't a redirect.
 272         noclobber = False
 273         apppagename = app.Name.replace('_', ' ')
 274         apppagename = apppagename.replace('{', '')
 275         apppagename = apppagename.replace('}', ' ')
 276         apppagename = apppagename.replace(':', ' ')
 277         apppagename = apppagename.replace('[', ' ')
 278         apppagename = apppagename.replace(']', ' ')
 279         # Drop double spaces caused mostly by replacing ':' above
 280         apppagename = apppagename.replace('  ', ' ')
 281         for expagename in site.allpages(prefix=apppagename,
 282                                         filterredir='nonredirects',
 283                                         generator=False):
 284             if expagename == apppagename:
 285                 noclobber = True
 286         # Another reason not to make the redirect page is if the app name
 287         # is the same as it's ID, because that will overwrite the real page
 288         # with an redirect to itself! (Although it seems like an odd
 289         # scenario this happens a lot, e.g. where there is metadata but no
 290         # builds or binaries to extract a name from.
 291         if apppagename == pagename:
 292             noclobber = True
 293         if not noclobber:
 294             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 295
 296     for tcat, genp in [(wikicat, generated_pages),
 297                        (wikiredircat, generated_redirects)]:
 298         catpages = site.Pages['Category:' + tcat]
 299         existingpages = []
 300         for page in catpages:
 301             existingpages.append(page.name)
 302             if page.name in genp:
 303                 pagetxt = page.edit()
 304                 if pagetxt != genp[page.name]:
 305                     logging.debug("Updating modified page " + page.name)
 306                     page.save(genp[page.name], summary='Auto-updated')
 307                 else:
 308                     logging.debug("Page " + page.name + " is unchanged")
 309             else:
 310                 logging.warn("Deleting page " + page.name)
 311                 page.delete('No longer published')
 312         for pagename, text in genp.items():
 313             logging.debug("Checking " + pagename)
 314             if pagename not in existingpages:
 315                 logging.debug("Creating page " + pagename)
 316                 try:
 317                     newpage = site.Pages[pagename]
 318                     newpage.save(text, summary='Auto-created')
 319                 except Exception as e:
 320                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 321
 322     # Purge server cache to ensure counts are up to date
 323     site.Pages['Repository Maintenance'].purge()
 324
 325     # Write a page with the last build log for this version code
 326     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 327     newpage = site.Pages[wiki_page_path]
 328     txt = ''
 329     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 330     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 331     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 332     txt += common.get_git_describe_link()
 333     txt += "\n\n"
 334     txt += common.get_android_tools_version_log()
 335     newpage.save(txt, summary='Run log')
 336     newpage = site.Pages['update']
 337     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 338
 339
 340 def delete_disabled_builds(apps, apkcache, repodirs):
 341     """Delete disabled build outputs.
 342
 343     :param apps: list of all applications, as per metadata.read_metadata
 344     :param apkcache: current apk cache information
 345     :param repodirs: the repo directories to process
 346     """
 347     for appid, app in apps.items():
 348         for build in app['builds']:
 349             if not build.disable:
 350                 continue
 351             apkfilename = common.get_release_filename(app, build)
 352             iconfilename = "%s.%s.png" % (
 353                 appid,
 354                 build.versionCode)
 355             for repodir in repodirs:
 356                 files = [
 357                     os.path.join(repodir, apkfilename),
 358                     os.path.join(repodir, apkfilename + '.asc'),
 359                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 360                 ]
 361                 for density in all_screen_densities:
 362                     repo_dir = get_icon_dir(repodir, density)
 363                     files.append(os.path.join(repo_dir, iconfilename))
 364
 365                 for f in files:
 366                     if os.path.exists(f):
 367                         logging.info("Deleting disabled build output " + f)
 368                         os.remove(f)
 369             if apkfilename in apkcache:
 370                 del apkcache[apkfilename]
 371
 372
 373 def resize_icon(iconpath, density):
 374
 375     if not os.path.isfile(iconpath):
 376         return
 377
 378     fp = None
 379     try:
 380         fp = open(iconpath, 'rb')
 381         im = Image.open(fp)
 382         size = dpi_to_px(density)
 383
 384         if any(length > size for length in im.size):
 385             oldsize = im.size
 386             im.thumbnail((size, size), Image.ANTIALIAS)
 387             logging.debug("%s was too large at %s - new size is %s" % (
 388                 iconpath, oldsize, im.size))
 389             im.save(iconpath, "PNG", optimize=True,
 390                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 391
 392     except Exception as e:
 393         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 394
 395     finally:
 396         if fp:
 397             fp.close()
 398
 399
 400 def resize_all_icons(repodirs):
 401     """Resize all icons that exceed the max size
 402
 403     :param repodirs: the repo directories to process
 404     """
 405     for repodir in repodirs:
 406         for density in screen_densities:
 407             icon_dir = get_icon_dir(repodir, density)
 408             icon_glob = os.path.join(icon_dir, '*.png')
 409             for iconpath in glob.glob(icon_glob):
 410                 resize_icon(iconpath, density)
 411
 412
 413 def getsig(apkpath):
 414     """ Get the signing certificate of an apk. To get the same md5 has that
 415     Android gets, we encode the .RSA certificate in a specific format and pass
 416     it hex-encoded to the md5 digest algorithm.
 417
 418     :param apkpath: path to the apk
 419     :returns: A string containing the md5 of the signature of the apk or None
 420               if an error occurred.
 421     """
 422
 423     with zipfile.ZipFile(apkpath, 'r') as apk:
 424         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 425
 426         if len(certs) < 1:
 427             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 428             return None
 429         if len(certs) > 1:
 430             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 431             return None
 432
 433         cert = apk.read(certs[0])
 434
 435     cert_encoded = common.get_certificate(cert)
 436
 437     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 438
 439
 440 def get_cache_file():
 441     return os.path.join('tmp', 'apkcache')
 442
 443
 444 def get_cache():
 445     """Get the cached dict of the APK index
 446
 447     Gather information about all the apk files in the repo directory,
 448     using cached data if possible. Some of the index operations take a
 449     long time, like calculating the SHA-256 and verifying the APK
 450     signature.
 451
 452     The cache is invalidated if the metadata version is different, or
 453     the 'allow_disabled_algorithms' config/option is different.  In
 454     those cases, there is no easy way to know what has changed from
 455     the cache, so just rerun the whole thing.
 456
 457     :return: apkcache
 458
 459     """
 460     apkcachefile = get_cache_file()
 461     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 462     if not options.clean and os.path.exists(apkcachefile):
 463         with open(apkcachefile, 'rb') as cf:
 464             apkcache = pickle.load(cf, encoding='utf-8')
 465         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 466            or apkcache.get('allow_disabled_algorithms') != ada:
 467             apkcache = {}
 468     else:
 469         apkcache = {}
 470
 471     apkcache["METADATA_VERSION"] = METADATA_VERSION
 472     apkcache['allow_disabled_algorithms'] = ada
 473
 474     return apkcache
 475
 476
 477 def write_cache(apkcache):
 478     apkcachefile = get_cache_file()
 479     cache_path = os.path.dirname(apkcachefile)
 480     if not os.path.exists(cache_path):
 481         os.makedirs(cache_path)
 482     with open(apkcachefile, 'wb') as cf:
 483         pickle.dump(apkcache, cf)
 484
 485
 486 def get_icon_bytes(apkzip, iconsrc):
 487     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 488     try:
 489         return apkzip.read(iconsrc)
 490     except KeyError:
 491         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 492
 493
 494 def sha256sum(filename):
 495     '''Calculate the sha256 of the given file'''
 496     sha = hashlib.sha256()
 497     with open(filename, 'rb') as f:
 498         while True:
 499             t = f.read(16384)
 500             if len(t) == 0:
 501                 break
 502             sha.update(t)
 503     return sha.hexdigest()
 504
 505
 506 def has_known_vulnerability(filename):
 507     """checks for known vulnerabilities in the APK
 508
 509     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 510     version.  Google also enforces this:
 511     https://support.google.com/faqs/answer/6376725?hl=en
 512
 513     Checks whether there are more than one classes.dex or AndroidManifest.xml
 514     files, which is invalid and an essential part of the "Master Key" attack.
 515     http://www.saurik.com/id/17
 516
 517     Janus is similar to Master Key but is perhaps easier to scan for.
 518     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 519     """
 520
 521     found_vuln = False
 522
 523     # statically load this pattern
 524     if not hasattr(has_known_vulnerability, "pattern"):
 525         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 526
 527     with open(filename.encode(), 'rb') as fp:
 528         first4 = fp.read(4)
 529     if first4 != b'\x50\x4b\x03\x04':
 530         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 531                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 532                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 533
 534     files_in_apk = set()
 535     with zipfile.ZipFile(filename) as zf:
 536         for name in zf.namelist():
 537             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 538                 lib = zf.open(name)
 539                 while True:
 540                     chunk = lib.read(4096)
 541                     if chunk == b'':
 542                         break
 543                     m = has_known_vulnerability.pattern.search(chunk)
 544                     if m:
 545                         version = m.group(1).decode('ascii')
 546                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 547                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 548                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 549                             logging.debug(_('"{path}" contains recent {name} ({version})')
 550                                           .format(path=filename, name=name, version=version))
 551                         else:
 552                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 553                                             .format(path=filename, name=name, version=version))
 554                             found_vuln = True
 555                         break
 556             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 557                 if name in files_in_apk:
 558                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 559                                     .format(apkfilename=filename, name=name))
 560                     found_vuln = True
 561                 files_in_apk.add(name)
 562     return found_vuln
 563
 564
 565 def insert_obbs(repodir, apps, apks):
 566     """Scans the .obb files in a given repo directory and adds them to the
 567     relevant APK instances.  OBB files have versionCodes like APK
 568     files, and they are loosely associated.  If there is an OBB file
 569     present, then any APK with the same or higher versionCode will use
 570     that OBB file.  There are two OBB types: main and patch, each APK
 571     can only have only have one of each.
 572
 573     https://developer.android.com/google/play/expansion-files.html
 574
 575     :param repodir: repo directory to scan
 576     :param apps: list of current, valid apps
 577     :param apks: current information on all APKs
 578
 579     """
 580
 581     def obbWarnDelete(f, msg):
 582         logging.warning(msg + ' ' + f)
 583         if options.delete_unknown:
 584             logging.error(_("Deleting unknown file: {path}").format(path=f))
 585             os.remove(f)
 586
 587     obbs = []
 588     java_Integer_MIN_VALUE = -pow(2, 31)
 589     currentPackageNames = apps.keys()
 590     for f in glob.glob(os.path.join(repodir, '*.obb')):
 591         obbfile = os.path.basename(f)
 592         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 593         chunks = obbfile.split('.')
 594         if chunks[0] != 'main' and chunks[0] != 'patch':
 595             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 596             continue
 597         if not re.match(r'^-?[0-9]+$', chunks[1]):
 598             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 599                           .format(name=chunks[0]))
 600             continue
 601         versionCode = int(chunks[1])
 602         packagename = ".".join(chunks[2:-1])
 603
 604         highestVersionCode = java_Integer_MIN_VALUE
 605         if packagename not in currentPackageNames:
 606             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 607             continue
 608         for apk in apks:
 609             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 610                 highestVersionCode = apk['versionCode']
 611         if versionCode > highestVersionCode:
 612             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 613                           .format(integer=str(versionCode)))
 614             continue
 615         obbsha256 = sha256sum(f)
 616         obbs.append((packagename, versionCode, obbfile, obbsha256))
 617
 618     for apk in apks:
 619         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 620             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 621                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 622                     apk['obbMainFile'] = obbfile
 623                     apk['obbMainFileSha256'] = obbsha256
 624                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 625                     apk['obbPatchFile'] = obbfile
 626                     apk['obbPatchFileSha256'] = obbsha256
 627             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 628                 break
 629
 630
 631 def translate_per_build_anti_features(apps, apks):
 632     """Grab the anti-features list from the build metadata
 633
 634     For most Anti-Features, they are really most applicable per-APK,
 635     not for an app.  An app can fix a vulnerability, add/remove
 636     tracking, etc.  This reads the 'antifeatures' list from the Build
 637     entries in the fdroiddata metadata file, then transforms it into
 638     the 'antiFeatures' list of unique items for the index.
 639
 640     The field key is all lower case in the metadata file to match the
 641     rest of the Build fields.  It is 'antiFeatures' camel case in the
 642     implementation, index, and fdroidclient since it is translated
 643     from the build 'antifeatures' field, not directly included.
 644
 645     """
 646
 647     antiFeatures = dict()
 648     for packageName, app in apps.items():
 649         d = dict()
 650         for build in app['builds']:
 651             afl = build.get('antifeatures')
 652             if afl:
 653                 d[int(build.versionCode)] = afl
 654         if len(d) > 0:
 655             antiFeatures[packageName] = d
 656
 657     for apk in apks:
 658         d = antiFeatures.get(apk['packageName'])
 659         if d:
 660             afl = d.get(apk['versionCode'])
 661             if afl:
 662                 apk['antiFeatures'].update(afl)
 663
 664
 665 def _get_localized_dict(app, locale):
 666     '''get the dict to add localized store metadata to'''
 667     if 'localized' not in app:
 668         app['localized'] = collections.OrderedDict()
 669     if locale not in app['localized']:
 670         app['localized'][locale] = collections.OrderedDict()
 671     return app['localized'][locale]
 672
 673
 674 def _set_localized_text_entry(app, locale, key, f):
 675     limit = config['char_limits'][key]
 676     localized = _get_localized_dict(app, locale)
 677     with open(f) as fp:
 678         text = fp.read()[:limit]
 679         if len(text) > 0:
 680             localized[key] = text
 681
 682
 683 def _set_author_entry(app, key, f):
 684     limit = config['char_limits']['author']
 685     with open(f) as fp:
 686         text = fp.read()[:limit]
 687         if len(text) > 0:
 688             app[key] = text
 689
 690
 691 def _strip_and_copy_image(inpath, outpath):
 692     """Remove any metadata from image and copy it to new path
 693
 694     Sadly, image metadata like EXIF can be used to exploit devices.
 695     It is not used at all in the F-Droid ecosystem, so its much safer
 696     just to remove it entirely.
 697
 698     """
 699
 700     extension = common.get_extension(inpath)[1]
 701     if os.path.isdir(outpath):
 702         outpath = os.path.join(outpath, os.path.basename(inpath))
 703     if extension == 'png':
 704         with open(inpath, 'rb') as fp:
 705             in_image = Image.open(fp)
 706             in_image.save(outpath, "PNG", optimize=True,
 707                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 708     elif extension == 'jpg' or extension == 'jpeg':
 709         with open(inpath, 'rb') as fp:
 710             in_image = Image.open(fp)
 711             data = list(in_image.getdata())
 712             out_image = Image.new(in_image.mode, in_image.size)
 713         out_image.putdata(data)
 714         out_image.save(outpath, "JPEG", optimize=True)
 715     else:
 716         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 717                               .format(extension=extension))
 718
 719
 720 def copy_triple_t_store_metadata(apps):
 721     """Include store metadata from the app's source repo
 722
 723     The Triple-T Gradle Play Publisher is a plugin that has a standard
 724     file layout for all of the metadata and graphics that the Google
 725     Play Store accepts.  Since F-Droid has the git repo, it can just
 726     pluck those files directly.  This method reads any text files into
 727     the app dict, then copies any graphics into the fdroid repo
 728     directory structure.
 729
 730     This needs to be run before insert_localized_app_metadata() so that
 731     the graphics files that are copied into the fdroid repo get
 732     properly indexed.
 733
 734     https://github.com/Triple-T/gradle-play-publisher#upload-images
 735     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 736
 737     """
 738
 739     if not os.path.isdir('build'):
 740         return  # nothing to do
 741
 742     for packageName, app in apps.items():
 743         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 744             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 745             for root, dirs, files in os.walk(d):
 746                 segments = root.split('/')
 747                 locale = segments[-2]
 748                 for f in files:
 749                     if f == 'fulldescription':
 750                         _set_localized_text_entry(app, locale, 'description',
 751                                                   os.path.join(root, f))
 752                         continue
 753                     elif f == 'shortdescription':
 754                         _set_localized_text_entry(app, locale, 'summary',
 755                                                   os.path.join(root, f))
 756                         continue
 757                     elif f == 'title':
 758                         _set_localized_text_entry(app, locale, 'name',
 759                                                   os.path.join(root, f))
 760                         continue
 761                     elif f == 'video':
 762                         _set_localized_text_entry(app, locale, 'video',
 763                                                   os.path.join(root, f))
 764                         continue
 765                     elif f == 'whatsnew':
 766                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 767                                                   os.path.join(root, f))
 768                         continue
 769                     elif f == 'contactEmail':
 770                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 771                         continue
 772                     elif f == 'contactPhone':
 773                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 774                         continue
 775                     elif f == 'contactWebsite':
 776                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 777                         continue
 778
 779                     base, extension = common.get_extension(f)
 780                     dirname = os.path.basename(root)
 781                     if extension in ALLOWED_EXTENSIONS \
 782                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 783                         if segments[-2] == 'listing':
 784                             locale = segments[-3]
 785                         else:
 786                             locale = segments[-2]
 787                         destdir = os.path.join('repo', packageName, locale, dirname)
 788                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 789                         sourcefile = os.path.join(root, f)
 790                         destfile = os.path.join(destdir, os.path.basename(f))
 791                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 792                         _strip_and_copy_image(sourcefile, destfile)
 793
 794
 795 def insert_localized_app_metadata(apps):
 796     """scans standard locations for graphics and localized text
 797
 798     Scans for localized description files, store graphics, and
 799     screenshot PNG files in statically defined screenshots directory
 800     and adds them to the app metadata.  The screenshots and graphic
 801     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 802     and must be in the following layout:
 803     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 804
 805     repo/packageName/locale/featureGraphic.png
 806     repo/packageName/locale/phoneScreenshots/1.png
 807     repo/packageName/locale/phoneScreenshots/2.png
 808
 809     The changelog files must be text files named with the versionCode
 810     ending with ".txt" and must be in the following layout:
 811     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 812
 813     repo/packageName/locale/changelogs/12345.txt
 814
 815     This will scan the each app's source repo then the metadata/ dir
 816     for these standard locations of changelog files.  If it finds
 817     them, they will be added to the dict of all packages, with the
 818     versions in the metadata/ folder taking precendence over the what
 819     is in the app's source repo.
 820
 821     Where "packageName" is the app's packageName and "locale" is the locale
 822     of the graphics, e.g. what language they are in, using the IETF RFC5646
 823     format (en-US, fr-CA, es-MX, etc).
 824
 825     This will also scan the app's git for a fastlane folder, and the
 826     metadata/ folder and the apps' source repos for standard locations
 827     of graphic and screenshot files.  If it finds them, it will copy
 828     them into the repo.  The fastlane files follow this pattern:
 829     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 830
 831     """
 832
 833     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 834     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 837
 838     for srcd in sorted(sourcedirs):
 839         if not os.path.isdir(srcd):
 840             continue
 841         for root, dirs, files in os.walk(srcd):
 842             segments = root.split('/')
 843             packageName = segments[1]
 844             if packageName not in apps:
 845                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 846                 continue
 847             locale = segments[-1]
 848             destdir = os.path.join('repo', packageName, locale)
 849
 850             # flavours specified in build receipt
 851             build_flavours = ""
 852             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 853                     and 'gradle' in apps[packageName].builds[-1]:
 854                 build_flavours = apps[packageName].builds[-1].gradle
 855
 856             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 857                 logging.debug("ignoring due to wrong flavour")
 858                 continue
 859
 860             for f in files:
 861                 if f in ('description.txt', 'full_description.txt'):
 862                     _set_localized_text_entry(apps[packageName], locale, 'description',
 863                                               os.path.join(root, f))
 864                     continue
 865                 elif f in ('summary.txt', 'short_description.txt'):
 866                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 867                                               os.path.join(root, f))
 868                     continue
 869                 elif f in ('name.txt', 'title.txt'):
 870                     _set_localized_text_entry(apps[packageName], locale, 'name',
 871                                               os.path.join(root, f))
 872                     continue
 873                 elif f == 'video.txt':
 874                     _set_localized_text_entry(apps[packageName], locale, 'video',
 875                                               os.path.join(root, f))
 876                     continue
 877                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 878                     locale = segments[-2]
 879                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 880                                               os.path.join(root, f))
 881                     continue
 882
 883                 base, extension = common.get_extension(f)
 884                 if locale == 'images':
 885                     locale = segments[-2]
 886                     destdir = os.path.join('repo', packageName, locale)
 887                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 888                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 889                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 890                     _strip_and_copy_image(os.path.join(root, f), destdir)
 891             for d in dirs:
 892                 if d in SCREENSHOT_DIRS:
 893                     if locale == 'images':
 894                         locale = segments[-2]
 895                         destdir = os.path.join('repo', packageName, locale)
 896                     for f in glob.glob(os.path.join(root, d, '*.*')):
 897                         _ignored, extension = common.get_extension(f)
 898                         if extension in ALLOWED_EXTENSIONS:
 899                             screenshotdestdir = os.path.join(destdir, d)
 900                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 901                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 902                             _strip_and_copy_image(f, screenshotdestdir)
 903
 904     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 905     for d in repofiles:
 906         if not os.path.isdir(d):
 907             continue
 908         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 909             if not os.path.isfile(f):
 910                 continue
 911             segments = f.split('/')
 912             packageName = segments[1]
 913             locale = segments[2]
 914             screenshotdir = segments[3]
 915             filename = os.path.basename(f)
 916             base, extension = common.get_extension(filename)
 917
 918             if packageName not in apps:
 919                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 920                                 .format(path=filename, name=packageName))
 921                 continue
 922             graphics = _get_localized_dict(apps[packageName], locale)
 923
 924             if extension not in ALLOWED_EXTENSIONS:
 925                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 926             elif base in GRAPHIC_NAMES:
 927                 # there can only be zero or one of these per locale
 928                 graphics[base] = filename
 929             elif screenshotdir in SCREENSHOT_DIRS:
 930                 # there can any number of these per locale
 931                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 932                 if screenshotdir not in graphics:
 933                     graphics[screenshotdir] = []
 934                 graphics[screenshotdir].append(filename)
 935             else:
 936                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 937
 938
 939 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 940     """Scan a repo for all files with an extension except APK/OBB
 941
 942     :param apkcache: current cached info about all repo files
 943     :param repodir: repo directory to scan
 944     :param knownapks: list of all known files, as per metadata.read_metadata
 945     :param use_date_from_file: use date from file (instead of current date)
 946                                for newly added files
 947     """
 948
 949     cachechanged = False
 950     repo_files = []
 951     repodir = repodir.encode('utf-8')
 952     for name in os.listdir(repodir):
 953         file_extension = common.get_file_extension(name)
 954         if file_extension == 'apk' or file_extension == 'obb':
 955             continue
 956         filename = os.path.join(repodir, name)
 957         name_utf8 = name.decode('utf-8')
 958         if filename.endswith(b'_src.tar.gz'):
 959             logging.debug(_('skipping source tarball: {path}')
 960                           .format(path=filename.decode('utf-8')))
 961             continue
 962         if not common.is_repo_file(filename):
 963             continue
 964         stat = os.stat(filename)
 965         if stat.st_size == 0:
 966             raise FDroidException(_('{path} is zero size!')
 967                                   .format(path=filename))
 968
 969         shasum = sha256sum(filename)
 970         usecache = False
 971         if name in apkcache:
 972             repo_file = apkcache[name]
 973             # added time is cached as tuple but used here as datetime instance
 974             if 'added' in repo_file:
 975                 a = repo_file['added']
 976                 if isinstance(a, datetime):
 977                     repo_file['added'] = a
 978                 else:
 979                     repo_file['added'] = datetime(*a[:6])
 980             if repo_file.get('hash') == shasum:
 981                 logging.debug(_("Reading {apkfilename} from cache")
 982                               .format(apkfilename=name_utf8))
 983                 usecache = True
 984             else:
 985                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 986                               .format(apkfilename=name_utf8))
 987
 988         if not usecache:
 989             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 990             repo_file = collections.OrderedDict()
 991             repo_file['name'] = os.path.splitext(name_utf8)[0]
 992             # TODO rename apkname globally to something more generic
 993             repo_file['apkName'] = name_utf8
 994             repo_file['hash'] = shasum
 995             repo_file['hashType'] = 'sha256'
 996             repo_file['versionCode'] = 0
 997             repo_file['versionName'] = shasum
 998             # the static ID is the SHA256 unless it is set in the metadata
 999             repo_file['packageName'] = shasum
1000
1001             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1002             if m:
1003                 repo_file['packageName'] = m.group(1)
1004                 repo_file['versionCode'] = int(m.group(2))
1005             srcfilename = name + b'_src.tar.gz'
1006             if os.path.exists(os.path.join(repodir, srcfilename)):
1007                 repo_file['srcname'] = srcfilename.decode('utf-8')
1008             repo_file['size'] = stat.st_size
1009
1010             apkcache[name] = repo_file
1011             cachechanged = True
1012
1013         if use_date_from_file:
1014             timestamp = stat.st_ctime
1015             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1016         else:
1017             default_date_param = None
1018
1019         # Record in knownapks, getting the added date at the same time..
1020         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1021                                     default_date=default_date_param)
1022         if added:
1023             repo_file['added'] = added
1024
1025         repo_files.append(repo_file)
1026
1027     return repo_files, cachechanged
1028
1029
1030 def scan_apk(apk_file):
1031     """
1032     Scans an APK file and returns dictionary with metadata of the APK.
1033
1034     Attention: This does *not* verify that the APK signature is correct.
1035
1036     :param apk_file: The (ideally absolute) path to the APK file
1037     :raises BuildException
1038     :return A dict containing APK metadata
1039     """
1040     apk = {
1041         'hash': sha256sum(apk_file),
1042         'hashType': 'sha256',
1043         'uses-permission': [],
1044         'uses-permission-sdk-23': [],
1045         'features': [],
1046         'icons_src': {},
1047         'icons': {},
1048         'antiFeatures': set(),
1049     }
1050
1051     if common.use_androguard():
1052         scan_apk_androguard(apk, apk_file)
1053     else:
1054         scan_apk_aapt(apk, apk_file)
1055
1056     # Get the signature, or rather the signing key fingerprints
1057     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1058     apk['sig'] = getsig(apk_file)
1059     if not apk['sig']:
1060         raise BuildException("Failed to get apk signature")
1061     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1062                                                                apk_file))
1063     if not apk.get('signer'):
1064         raise BuildException("Failed to get apk signing key fingerprint")
1065
1066     # Get size of the APK
1067     apk['size'] = os.path.getsize(apk_file)
1068
1069     if 'minSdkVersion' not in apk:
1070         logging.warning("No SDK version information found in {0}".format(apk_file))
1071         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
1072     if 'targetSdkVersion' not in apk:
1073         apk['targetSdkVersion'] = apk['minSdkVersion']
1074
1075     # Check for known vulnerabilities
1076     if has_known_vulnerability(apk_file):
1077         apk['antiFeatures'].add('KnownVuln')
1078
1079     return apk
1080
1081
1082 def _get_apk_icons_src(apkfile, icon_name):
1083     """Extract the paths to the app icon in all available densities
1084
1085     """
1086     icons_src = dict()
1087     density_re = re.compile('^res/(.*)/' + icon_name + '\.(png|xml)$')
1088     with zipfile.ZipFile(apkfile) as zf:
1089         for filename in zf.namelist():
1090             m = density_re.match(filename)
1091             if m:
1092                 folder = m.group(1).split('-')
1093                 if len(folder) > 1:
1094                     density = screen_resolutions[folder[1]]
1095                 else:
1096                     density = '160'
1097                 icons_src[density] = m.group(0)
1098     if icons_src.get('-1') is None:
1099         icons_src['-1'] = icons_src['160']
1100     return icons_src
1101
1102
1103 def scan_apk_aapt(apk, apkfile):
1104     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1105     if p.returncode != 0:
1106         if options.delete_unknown:
1107             if os.path.exists(apkfile):
1108                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1109                 os.remove(apkfile)
1110             else:
1111                 logging.error("Could not find {0} to remove it".format(apkfile))
1112         else:
1113             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1114         raise BuildException(_("Invalid APK"))
1115     icon_name = None
1116     for line in p.output.splitlines():
1117         if line.startswith("package:"):
1118             try:
1119                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1120                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1121                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1122             except Exception as e:
1123                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1124         elif line.startswith("application:"):
1125             m = re.match(APK_LABEL_ICON_PAT, line)
1126             if m:
1127                 apk['name'] = m.group(1)
1128                 icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
1129         elif not apk.get('name') and line.startswith("launchable-activity:"):
1130             # Only use launchable-activity as fallback to application
1131             apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)
1132         elif line.startswith("sdkVersion:"):
1133             m = re.match(APK_SDK_VERSION_PAT, line)
1134             if m is None:
1135                 logging.error(line.replace('sdkVersion:', '')
1136                               + ' is not a valid minSdkVersion!')
1137             else:
1138                 apk['minSdkVersion'] = m.group(1)
1139         elif line.startswith("targetSdkVersion:"):
1140             m = re.match(APK_SDK_VERSION_PAT, line)
1141             if m is None:
1142                 logging.error(line.replace('targetSdkVersion:', '')
1143                               + ' is not a valid targetSdkVersion!')
1144             else:
1145                 apk['targetSdkVersion'] = m.group(1)
1146         elif line.startswith("maxSdkVersion:"):
1147             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1148         elif line.startswith("native-code:"):
1149             apk['nativecode'] = []
1150             for arch in line[13:].split(' '):
1151                 apk['nativecode'].append(arch[1:-1])
1152         elif line.startswith('uses-permission:'):
1153             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1154             if perm_match['maxSdkVersion']:
1155                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1156             permission = UsesPermission(
1157                 perm_match['name'],
1158                 perm_match['maxSdkVersion']
1159             )
1160
1161             apk['uses-permission'].append(permission)
1162         elif line.startswith('uses-permission-sdk-23:'):
1163             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1164             if perm_match['maxSdkVersion']:
1165                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1166             permission_sdk_23 = UsesPermissionSdk23(
1167                 perm_match['name'],
1168                 perm_match['maxSdkVersion']
1169             )
1170
1171             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1172
1173         elif line.startswith('uses-feature:'):
1174             feature = re.match(APK_FEATURE_PAT, line).group(1)
1175             # Filter out this, it's only added with the latest SDK tools and
1176             # causes problems for lots of apps.
1177             if feature != "android.hardware.screen.portrait" \
1178                     and feature != "android.hardware.screen.landscape":
1179                 if feature.startswith("android.feature."):
1180                     feature = feature[16:]
1181                 apk['features'].add(feature)
1182     apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1183
1184
1185 def scan_apk_androguard(apk, apkfile):
1186     try:
1187         from androguard.core.bytecodes.apk import APK
1188         apkobject = APK(apkfile)
1189         if apkobject.is_valid_APK():
1190             arsc = apkobject.get_android_resources()
1191         else:
1192             if options.delete_unknown:
1193                 if os.path.exists(apkfile):
1194                     logging.error(_("Failed to get apk information, deleting {path}")
1195                                   .format(path=apkfile))
1196                     os.remove(apkfile)
1197                 else:
1198                     logging.error(_("Could not find {path} to remove it")
1199                                   .format(path=apkfile))
1200             else:
1201                 logging.error(_("Failed to get apk information, skipping {path}")
1202                               .format(path=apkfile))
1203             raise BuildException(_("Invalid APK"))
1204     except ImportError:
1205         raise FDroidException("androguard library is not installed and aapt not present")
1206     except FileNotFoundError:
1207         logging.error(_("Could not open apk file for analysis"))
1208         raise BuildException(_("Invalid APK"))
1209
1210     apk['packageName'] = apkobject.get_package()
1211     apk['versionCode'] = int(apkobject.get_androidversion_code())
1212     apk['versionName'] = apkobject.get_androidversion_name()
1213     if apk['versionName'][0] == "@":
1214         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1215         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1216         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1217     apk['name'] = apkobject.get_app_name()
1218
1219     if apkobject.get_max_sdk_version() is not None:
1220         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1221     if apkobject.get_min_sdk_version() is not None:
1222         apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1223     if apkobject.get_target_sdk_version() is not None:
1224         apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1225
1226     icon_id_str = apkobject.get_element("application", "icon")
1227     if icon_id_str:
1228         icon_id = int(icon_id_str.replace("@", "0x"), 16)
1229         icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1230         apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1231
1232     arch_re = re.compile("^lib/(.*)/.*$")
1233     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1234     if len(arch) >= 1:
1235         apk['nativecode'] = []
1236         apk['nativecode'].extend(sorted(list(arch)))
1237
1238     xml = apkobject.get_android_manifest_xml()
1239
1240     for item in xml.findall('uses-permission'):
1241         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1242         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1243         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1244         permission = UsesPermission(
1245             name,
1246             maxSdkVersion
1247         )
1248         apk['uses-permission'].append(permission)
1249     for name, maxSdkVersion in apkobject.get_uses_implied_permission_list():
1250         permission = UsesPermission(
1251             name,
1252             maxSdkVersion
1253         )
1254         apk['uses-permission'].append(permission)
1255
1256     for item in xml.findall('uses-permission-sdk-23'):
1257         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1258         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1259         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1260         permission_sdk_23 = UsesPermissionSdk23(
1261             name,
1262             maxSdkVersion
1263         )
1264         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1265
1266     for item in xml.findall('uses-feature'):
1267         key = '{' + xml.nsmap['android'] + '}name'
1268         if key not in item.attrib:
1269             continue
1270         feature = str(item.attrib[key])
1271         if feature != "android.hardware.screen.portrait" \
1272                 and feature != "android.hardware.screen.landscape":
1273             if feature.startswith("android.feature."):
1274                 feature = feature[16:]
1275         required = item.attrib.get('{' + xml.nsmap['android'] + '}required')
1276         if required is None or required == 'true':
1277             apk['features'].append(feature)
1278
1279
1280 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1281                 allow_disabled_algorithms=False, archive_bad_sig=False):
1282     """Processes the apk with the given filename in the given repo directory.
1283
1284     This also extracts the icons.
1285
1286     :param apkcache: current apk cache information
1287     :param apkfilename: the filename of the apk to scan
1288     :param repodir: repo directory to scan
1289     :param knownapks: known apks info
1290     :param use_date_from_apk: use date from APK (instead of current date)
1291                               for newly added APKs
1292     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1293                                       disabled algorithms in the signature (e.g. MD5)
1294     :param archive_bad_sig: move APKs with a bad signature to the archive
1295     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1296      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1297     """
1298
1299     apk = {}
1300     apkfile = os.path.join(repodir, apkfilename)
1301
1302     cachechanged = False
1303     usecache = False
1304     if apkfilename in apkcache:
1305         apk = apkcache[apkfilename]
1306         if apk.get('hash') == sha256sum(apkfile):
1307             logging.debug(_("Reading {apkfilename} from cache")
1308                           .format(apkfilename=apkfilename))
1309             usecache = True
1310         else:
1311             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1312                           .format(apkfilename=apkfilename))
1313
1314     if not usecache:
1315         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1316
1317         try:
1318             apk = scan_apk(apkfile)
1319         except BuildException:
1320             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1321                             .format(apkfilename=apkfilename))
1322             return True, None, False
1323
1324         # Check for debuggable apks...
1325         if common.is_apk_and_debuggable(apkfile):
1326             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1327
1328         if options.rename_apks:
1329             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1330             std_short_name = os.path.join(repodir, n)
1331             if apkfile != std_short_name:
1332                 if os.path.exists(std_short_name):
1333                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1334                     if apkfile != std_long_name:
1335                         if os.path.exists(std_long_name):
1336                             dupdir = os.path.join('duplicates', repodir)
1337                             if not os.path.isdir(dupdir):
1338                                 os.makedirs(dupdir, exist_ok=True)
1339                             dupfile = os.path.join('duplicates', std_long_name)
1340                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1341                             os.rename(apkfile, dupfile)
1342                             return True, None, False
1343                         else:
1344                             os.rename(apkfile, std_long_name)
1345                     apkfile = std_long_name
1346                 else:
1347                     os.rename(apkfile, std_short_name)
1348                     apkfile = std_short_name
1349                 apkfilename = apkfile[len(repodir) + 1:]
1350
1351         apk['apkName'] = apkfilename
1352         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1353         if os.path.exists(os.path.join(repodir, srcfilename)):
1354             apk['srcname'] = srcfilename
1355
1356         # verify the jar signature is correct, allow deprecated
1357         # algorithms only if the APK is in the archive.
1358         skipapk = False
1359         if not common.verify_apk_signature(apkfile):
1360             if repodir == 'archive' or allow_disabled_algorithms:
1361                 if common.verify_old_apk_signature(apkfile):
1362                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1363                 else:
1364                     skipapk = True
1365             else:
1366                 skipapk = True
1367
1368         if skipapk:
1369             if archive_bad_sig:
1370                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1371                                 .format(apkfilename=apkfilename))
1372                 move_apk_between_sections(repodir, 'archive', apk)
1373             else:
1374                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1375                                 .format(apkfilename=apkfilename))
1376             return True, None, False
1377
1378         apkzip = zipfile.ZipFile(apkfile, 'r')
1379
1380         manifest = apkzip.getinfo('AndroidManifest.xml')
1381         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1382         if (1980, 0, 0) != manifest.date_time[0:3]:
1383             try:
1384                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1385             except ValueError as e:
1386                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1387                                 .format(apkfilename=apkfile) + str(e))
1388
1389         # extract icons from APK zip file
1390         iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
1391         try:
1392             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1393         finally:
1394             apkzip.close()  # ensure that APK zip file gets closed
1395
1396         # resize existing icons for densities missing in the APK
1397         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1398
1399         if use_date_from_apk and manifest.date_time[1] != 0:
1400             default_date_param = datetime(*manifest.date_time)
1401         else:
1402             default_date_param = None
1403
1404         # Record in known apks, getting the added date at the same time..
1405         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1406                                     default_date=default_date_param)
1407         if added:
1408             apk['added'] = added
1409
1410         apkcache[apkfilename] = apk
1411         cachechanged = True
1412
1413     return False, apk, cachechanged
1414
1415
1416 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1417     """Processes the apks in the given repo directory.
1418
1419     This also extracts the icons.
1420
1421     :param apkcache: current apk cache information
1422     :param repodir: repo directory to scan
1423     :param knownapks: known apks info
1424     :param use_date_from_apk: use date from APK (instead of current date)
1425                               for newly added APKs
1426     :returns: (apks, cachechanged) where apks is a list of apk information,
1427               and cachechanged is True if the apkcache got changed.
1428     """
1429
1430     cachechanged = False
1431
1432     for icon_dir in get_all_icon_dirs(repodir):
1433         if os.path.exists(icon_dir):
1434             if options.clean:
1435                 shutil.rmtree(icon_dir)
1436                 os.makedirs(icon_dir)
1437         else:
1438             os.makedirs(icon_dir)
1439
1440     apks = []
1441     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1442         apkfilename = apkfile[len(repodir) + 1:]
1443         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1444         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1445                                              use_date_from_apk, ada, True)
1446         if skip:
1447             continue
1448         apks.append(apk)
1449         cachechanged = cachechanged or cachethis
1450
1451     return apks, cachechanged
1452
1453
1454 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1455     """Extracts PNG icons from an APK with the supported pixel densities
1456
1457     Extracts icons from the given APK zip in various densities, saves
1458     them into given repo directory and stores their names in the APK
1459     metadata dictionary.  If the icon is an XML icon, then this tries
1460     to find PNG icon that can replace it.
1461
1462     :param icon_filename: A string representing the icon's file name
1463     :param apk: A populated dictionary containing APK metadata.
1464                 Needs to have 'icons_src' key
1465     :param apkzip: An opened zipfile.ZipFile of the APK file
1466     :param repo_dir: The directory of the APK's repository
1467     :return: A list of icon densities that are missing
1468
1469     """
1470     res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1471     pngs = dict()
1472     for f in apkzip.namelist():
1473         m = res_name_re.match(f)
1474         if m and m.group(4) == 'png':
1475             density = screen_resolutions[m.group(2)]
1476             pngs[m.group(3) + '/' + density] = m.group(0)
1477
1478     icon_type = None
1479     empty_densities = []
1480     for density in screen_densities:
1481         if density not in apk['icons_src']:
1482             empty_densities.append(density)
1483             continue
1484         icon_src = apk['icons_src'][density]
1485         icon_dir = get_icon_dir(repo_dir, density)
1486         icon_type = '.png'
1487
1488         # Extract the icon files per density
1489         if icon_src.endswith('.xml'):
1490             m = res_name_re.match(icon_src)
1491             if m:
1492                 name = pngs.get(m.group(3) + '/' + str(density))
1493                 if name:
1494                     icon_src = name
1495             if icon_src.endswith('.xml'):
1496                 empty_densities.append(density)
1497                 icon_type = '.xml'
1498         icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
1499
1500         try:
1501             with open(icon_dest, 'wb') as f:
1502                 f.write(get_icon_bytes(apkzip, icon_src))
1503             apk['icons'][density] = icon_filename + icon_type
1504         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1505             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1506             del apk['icons_src'][density]
1507             empty_densities.append(density)
1508
1509     # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
1510     if '-1' in apk['icons_src']:
1511         icon_src = apk['icons_src']['-1']
1512         icon_type = icon_src[-4:]
1513         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
1514         with open(icon_path, 'wb') as f:
1515             f.write(get_icon_bytes(apkzip, icon_src))
1516         if icon_type == '.png':
1517             im = None
1518             try:
1519                 im = Image.open(icon_path)
1520                 dpi = px_to_dpi(im.size[0])
1521                 for density in screen_densities:
1522                     if density in apk['icons']:
1523                         break
1524                     if density == screen_densities[-1] or dpi >= int(density):
1525                         apk['icons'][density] = icon_filename + icon_type
1526                         shutil.move(icon_path,
1527                                     os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
1528                         empty_densities.remove(density)
1529                         break
1530             except Exception as e:
1531                 logging.warning(_("Failed reading {path}: {error}")
1532                                 .format(path=icon_path, error=e))
1533             finally:
1534                 if im and hasattr(im, 'close'):
1535                     im.close()
1536
1537     if apk['icons']:
1538         apk['icon'] = icon_filename + icon_type
1539
1540     return empty_densities
1541
1542
1543 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1544     """
1545     Resize existing PNG icons for densities missing in the APK to ensure all densities are available
1546
1547     :param empty_densities: A list of icon densities that are missing
1548     :param icon_filename: A string representing the icon's file name
1549     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1550     :param repo_dir: The directory of the APK's repository
1551
1552     """
1553     icon_filename += '.png'
1554     # First try resizing down to not lose quality
1555     last_density = None
1556     for density in screen_densities:
1557         if density == '65534':  # not possible to generate 'anydpi' from other densities
1558             continue
1559         if density not in empty_densities:
1560             last_density = density
1561             continue
1562         if last_density is None:
1563             continue
1564         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1565
1566         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1567         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1568         fp = None
1569         try:
1570             fp = open(last_icon_path, 'rb')
1571             im = Image.open(fp)
1572
1573             size = dpi_to_px(density)
1574
1575             im.thumbnail((size, size), Image.ANTIALIAS)
1576             im.save(icon_path, "PNG", optimize=True,
1577                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1578             empty_densities.remove(density)
1579         except Exception as e:
1580             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1581         finally:
1582             if fp:
1583                 fp.close()
1584
1585     # Then just copy from the highest resolution available
1586     last_density = None
1587     for density in reversed(screen_densities):
1588         if density not in empty_densities:
1589             last_density = density
1590             continue
1591
1592         if last_density is None:
1593             continue
1594
1595         shutil.copyfile(
1596             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1597             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1598         )
1599         empty_densities.remove(density)
1600
1601     for density in screen_densities:
1602         icon_dir = get_icon_dir(repo_dir, density)
1603         icon_dest = os.path.join(icon_dir, icon_filename)
1604         resize_icon(icon_dest, density)
1605
1606     # Copy from icons-mdpi to icons since mdpi is the baseline density
1607     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1608     if os.path.isfile(baseline):
1609         apk['icons']['0'] = icon_filename
1610         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1611
1612
1613 def apply_info_from_latest_apk(apps, apks):
1614     """
1615     Some information from the apks needs to be applied up to the application level.
1616     When doing this, we use the info from the most recent version's apk.
1617     We deal with figuring out when the app was added and last updated at the same time.
1618     """
1619     for appid, app in apps.items():
1620         bestver = UNSET_VERSION_CODE
1621         for apk in apks:
1622             if apk['packageName'] == appid:
1623                 if apk['versionCode'] > bestver:
1624                     bestver = apk['versionCode']
1625                     bestapk = apk
1626
1627                 if 'added' in apk:
1628                     if not app.added or apk['added'] < app.added:
1629                         app.added = apk['added']
1630                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1631                         app.lastUpdated = apk['added']
1632
1633         if not app.added:
1634             logging.debug("Don't know when " + appid + " was added")
1635         if not app.lastUpdated:
1636             logging.debug("Don't know when " + appid + " was last updated")
1637
1638         if bestver == UNSET_VERSION_CODE:
1639
1640             if app.Name is None:
1641                 app.Name = app.AutoName or appid
1642             app.icon = None
1643             logging.debug("Application " + appid + " has no packages")
1644         else:
1645             if app.Name is None:
1646                 app.Name = bestapk['name']
1647             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1648             if app.CurrentVersionCode is None:
1649                 app.CurrentVersionCode = str(bestver)
1650
1651
1652 def make_categories_txt(repodir, categories):
1653     '''Write a category list in the repo to allow quick access'''
1654     catdata = ''
1655     for cat in sorted(categories):
1656         catdata += cat + '\n'
1657     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1658         f.write(catdata)
1659
1660
1661 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1662
1663     def filter_apk_list_sorted(apk_list):
1664         res = []
1665         for apk in apk_list:
1666             if apk['packageName'] == appid:
1667                 res.append(apk)
1668
1669         # Sort the apk list by version code. First is highest/newest.
1670         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1671
1672     for appid, app in apps.items():
1673
1674         if app.ArchivePolicy:
1675             keepversions = int(app.ArchivePolicy[:-9])
1676         else:
1677             keepversions = defaultkeepversions
1678
1679         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1680                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1681
1682         current_app_apks = filter_apk_list_sorted(apks)
1683         if len(current_app_apks) > keepversions:
1684             # Move back the ones we don't want.
1685             for apk in current_app_apks[keepversions:]:
1686                 move_apk_between_sections(repodir, archivedir, apk)
1687                 archapks.append(apk)
1688                 apks.remove(apk)
1689
1690         current_app_archapks = filter_apk_list_sorted(archapks)
1691         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1692             kept = 0
1693             # Move forward the ones we want again, except DisableAlgorithm
1694             for apk in current_app_archapks:
1695                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1696                     move_apk_between_sections(archivedir, repodir, apk)
1697                     archapks.remove(apk)
1698                     apks.append(apk)
1699                     kept += 1
1700                 if kept == keepversions:
1701                     break
1702
1703
1704 def move_apk_between_sections(from_dir, to_dir, apk):
1705     """move an APK from repo to archive or vice versa"""
1706
1707     def _move_file(from_dir, to_dir, filename, ignore_missing):
1708         from_path = os.path.join(from_dir, filename)
1709         if ignore_missing and not os.path.exists(from_path):
1710             return
1711         to_path = os.path.join(to_dir, filename)
1712         if not os.path.exists(to_dir):
1713             os.mkdir(to_dir)
1714         shutil.move(from_path, to_path)
1715
1716     if from_dir == to_dir:
1717         return
1718
1719     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1720     _move_file(from_dir, to_dir, apk['apkName'], False)
1721     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1722     for density in all_screen_densities:
1723         from_icon_dir = get_icon_dir(from_dir, density)
1724         to_icon_dir = get_icon_dir(to_dir, density)
1725         if density not in apk.get('icons', []):
1726             continue
1727         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1728     if 'srcname' in apk:
1729         _move_file(from_dir, to_dir, apk['srcname'], False)
1730
1731
1732 def add_apks_to_per_app_repos(repodir, apks):
1733     apks_per_app = dict()
1734     for apk in apks:
1735         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1736         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1737         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1738         apks_per_app[apk['packageName']] = apk
1739
1740         if not os.path.exists(apk['per_app_icons']):
1741             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1742             os.makedirs(apk['per_app_icons'])
1743
1744         apkpath = os.path.join(repodir, apk['apkName'])
1745         shutil.copy(apkpath, apk['per_app_repo'])
1746         apksigpath = apkpath + '.sig'
1747         if os.path.exists(apksigpath):
1748             shutil.copy(apksigpath, apk['per_app_repo'])
1749         apkascpath = apkpath + '.asc'
1750         if os.path.exists(apkascpath):
1751             shutil.copy(apkascpath, apk['per_app_repo'])
1752
1753
1754 def create_metadata_from_template(apk):
1755     '''create a new metadata file using internal or external template
1756
1757     Generate warnings for apk's with no metadata (or create skeleton
1758     metadata files, if requested on the command line).  Though the
1759     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1760     since those impose things on the metadata file made from the
1761     template: field sort order, empty field value, formatting, etc.
1762     '''
1763
1764     import yaml
1765     if os.path.exists('template.yml'):
1766         with open('template.yml') as f:
1767             metatxt = f.read()
1768         if 'name' in apk and apk['name'] != '':
1769             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1770                              r'\1 ' + apk['name'],
1771                              metatxt,
1772                              flags=re.IGNORECASE | re.MULTILINE)
1773         else:
1774             logging.warning(_('{appid} does not have a name! Using package name instead.')
1775                             .format(appid=apk['packageName']))
1776             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1777                              r'\1 ' + apk['packageName'],
1778                              metatxt,
1779                              flags=re.IGNORECASE | re.MULTILINE)
1780         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1781             f.write(metatxt)
1782     else:
1783         app = dict()
1784         app['Categories'] = [os.path.basename(os.getcwd())]
1785         # include some blanks as part of the template
1786         app['AuthorName'] = ''
1787         app['Summary'] = ''
1788         app['WebSite'] = ''
1789         app['IssueTracker'] = ''
1790         app['SourceCode'] = ''
1791         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1792         if 'name' in apk and apk['name'] != '':
1793             app['Name'] = apk['name']
1794         else:
1795             logging.warning(_('{appid} does not have a name! Using package name instead.')
1796                             .format(appid=apk['packageName']))
1797             app['Name'] = apk['packageName']
1798         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1799             yaml.dump(app, f, default_flow_style=False)
1800     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1801
1802
1803 config = None
1804 options = None
1805 start_timestamp = time.gmtime()
1806
1807
1808 def main():
1809
1810     global config, options
1811
1812     # Parse command line...
1813     parser = ArgumentParser()
1814     common.setup_global_opts(parser)
1815     parser.add_argument("--create-key", action="store_true", default=False,
1816                         help=_("Add a repo signing key to an unsigned repo"))
1817     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1818                         help=_("Add skeleton metadata files for APKs that are missing them"))
1819     parser.add_argument("--delete-unknown", action="store_true", default=False,
1820                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1821     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1822                         help=_("Report on build data status"))
1823     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1824                         help=_("Interactively ask about things that need updating."))
1825     parser.add_argument("-I", "--icons", action="store_true", default=False,
1826                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1827     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1828                         help=_("Specify editor to use in interactive mode. Default " +
1829                                "is {path}").format(path='/etc/alternatives/editor'))
1830     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1831                         help=_("Update the wiki"))
1832     parser.add_argument("--pretty", action="store_true", default=False,
1833                         help=_("Produce human-readable XML/JSON for index files"))
1834     parser.add_argument("--clean", action="store_true", default=False,
1835                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1836     parser.add_argument("--nosign", action="store_true", default=False,
1837                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1838     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1839                         help=_("Use date from APK instead of current time for newly added APKs"))
1840     parser.add_argument("--rename-apks", action="store_true", default=False,
1841                         help=_("Rename APK files that do not match package.name_123.apk"))
1842     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1843                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1844     metadata.add_metadata_arguments(parser)
1845     options = parser.parse_args()
1846     metadata.warnings_action = options.W
1847
1848     config = common.read_config(options)
1849
1850     if not ('jarsigner' in config and 'keytool' in config):
1851         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1852
1853     repodirs = ['repo']
1854     if config['archive_older'] != 0:
1855         repodirs.append('archive')
1856         if not os.path.exists('archive'):
1857             os.mkdir('archive')
1858
1859     if options.icons:
1860         resize_all_icons(repodirs)
1861         sys.exit(0)
1862
1863     if options.rename_apks:
1864         options.clean = True
1865
1866     # check that icons exist now, rather than fail at the end of `fdroid update`
1867     for k in ['repo_icon', 'archive_icon']:
1868         if k in config:
1869             if not os.path.exists(config[k]):
1870                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1871                                  .format(name=k, path=config[k]))
1872                 sys.exit(1)
1873
1874     # if the user asks to create a keystore, do it now, reusing whatever it can
1875     if options.create_key:
1876         if os.path.exists(config['keystore']):
1877             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1878             logging.critical("\t'" + config['keystore'] + "'")
1879             sys.exit(1)
1880
1881         if 'repo_keyalias' not in config:
1882             config['repo_keyalias'] = socket.getfqdn()
1883             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1884         if 'keydname' not in config:
1885             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1886             common.write_to_config(config, 'keydname', config['keydname'])
1887         if 'keystore' not in config:
1888             config['keystore'] = common.default_config['keystore']
1889             common.write_to_config(config, 'keystore', config['keystore'])
1890
1891         password = common.genpassword()
1892         if 'keystorepass' not in config:
1893             config['keystorepass'] = password
1894             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1895         if 'keypass' not in config:
1896             config['keypass'] = password
1897             common.write_to_config(config, 'keypass', config['keypass'])
1898         common.genkeystore(config)
1899
1900     # Get all apps...
1901     apps = metadata.read_metadata()
1902
1903     # Generate a list of categories...
1904     categories = set()
1905     for app in apps.values():
1906         categories.update(app.Categories)
1907
1908     # Read known apks data (will be updated and written back when we've finished)
1909     knownapks = common.KnownApks()
1910
1911     # Get APK cache
1912     apkcache = get_cache()
1913
1914     # Delete builds for disabled apps
1915     delete_disabled_builds(apps, apkcache, repodirs)
1916
1917     # Scan all apks in the main repo
1918     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1919
1920     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1921                                            options.use_date_from_apk)
1922     cachechanged = cachechanged or fcachechanged
1923     apks += files
1924     for apk in apks:
1925         if apk['packageName'] not in apps:
1926             if options.create_metadata:
1927                 create_metadata_from_template(apk)
1928                 apps = metadata.read_metadata()
1929             else:
1930                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1931                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1932                 if options.delete_unknown:
1933                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1934                                  .format(apkfilename=apk['apkName']))
1935                     rmf = os.path.join(repodirs[0], apk['apkName'])
1936                     if not os.path.exists(rmf):
1937                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1938                     else:
1939                         os.remove(rmf)
1940                 else:
1941                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1942
1943     copy_triple_t_store_metadata(apps)
1944     insert_obbs(repodirs[0], apps, apks)
1945     insert_localized_app_metadata(apps)
1946     translate_per_build_anti_features(apps, apks)
1947
1948     # Scan the archive repo for apks as well
1949     if len(repodirs) > 1:
1950         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1951         if cc:
1952             cachechanged = True
1953     else:
1954         archapks = []
1955
1956     # Apply information from latest apks to the application and update dates
1957     apply_info_from_latest_apk(apps, apks + archapks)
1958
1959     # Sort the app list by name, then the web site doesn't have to by default.
1960     # (we had to wait until we'd scanned the apks to do this, because mostly the
1961     # name comes from there!)
1962     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1963
1964     # APKs are placed into multiple repos based on the app package, providing
1965     # per-app subscription feeds for nightly builds and things like it
1966     if config['per_app_repos']:
1967         add_apks_to_per_app_repos(repodirs[0], apks)
1968         for appid, app in apps.items():
1969             repodir = os.path.join(appid, 'fdroid', 'repo')
1970             appdict = dict()
1971             appdict[appid] = app
1972             if os.path.isdir(repodir):
1973                 index.make(appdict, [appid], apks, repodir, False)
1974             else:
1975                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1976         return
1977
1978     if len(repodirs) > 1:
1979         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1980
1981     # Make the index for the main repo...
1982     index.make(apps, sortedids, apks, repodirs[0], False)
1983     make_categories_txt(repodirs[0], categories)
1984
1985     # If there's an archive repo,  make the index for it. We already scanned it
1986     # earlier on.
1987     if len(repodirs) > 1:
1988         index.make(apps, sortedids, archapks, repodirs[1], True)
1989
1990     git_remote = config.get('binary_transparency_remote')
1991     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1992         from . import btlog
1993         btlog.make_binary_transparency_log(repodirs)
1994
1995     if config['update_stats']:
1996         # Update known apks info...
1997         knownapks.writeifchanged()
1998
1999         # Generate latest apps data for widget
2000         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
2001             data = ''
2002             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
2003                 for line in f:
2004                     appid = line.rstrip()
2005                     data += appid + "\t"
2006                     app = apps[appid]
2007                     data += app.Name + "\t"
2008                     if app.icon is not None:
2009                         data += app.icon + "\t"
2010                     data += app.License + "\n"
2011             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
2012                 f.write(data)
2013
2014     if cachechanged:
2015         write_cache(apkcache)
2016
2017     # Update the wiki...
2018     if options.wiki:
2019         update_wiki(apps, sortedids, apks + archapks)
2020
2021     logging.info(_("Finished"))
2022
2023
2024 if __name__ == "__main__":
2025     main()