fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*)'")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
  63 # resolutions must end with 'dpi'
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "undefineddpi": '-1',
  72     "anydpi": '65534',
  73     "nodpi": '65535'
  74 }
  75
  76 all_screen_densities = ['0'] + screen_densities
  77
  78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  80
  81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  84                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  85
  86 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  87
  88
  89 def dpi_to_px(density):
  90     return (int(density) * 48) / 160
  91
  92
  93 def px_to_dpi(px):
  94     return (int(px) * 160) / 48
  95
  96
  97 def get_icon_dir(repodir, density):
  98     if density == '0' or density == '65534':
  99         return os.path.join(repodir, "icons")
 100     else:
 101         return os.path.join(repodir, "icons-%s" % density)
 102
 103
 104 def get_icon_dirs(repodir):
 105     for density in screen_densities:
 106         yield get_icon_dir(repodir, density)
 107
 108
 109 def get_all_icon_dirs(repodir):
 110     for density in all_screen_densities:
 111         yield get_icon_dir(repodir, density)
 112
 113
 114 def update_wiki(apps, sortedids, apks):
 115     """Update the wiki
 116
 117     :param apps: fully populated list of all applications
 118     :param apks: all apks, except...
 119     """
 120     logging.info("Updating wiki")
 121     wikicat = 'Apps'
 122     wikiredircat = 'App Redirects'
 123     import mwclient
 124     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 125                          path=config['wiki_path'])
 126     site.login(config['wiki_user'], config['wiki_password'])
 127     generated_pages = {}
 128     generated_redirects = {}
 129
 130     for appid in sortedids:
 131         app = metadata.App(apps[appid])
 132
 133         wikidata = ''
 134         if app.Disabled:
 135             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 136         if app.AntiFeatures:
 137             for af in sorted(app.AntiFeatures):
 138                 wikidata += '{{AntiFeature|' + af + '}}\n'
 139         if app.RequiresRoot:
 140             requiresroot = 'Yes'
 141         else:
 142             requiresroot = 'No'
 143         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 144             appid,
 145             app.Name,
 146             app.added.strftime('%Y-%m-%d') if app.added else '',
 147             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 148             app.SourceCode,
 149             app.IssueTracker,
 150             app.WebSite,
 151             app.Changelog,
 152             app.Donate,
 153             app.FlattrID,
 154             app.LiberapayID,
 155             app.Bitcoin,
 156             app.Litecoin,
 157             app.License,
 158             requiresroot,
 159             app.AuthorName,
 160             app.AuthorEmail)
 161
 162         if app.Provides:
 163             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 164
 165         wikidata += app.Summary
 166         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 167
 168         wikidata += "=Description=\n"
 169         wikidata += metadata.description_wiki(app.Description) + "\n"
 170
 171         wikidata += "=Maintainer Notes=\n"
 172         if app.MaintainerNotes:
 173             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 174         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 175
 176         # Get a list of all packages for this application...
 177         apklist = []
 178         gotcurrentver = False
 179         cantupdate = False
 180         buildfails = False
 181         for apk in apks:
 182             if apk['packageName'] == appid:
 183                 if str(apk['versionCode']) == app.CurrentVersionCode:
 184                     gotcurrentver = True
 185                 apklist.append(apk)
 186         # Include ones we can't build, as a special case...
 187         for build in app.builds:
 188             if build.disable:
 189                 if build.versionCode == app.CurrentVersionCode:
 190                     cantupdate = True
 191                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 192                 apklist.append({'versionCode': int(build.versionCode),
 193                                 'versionName': build.versionName,
 194                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 195                                 })
 196             else:
 197                 builtit = False
 198                 for apk in apklist:
 199                     if apk['versionCode'] == int(build.versionCode):
 200                         builtit = True
 201                         break
 202                 if not builtit:
 203                     buildfails = True
 204                     apklist.append({'versionCode': int(build.versionCode),
 205                                     'versionName': build.versionName,
 206                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 207                                     })
 208         if app.CurrentVersionCode == '0':
 209             cantupdate = True
 210         # Sort with most recent first...
 211         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 212
 213         wikidata += "=Versions=\n"
 214         if len(apklist) == 0:
 215             wikidata += "We currently have no versions of this app available."
 216         elif not gotcurrentver:
 217             wikidata += "We don't have the current version of this app."
 218         else:
 219             wikidata += "We have the current version of this app."
 220         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 221         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 222         if len(app.NoSourceSince) > 0:
 223             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 224         if len(app.CurrentVersion) > 0:
 225             wikidata += "The current (recommended) version is " + app.CurrentVersion
 226             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 227         validapks = 0
 228         for apk in apklist:
 229             wikidata += "==" + apk['versionName'] + "==\n"
 230
 231             if 'buildproblem' in apk:
 232                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 233             else:
 234                 validapks += 1
 235                 wikidata += "This version is built and signed by "
 236                 if 'srcname' in apk:
 237                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 238                 else:
 239                     wikidata += "the original developer.\n\n"
 240             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 241
 242         wikidata += '\n[[Category:' + wikicat + ']]\n'
 243         if len(app.NoSourceSince) > 0:
 244             wikidata += '\n[[Category:Apps missing source code]]\n'
 245         if validapks == 0 and not app.Disabled:
 246             wikidata += '\n[[Category:Apps with no packages]]\n'
 247         if cantupdate and not app.Disabled:
 248             wikidata += "\n[[Category:Apps we cannot update]]\n"
 249         if buildfails and not app.Disabled:
 250             wikidata += "\n[[Category:Apps with failing builds]]\n"
 251         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 252             wikidata += '\n[[Category:Apps to Update]]\n'
 253         if app.Disabled:
 254             wikidata += '\n[[Category:Apps that are disabled]]\n'
 255         if app.UpdateCheckMode == 'None' and not app.Disabled:
 256             wikidata += '\n[[Category:Apps with no update check]]\n'
 257         for appcat in app.Categories:
 258             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 259
 260         # We can't have underscores in the page name, even if they're in
 261         # the package ID, because MediaWiki messes with them...
 262         pagename = appid.replace('_', ' ')
 263
 264         # Drop a trailing newline, because mediawiki is going to drop it anyway
 265         # and it we don't we'll think the page has changed when it hasn't...
 266         if wikidata.endswith('\n'):
 267             wikidata = wikidata[:-1]
 268
 269         generated_pages[pagename] = wikidata
 270
 271         # Make a redirect from the name to the ID too, unless there's
 272         # already an existing page with the name and it isn't a redirect.
 273         noclobber = False
 274         apppagename = app.Name.replace('_', ' ')
 275         apppagename = apppagename.replace('{', '')
 276         apppagename = apppagename.replace('}', ' ')
 277         apppagename = apppagename.replace(':', ' ')
 278         apppagename = apppagename.replace('[', ' ')
 279         apppagename = apppagename.replace(']', ' ')
 280         # Drop double spaces caused mostly by replacing ':' above
 281         apppagename = apppagename.replace('  ', ' ')
 282         for expagename in site.allpages(prefix=apppagename,
 283                                         filterredir='nonredirects',
 284                                         generator=False):
 285             if expagename == apppagename:
 286                 noclobber = True
 287         # Another reason not to make the redirect page is if the app name
 288         # is the same as it's ID, because that will overwrite the real page
 289         # with an redirect to itself! (Although it seems like an odd
 290         # scenario this happens a lot, e.g. where there is metadata but no
 291         # builds or binaries to extract a name from.
 292         if apppagename == pagename:
 293             noclobber = True
 294         if not noclobber:
 295             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 296
 297     for tcat, genp in [(wikicat, generated_pages),
 298                        (wikiredircat, generated_redirects)]:
 299         catpages = site.Pages['Category:' + tcat]
 300         existingpages = []
 301         for page in catpages:
 302             existingpages.append(page.name)
 303             if page.name in genp:
 304                 pagetxt = page.edit()
 305                 if pagetxt != genp[page.name]:
 306                     logging.debug("Updating modified page " + page.name)
 307                     page.save(genp[page.name], summary='Auto-updated')
 308                 else:
 309                     logging.debug("Page " + page.name + " is unchanged")
 310             else:
 311                 logging.warn("Deleting page " + page.name)
 312                 page.delete('No longer published')
 313         for pagename, text in genp.items():
 314             logging.debug("Checking " + pagename)
 315             if pagename not in existingpages:
 316                 logging.debug("Creating page " + pagename)
 317                 try:
 318                     newpage = site.Pages[pagename]
 319                     newpage.save(text, summary='Auto-created')
 320                 except Exception as e:
 321                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 322
 323     # Purge server cache to ensure counts are up to date
 324     site.Pages['Repository Maintenance'].purge()
 325
 326     # Write a page with the last build log for this version code
 327     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 328     newpage = site.Pages[wiki_page_path]
 329     txt = ''
 330     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 331     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 332     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 333     txt += common.get_git_describe_link()
 334     txt += "\n\n"
 335     txt += common.get_android_tools_version_log()
 336     newpage.save(txt, summary='Run log')
 337     newpage = site.Pages['update']
 338     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 339
 340
 341 def delete_disabled_builds(apps, apkcache, repodirs):
 342     """Delete disabled build outputs.
 343
 344     :param apps: list of all applications, as per metadata.read_metadata
 345     :param apkcache: current apk cache information
 346     :param repodirs: the repo directories to process
 347     """
 348     for appid, app in apps.items():
 349         for build in app['builds']:
 350             if not build.disable:
 351                 continue
 352             apkfilename = common.get_release_filename(app, build)
 353             iconfilename = "%s.%s.png" % (
 354                 appid,
 355                 build.versionCode)
 356             for repodir in repodirs:
 357                 files = [
 358                     os.path.join(repodir, apkfilename),
 359                     os.path.join(repodir, apkfilename + '.asc'),
 360                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 361                 ]
 362                 for density in all_screen_densities:
 363                     repo_dir = get_icon_dir(repodir, density)
 364                     files.append(os.path.join(repo_dir, iconfilename))
 365
 366                 for f in files:
 367                     if os.path.exists(f):
 368                         logging.info("Deleting disabled build output " + f)
 369                         os.remove(f)
 370             if apkfilename in apkcache:
 371                 del apkcache[apkfilename]
 372
 373
 374 def resize_icon(iconpath, density):
 375
 376     if not os.path.isfile(iconpath):
 377         return
 378
 379     fp = None
 380     try:
 381         fp = open(iconpath, 'rb')
 382         im = Image.open(fp)
 383         size = dpi_to_px(density)
 384
 385         if any(length > size for length in im.size):
 386             oldsize = im.size
 387             im.thumbnail((size, size), Image.ANTIALIAS)
 388             logging.debug("%s was too large at %s - new size is %s" % (
 389                 iconpath, oldsize, im.size))
 390             im.save(iconpath, "PNG", optimize=True,
 391                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 392
 393     except Exception as e:
 394         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 395
 396     finally:
 397         if fp:
 398             fp.close()
 399
 400
 401 def resize_all_icons(repodirs):
 402     """Resize all icons that exceed the max size
 403
 404     :param repodirs: the repo directories to process
 405     """
 406     for repodir in repodirs:
 407         for density in screen_densities:
 408             icon_dir = get_icon_dir(repodir, density)
 409             icon_glob = os.path.join(icon_dir, '*.png')
 410             for iconpath in glob.glob(icon_glob):
 411                 resize_icon(iconpath, density)
 412
 413
 414 def getsig(apkpath):
 415     """ Get the signing certificate of an apk. To get the same md5 has that
 416     Android gets, we encode the .RSA certificate in a specific format and pass
 417     it hex-encoded to the md5 digest algorithm.
 418
 419     :param apkpath: path to the apk
 420     :returns: A string containing the md5 of the signature of the apk or None
 421               if an error occurred.
 422     """
 423
 424     with zipfile.ZipFile(apkpath, 'r') as apk:
 425         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 426
 427         if len(certs) < 1:
 428             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 429             return None
 430         if len(certs) > 1:
 431             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 432             return None
 433
 434         cert = apk.read(certs[0])
 435
 436     cert_encoded = common.get_certificate(cert)
 437
 438     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 439
 440
 441 def get_cache_file():
 442     return os.path.join('tmp', 'apkcache')
 443
 444
 445 def get_cache():
 446     """Get the cached dict of the APK index
 447
 448     Gather information about all the apk files in the repo directory,
 449     using cached data if possible. Some of the index operations take a
 450     long time, like calculating the SHA-256 and verifying the APK
 451     signature.
 452
 453     The cache is invalidated if the metadata version is different, or
 454     the 'allow_disabled_algorithms' config/option is different.  In
 455     those cases, there is no easy way to know what has changed from
 456     the cache, so just rerun the whole thing.
 457
 458     :return: apkcache
 459
 460     """
 461     apkcachefile = get_cache_file()
 462     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 463     if not options.clean and os.path.exists(apkcachefile):
 464         with open(apkcachefile, 'rb') as cf:
 465             apkcache = pickle.load(cf, encoding='utf-8')
 466         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 467            or apkcache.get('allow_disabled_algorithms') != ada:
 468             apkcache = {}
 469     else:
 470         apkcache = {}
 471
 472     apkcache["METADATA_VERSION"] = METADATA_VERSION
 473     apkcache['allow_disabled_algorithms'] = ada
 474
 475     return apkcache
 476
 477
 478 def write_cache(apkcache):
 479     apkcachefile = get_cache_file()
 480     cache_path = os.path.dirname(apkcachefile)
 481     if not os.path.exists(cache_path):
 482         os.makedirs(cache_path)
 483     with open(apkcachefile, 'wb') as cf:
 484         pickle.dump(apkcache, cf)
 485
 486
 487 def get_icon_bytes(apkzip, iconsrc):
 488     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 489     try:
 490         return apkzip.read(iconsrc)
 491     except KeyError:
 492         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 493
 494
 495 def sha256sum(filename):
 496     '''Calculate the sha256 of the given file'''
 497     sha = hashlib.sha256()
 498     with open(filename, 'rb') as f:
 499         while True:
 500             t = f.read(16384)
 501             if len(t) == 0:
 502                 break
 503             sha.update(t)
 504     return sha.hexdigest()
 505
 506
 507 def has_known_vulnerability(filename):
 508     """checks for known vulnerabilities in the APK
 509
 510     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 511     version.  Google also enforces this:
 512     https://support.google.com/faqs/answer/6376725?hl=en
 513
 514     Checks whether there are more than one classes.dex or AndroidManifest.xml
 515     files, which is invalid and an essential part of the "Master Key" attack.
 516     http://www.saurik.com/id/17
 517
 518     Janus is similar to Master Key but is perhaps easier to scan for.
 519     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 520     """
 521
 522     found_vuln = False
 523
 524     # statically load this pattern
 525     if not hasattr(has_known_vulnerability, "pattern"):
 526         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 527
 528     with open(filename.encode(), 'rb') as fp:
 529         first4 = fp.read(4)
 530     if first4 != b'\x50\x4b\x03\x04':
 531         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 532                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 533                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 534
 535     files_in_apk = set()
 536     with zipfile.ZipFile(filename) as zf:
 537         for name in zf.namelist():
 538             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 539                 lib = zf.open(name)
 540                 while True:
 541                     chunk = lib.read(4096)
 542                     if chunk == b'':
 543                         break
 544                     m = has_known_vulnerability.pattern.search(chunk)
 545                     if m:
 546                         version = m.group(1).decode('ascii')
 547                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 548                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 549                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 550                             logging.debug(_('"{path}" contains recent {name} ({version})')
 551                                           .format(path=filename, name=name, version=version))
 552                         else:
 553                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 554                                             .format(path=filename, name=name, version=version))
 555                             found_vuln = True
 556                         break
 557             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 558                 if name in files_in_apk:
 559                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 560                                     .format(apkfilename=filename, name=name))
 561                     found_vuln = True
 562                 files_in_apk.add(name)
 563     return found_vuln
 564
 565
 566 def insert_obbs(repodir, apps, apks):
 567     """Scans the .obb files in a given repo directory and adds them to the
 568     relevant APK instances.  OBB files have versionCodes like APK
 569     files, and they are loosely associated.  If there is an OBB file
 570     present, then any APK with the same or higher versionCode will use
 571     that OBB file.  There are two OBB types: main and patch, each APK
 572     can only have only have one of each.
 573
 574     https://developer.android.com/google/play/expansion-files.html
 575
 576     :param repodir: repo directory to scan
 577     :param apps: list of current, valid apps
 578     :param apks: current information on all APKs
 579
 580     """
 581
 582     def obbWarnDelete(f, msg):
 583         logging.warning(msg + ' ' + f)
 584         if options.delete_unknown:
 585             logging.error(_("Deleting unknown file: {path}").format(path=f))
 586             os.remove(f)
 587
 588     obbs = []
 589     java_Integer_MIN_VALUE = -pow(2, 31)
 590     currentPackageNames = apps.keys()
 591     for f in glob.glob(os.path.join(repodir, '*.obb')):
 592         obbfile = os.path.basename(f)
 593         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 594         chunks = obbfile.split('.')
 595         if chunks[0] != 'main' and chunks[0] != 'patch':
 596             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 597             continue
 598         if not re.match(r'^-?[0-9]+$', chunks[1]):
 599             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 600                           .format(name=chunks[0]))
 601             continue
 602         versionCode = int(chunks[1])
 603         packagename = ".".join(chunks[2:-1])
 604
 605         highestVersionCode = java_Integer_MIN_VALUE
 606         if packagename not in currentPackageNames:
 607             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 608             continue
 609         for apk in apks:
 610             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 611                 highestVersionCode = apk['versionCode']
 612         if versionCode > highestVersionCode:
 613             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 614                           .format(integer=str(versionCode)))
 615             continue
 616         obbsha256 = sha256sum(f)
 617         obbs.append((packagename, versionCode, obbfile, obbsha256))
 618
 619     for apk in apks:
 620         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 621             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 622                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 623                     apk['obbMainFile'] = obbfile
 624                     apk['obbMainFileSha256'] = obbsha256
 625                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 626                     apk['obbPatchFile'] = obbfile
 627                     apk['obbPatchFileSha256'] = obbsha256
 628             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 629                 break
 630
 631
 632 def translate_per_build_anti_features(apps, apks):
 633     """Grab the anti-features list from the build metadata
 634
 635     For most Anti-Features, they are really most applicable per-APK,
 636     not for an app.  An app can fix a vulnerability, add/remove
 637     tracking, etc.  This reads the 'antifeatures' list from the Build
 638     entries in the fdroiddata metadata file, then transforms it into
 639     the 'antiFeatures' list of unique items for the index.
 640
 641     The field key is all lower case in the metadata file to match the
 642     rest of the Build fields.  It is 'antiFeatures' camel case in the
 643     implementation, index, and fdroidclient since it is translated
 644     from the build 'antifeatures' field, not directly included.
 645
 646     """
 647
 648     antiFeatures = dict()
 649     for packageName, app in apps.items():
 650         d = dict()
 651         for build in app['builds']:
 652             afl = build.get('antifeatures')
 653             if afl:
 654                 d[int(build.versionCode)] = afl
 655         if len(d) > 0:
 656             antiFeatures[packageName] = d
 657
 658     for apk in apks:
 659         d = antiFeatures.get(apk['packageName'])
 660         if d:
 661             afl = d.get(apk['versionCode'])
 662             if afl:
 663                 apk['antiFeatures'].update(afl)
 664
 665
 666 def _get_localized_dict(app, locale):
 667     '''get the dict to add localized store metadata to'''
 668     if 'localized' not in app:
 669         app['localized'] = collections.OrderedDict()
 670     if locale not in app['localized']:
 671         app['localized'][locale] = collections.OrderedDict()
 672     return app['localized'][locale]
 673
 674
 675 def _set_localized_text_entry(app, locale, key, f):
 676     limit = config['char_limits'][key]
 677     localized = _get_localized_dict(app, locale)
 678     with open(f) as fp:
 679         text = fp.read()[:limit]
 680         if len(text) > 0:
 681             localized[key] = text
 682
 683
 684 def _set_author_entry(app, key, f):
 685     limit = config['char_limits']['author']
 686     with open(f) as fp:
 687         text = fp.read()[:limit]
 688         if len(text) > 0:
 689             app[key] = text
 690
 691
 692 def _strip_and_copy_image(inpath, outpath):
 693     """Remove any metadata from image and copy it to new path
 694
 695     Sadly, image metadata like EXIF can be used to exploit devices.
 696     It is not used at all in the F-Droid ecosystem, so its much safer
 697     just to remove it entirely.
 698
 699     """
 700
 701     extension = common.get_extension(inpath)[1]
 702     if os.path.isdir(outpath):
 703         outpath = os.path.join(outpath, os.path.basename(inpath))
 704     if extension == 'png':
 705         with open(inpath, 'rb') as fp:
 706             in_image = Image.open(fp)
 707             in_image.save(outpath, "PNG", optimize=True,
 708                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 709     elif extension == 'jpg' or extension == 'jpeg':
 710         with open(inpath, 'rb') as fp:
 711             in_image = Image.open(fp)
 712             data = list(in_image.getdata())
 713             out_image = Image.new(in_image.mode, in_image.size)
 714         out_image.putdata(data)
 715         out_image.save(outpath, "JPEG", optimize=True)
 716     else:
 717         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 718                               .format(extension=extension))
 719
 720
 721 def copy_triple_t_store_metadata(apps):
 722     """Include store metadata from the app's source repo
 723
 724     The Triple-T Gradle Play Publisher is a plugin that has a standard
 725     file layout for all of the metadata and graphics that the Google
 726     Play Store accepts.  Since F-Droid has the git repo, it can just
 727     pluck those files directly.  This method reads any text files into
 728     the app dict, then copies any graphics into the fdroid repo
 729     directory structure.
 730
 731     This needs to be run before insert_localized_app_metadata() so that
 732     the graphics files that are copied into the fdroid repo get
 733     properly indexed.
 734
 735     https://github.com/Triple-T/gradle-play-publisher#upload-images
 736     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 737
 738     """
 739
 740     if not os.path.isdir('build'):
 741         return  # nothing to do
 742
 743     for packageName, app in apps.items():
 744         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 745             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 746             for root, dirs, files in os.walk(d):
 747                 segments = root.split('/')
 748                 locale = segments[-2]
 749                 for f in files:
 750                     if f == 'fulldescription':
 751                         _set_localized_text_entry(app, locale, 'description',
 752                                                   os.path.join(root, f))
 753                         continue
 754                     elif f == 'shortdescription':
 755                         _set_localized_text_entry(app, locale, 'summary',
 756                                                   os.path.join(root, f))
 757                         continue
 758                     elif f == 'title':
 759                         _set_localized_text_entry(app, locale, 'name',
 760                                                   os.path.join(root, f))
 761                         continue
 762                     elif f == 'video':
 763                         _set_localized_text_entry(app, locale, 'video',
 764                                                   os.path.join(root, f))
 765                         continue
 766                     elif f == 'whatsnew':
 767                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 768                                                   os.path.join(root, f))
 769                         continue
 770                     elif f == 'contactEmail':
 771                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 772                         continue
 773                     elif f == 'contactPhone':
 774                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 775                         continue
 776                     elif f == 'contactWebsite':
 777                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 778                         continue
 779
 780                     base, extension = common.get_extension(f)
 781                     dirname = os.path.basename(root)
 782                     if extension in ALLOWED_EXTENSIONS \
 783                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 784                         if segments[-2] == 'listing':
 785                             locale = segments[-3]
 786                         else:
 787                             locale = segments[-2]
 788                         destdir = os.path.join('repo', packageName, locale, dirname)
 789                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 790                         sourcefile = os.path.join(root, f)
 791                         destfile = os.path.join(destdir, os.path.basename(f))
 792                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 793                         _strip_and_copy_image(sourcefile, destfile)
 794
 795
 796 def insert_localized_app_metadata(apps):
 797     """scans standard locations for graphics and localized text
 798
 799     Scans for localized description files, store graphics, and
 800     screenshot PNG files in statically defined screenshots directory
 801     and adds them to the app metadata.  The screenshots and graphic
 802     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 803     and must be in the following layout:
 804     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 805
 806     repo/packageName/locale/featureGraphic.png
 807     repo/packageName/locale/phoneScreenshots/1.png
 808     repo/packageName/locale/phoneScreenshots/2.png
 809
 810     The changelog files must be text files named with the versionCode
 811     ending with ".txt" and must be in the following layout:
 812     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 813
 814     repo/packageName/locale/changelogs/12345.txt
 815
 816     This will scan the each app's source repo then the metadata/ dir
 817     for these standard locations of changelog files.  If it finds
 818     them, they will be added to the dict of all packages, with the
 819     versions in the metadata/ folder taking precendence over the what
 820     is in the app's source repo.
 821
 822     Where "packageName" is the app's packageName and "locale" is the locale
 823     of the graphics, e.g. what language they are in, using the IETF RFC5646
 824     format (en-US, fr-CA, es-MX, etc).
 825
 826     This will also scan the app's git for a fastlane folder, and the
 827     metadata/ folder and the apps' source repos for standard locations
 828     of graphic and screenshot files.  If it finds them, it will copy
 829     them into the repo.  The fastlane files follow this pattern:
 830     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 831
 832     """
 833
 834     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 837     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 838
 839     for srcd in sorted(sourcedirs):
 840         if not os.path.isdir(srcd):
 841             continue
 842         for root, dirs, files in os.walk(srcd):
 843             segments = root.split('/')
 844             packageName = segments[1]
 845             if packageName not in apps:
 846                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 847                 continue
 848             locale = segments[-1]
 849             destdir = os.path.join('repo', packageName, locale)
 850
 851             # flavours specified in build receipt
 852             build_flavours = ""
 853             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 854                     and 'gradle' in apps[packageName].builds[-1]:
 855                 build_flavours = apps[packageName].builds[-1].gradle
 856
 857             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 858                 logging.debug("ignoring due to wrong flavour")
 859                 continue
 860
 861             for f in files:
 862                 if f in ('description.txt', 'full_description.txt'):
 863                     _set_localized_text_entry(apps[packageName], locale, 'description',
 864                                               os.path.join(root, f))
 865                     continue
 866                 elif f in ('summary.txt', 'short_description.txt'):
 867                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 868                                               os.path.join(root, f))
 869                     continue
 870                 elif f in ('name.txt', 'title.txt'):
 871                     _set_localized_text_entry(apps[packageName], locale, 'name',
 872                                               os.path.join(root, f))
 873                     continue
 874                 elif f == 'video.txt':
 875                     _set_localized_text_entry(apps[packageName], locale, 'video',
 876                                               os.path.join(root, f))
 877                     continue
 878                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 879                     locale = segments[-2]
 880                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 881                                               os.path.join(root, f))
 882                     continue
 883
 884                 base, extension = common.get_extension(f)
 885                 if locale == 'images':
 886                     locale = segments[-2]
 887                     destdir = os.path.join('repo', packageName, locale)
 888                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 889                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 890                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 891                     _strip_and_copy_image(os.path.join(root, f), destdir)
 892             for d in dirs:
 893                 if d in SCREENSHOT_DIRS:
 894                     if locale == 'images':
 895                         locale = segments[-2]
 896                         destdir = os.path.join('repo', packageName, locale)
 897                     for f in glob.glob(os.path.join(root, d, '*.*')):
 898                         _ignored, extension = common.get_extension(f)
 899                         if extension in ALLOWED_EXTENSIONS:
 900                             screenshotdestdir = os.path.join(destdir, d)
 901                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 902                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 903                             _strip_and_copy_image(f, screenshotdestdir)
 904
 905     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 906     for d in repofiles:
 907         if not os.path.isdir(d):
 908             continue
 909         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 910             if not os.path.isfile(f):
 911                 continue
 912             segments = f.split('/')
 913             packageName = segments[1]
 914             locale = segments[2]
 915             screenshotdir = segments[3]
 916             filename = os.path.basename(f)
 917             base, extension = common.get_extension(filename)
 918
 919             if packageName not in apps:
 920                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 921                                 .format(path=filename, name=packageName))
 922                 continue
 923             graphics = _get_localized_dict(apps[packageName], locale)
 924
 925             if extension not in ALLOWED_EXTENSIONS:
 926                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 927             elif base in GRAPHIC_NAMES:
 928                 # there can only be zero or one of these per locale
 929                 graphics[base] = filename
 930             elif screenshotdir in SCREENSHOT_DIRS:
 931                 # there can any number of these per locale
 932                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 933                 if screenshotdir not in graphics:
 934                     graphics[screenshotdir] = []
 935                 graphics[screenshotdir].append(filename)
 936             else:
 937                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 938
 939
 940 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 941     """Scan a repo for all files with an extension except APK/OBB
 942
 943     :param apkcache: current cached info about all repo files
 944     :param repodir: repo directory to scan
 945     :param knownapks: list of all known files, as per metadata.read_metadata
 946     :param use_date_from_file: use date from file (instead of current date)
 947                                for newly added files
 948     """
 949
 950     cachechanged = False
 951     repo_files = []
 952     repodir = repodir.encode('utf-8')
 953     for name in os.listdir(repodir):
 954         file_extension = common.get_file_extension(name)
 955         if file_extension == 'apk' or file_extension == 'obb':
 956             continue
 957         filename = os.path.join(repodir, name)
 958         name_utf8 = name.decode('utf-8')
 959         if filename.endswith(b'_src.tar.gz'):
 960             logging.debug(_('skipping source tarball: {path}')
 961                           .format(path=filename.decode('utf-8')))
 962             continue
 963         if not common.is_repo_file(filename):
 964             continue
 965         stat = os.stat(filename)
 966         if stat.st_size == 0:
 967             raise FDroidException(_('{path} is zero size!')
 968                                   .format(path=filename))
 969
 970         shasum = sha256sum(filename)
 971         usecache = False
 972         if name in apkcache:
 973             repo_file = apkcache[name]
 974             # added time is cached as tuple but used here as datetime instance
 975             if 'added' in repo_file:
 976                 a = repo_file['added']
 977                 if isinstance(a, datetime):
 978                     repo_file['added'] = a
 979                 else:
 980                     repo_file['added'] = datetime(*a[:6])
 981             if repo_file.get('hash') == shasum:
 982                 logging.debug(_("Reading {apkfilename} from cache")
 983                               .format(apkfilename=name_utf8))
 984                 usecache = True
 985             else:
 986                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 987                               .format(apkfilename=name_utf8))
 988
 989         if not usecache:
 990             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 991             repo_file = collections.OrderedDict()
 992             repo_file['name'] = os.path.splitext(name_utf8)[0]
 993             # TODO rename apkname globally to something more generic
 994             repo_file['apkName'] = name_utf8
 995             repo_file['hash'] = shasum
 996             repo_file['hashType'] = 'sha256'
 997             repo_file['versionCode'] = 0
 998             repo_file['versionName'] = shasum
 999             # the static ID is the SHA256 unless it is set in the metadata
1000             repo_file['packageName'] = shasum
1001
1002             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1003             if m:
1004                 repo_file['packageName'] = m.group(1)
1005                 repo_file['versionCode'] = int(m.group(2))
1006             srcfilename = name + b'_src.tar.gz'
1007             if os.path.exists(os.path.join(repodir, srcfilename)):
1008                 repo_file['srcname'] = srcfilename.decode('utf-8')
1009             repo_file['size'] = stat.st_size
1010
1011             apkcache[name] = repo_file
1012             cachechanged = True
1013
1014         if use_date_from_file:
1015             timestamp = stat.st_ctime
1016             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1017         else:
1018             default_date_param = None
1019
1020         # Record in knownapks, getting the added date at the same time..
1021         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1022                                     default_date=default_date_param)
1023         if added:
1024             repo_file['added'] = added
1025
1026         repo_files.append(repo_file)
1027
1028     return repo_files, cachechanged
1029
1030
1031 def scan_apk(apk_file):
1032     """
1033     Scans an APK file and returns dictionary with metadata of the APK.
1034
1035     Attention: This does *not* verify that the APK signature is correct.
1036
1037     :param apk_file: The (ideally absolute) path to the APK file
1038     :raises BuildException
1039     :return A dict containing APK metadata
1040     """
1041     apk = {
1042         'hash': sha256sum(apk_file),
1043         'hashType': 'sha256',
1044         'uses-permission': [],
1045         'uses-permission-sdk-23': [],
1046         'features': [],
1047         'icons_src': {},
1048         'icons': {},
1049         'antiFeatures': set(),
1050     }
1051
1052     if common.use_androguard():
1053         scan_apk_androguard(apk, apk_file)
1054     else:
1055         scan_apk_aapt(apk, apk_file)
1056
1057     # Get the signature, or rather the signing key fingerprints
1058     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1059     apk['sig'] = getsig(apk_file)
1060     if not apk['sig']:
1061         raise BuildException("Failed to get apk signature")
1062     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1063                                                                apk_file))
1064     if not apk.get('signer'):
1065         raise BuildException("Failed to get apk signing key fingerprint")
1066
1067     # Get size of the APK
1068     apk['size'] = os.path.getsize(apk_file)
1069
1070     if 'minSdkVersion' not in apk:
1071         logging.warning("No SDK version information found in {0}".format(apk_file))
1072         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
1073     if 'targetSdkVersion' not in apk:
1074         apk['targetSdkVersion'] = apk['minSdkVersion']
1075
1076     # Check for known vulnerabilities
1077     if has_known_vulnerability(apk_file):
1078         apk['antiFeatures'].add('KnownVuln')
1079
1080     return apk
1081
1082
1083 def _get_apk_icons_src(apkfile, icon_name):
1084     """Extract the paths to the app icon in all available densities
1085
1086     """
1087     icons_src = dict()
1088     density_re = re.compile('^res/(.*)/' + icon_name + '\.(png|xml)$')
1089     with zipfile.ZipFile(apkfile) as zf:
1090         for filename in zf.namelist():
1091             m = density_re.match(filename)
1092             if m:
1093                 folder = m.group(1).split('-')
1094                 if len(folder) > 1 and folder[1].endswith('dpi'):
1095                     density = screen_resolutions[folder[1]]
1096                 else:
1097                     density = '160'
1098                 icons_src[density] = m.group(0)
1099     if icons_src.get('-1') is None and '160' in icons_src:
1100         icons_src['-1'] = icons_src['160']
1101     return icons_src
1102
1103
1104 def scan_apk_aapt(apk, apkfile):
1105     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1106     if p.returncode != 0:
1107         if options.delete_unknown:
1108             if os.path.exists(apkfile):
1109                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1110                 os.remove(apkfile)
1111             else:
1112                 logging.error("Could not find {0} to remove it".format(apkfile))
1113         else:
1114             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1115         raise BuildException(_("Invalid APK"))
1116     icon_name = None
1117     for line in p.output.splitlines():
1118         if line.startswith("package:"):
1119             try:
1120                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1121                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1122                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1123             except Exception as e:
1124                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1125         elif line.startswith("application:"):
1126             m = re.match(APK_LABEL_ICON_PAT, line)
1127             if m:
1128                 apk['name'] = m.group(1)
1129                 icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
1130         elif not apk.get('name') and line.startswith("launchable-activity:"):
1131             # Only use launchable-activity as fallback to application
1132             apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)
1133         elif line.startswith("sdkVersion:"):
1134             m = re.match(APK_SDK_VERSION_PAT, line)
1135             if m is None:
1136                 logging.error(line.replace('sdkVersion:', '')
1137                               + ' is not a valid minSdkVersion!')
1138             else:
1139                 apk['minSdkVersion'] = m.group(1)
1140         elif line.startswith("targetSdkVersion:"):
1141             m = re.match(APK_SDK_VERSION_PAT, line)
1142             if m is None:
1143                 logging.error(line.replace('targetSdkVersion:', '')
1144                               + ' is not a valid targetSdkVersion!')
1145             else:
1146                 apk['targetSdkVersion'] = m.group(1)
1147         elif line.startswith("maxSdkVersion:"):
1148             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1149         elif line.startswith("native-code:"):
1150             apk['nativecode'] = []
1151             for arch in line[13:].split(' '):
1152                 apk['nativecode'].append(arch[1:-1])
1153         elif line.startswith('uses-permission:'):
1154             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1155             if perm_match['maxSdkVersion']:
1156                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1157             permission = UsesPermission(
1158                 perm_match['name'],
1159                 perm_match['maxSdkVersion']
1160             )
1161
1162             apk['uses-permission'].append(permission)
1163         elif line.startswith('uses-permission-sdk-23:'):
1164             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1165             if perm_match['maxSdkVersion']:
1166                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1167             permission_sdk_23 = UsesPermissionSdk23(
1168                 perm_match['name'],
1169                 perm_match['maxSdkVersion']
1170             )
1171
1172             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1173
1174         elif line.startswith('uses-feature:'):
1175             feature = re.match(APK_FEATURE_PAT, line).group(1)
1176             # Filter out this, it's only added with the latest SDK tools and
1177             # causes problems for lots of apps.
1178             if feature != "android.hardware.screen.portrait" \
1179                     and feature != "android.hardware.screen.landscape":
1180                 if feature.startswith("android.feature."):
1181                     feature = feature[16:]
1182                 apk['features'].add(feature)
1183     apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1184
1185
1186 def scan_apk_androguard(apk, apkfile):
1187     try:
1188         from androguard.core.bytecodes.apk import APK
1189         apkobject = APK(apkfile)
1190         if apkobject.is_valid_APK():
1191             arsc = apkobject.get_android_resources()
1192         else:
1193             if options.delete_unknown:
1194                 if os.path.exists(apkfile):
1195                     logging.error(_("Failed to get apk information, deleting {path}")
1196                                   .format(path=apkfile))
1197                     os.remove(apkfile)
1198                 else:
1199                     logging.error(_("Could not find {path} to remove it")
1200                                   .format(path=apkfile))
1201             else:
1202                 logging.error(_("Failed to get apk information, skipping {path}")
1203                               .format(path=apkfile))
1204             raise BuildException(_("Invalid APK"))
1205     except ImportError:
1206         raise FDroidException("androguard library is not installed and aapt not present")
1207     except FileNotFoundError:
1208         logging.error(_("Could not open apk file for analysis"))
1209         raise BuildException(_("Invalid APK"))
1210
1211     apk['packageName'] = apkobject.get_package()
1212     apk['versionCode'] = int(apkobject.get_androidversion_code())
1213     apk['versionName'] = apkobject.get_androidversion_name()
1214     if apk['versionName'][0] == "@":
1215         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1216         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1217         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1218     apk['name'] = apkobject.get_app_name()
1219
1220     if apkobject.get_max_sdk_version() is not None:
1221         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1222     if apkobject.get_min_sdk_version() is not None:
1223         apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1224     if apkobject.get_target_sdk_version() is not None:
1225         apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1226
1227     icon_id_str = apkobject.get_element("application", "icon")
1228     if icon_id_str:
1229         icon_id = int(icon_id_str.replace("@", "0x"), 16)
1230         icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1231         apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1232
1233     arch_re = re.compile("^lib/(.*)/.*$")
1234     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1235     if len(arch) >= 1:
1236         apk['nativecode'] = []
1237         apk['nativecode'].extend(sorted(list(arch)))
1238
1239     xml = apkobject.get_android_manifest_xml()
1240
1241     for item in xml.findall('uses-permission'):
1242         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1243         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1244         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1245         permission = UsesPermission(
1246             name,
1247             maxSdkVersion
1248         )
1249         apk['uses-permission'].append(permission)
1250     for name, maxSdkVersion in apkobject.get_uses_implied_permission_list():
1251         permission = UsesPermission(
1252             name,
1253             maxSdkVersion
1254         )
1255         apk['uses-permission'].append(permission)
1256
1257     for item in xml.findall('uses-permission-sdk-23'):
1258         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1259         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1260         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1261         permission_sdk_23 = UsesPermissionSdk23(
1262             name,
1263             maxSdkVersion
1264         )
1265         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1266
1267     for item in xml.findall('uses-feature'):
1268         key = '{' + xml.nsmap['android'] + '}name'
1269         if key not in item.attrib:
1270             continue
1271         feature = str(item.attrib[key])
1272         if feature != "android.hardware.screen.portrait" \
1273                 and feature != "android.hardware.screen.landscape":
1274             if feature.startswith("android.feature."):
1275                 feature = feature[16:]
1276         required = item.attrib.get('{' + xml.nsmap['android'] + '}required')
1277         if required is None or required == 'true':
1278             apk['features'].append(feature)
1279
1280
1281 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1282                 allow_disabled_algorithms=False, archive_bad_sig=False):
1283     """Processes the apk with the given filename in the given repo directory.
1284
1285     This also extracts the icons.
1286
1287     :param apkcache: current apk cache information
1288     :param apkfilename: the filename of the apk to scan
1289     :param repodir: repo directory to scan
1290     :param knownapks: known apks info
1291     :param use_date_from_apk: use date from APK (instead of current date)
1292                               for newly added APKs
1293     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1294                                       disabled algorithms in the signature (e.g. MD5)
1295     :param archive_bad_sig: move APKs with a bad signature to the archive
1296     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1297      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1298     """
1299
1300     apk = {}
1301     apkfile = os.path.join(repodir, apkfilename)
1302
1303     cachechanged = False
1304     usecache = False
1305     if apkfilename in apkcache:
1306         apk = apkcache[apkfilename]
1307         if apk.get('hash') == sha256sum(apkfile):
1308             logging.debug(_("Reading {apkfilename} from cache")
1309                           .format(apkfilename=apkfilename))
1310             usecache = True
1311         else:
1312             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1313                           .format(apkfilename=apkfilename))
1314
1315     if not usecache:
1316         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1317
1318         try:
1319             apk = scan_apk(apkfile)
1320         except BuildException:
1321             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1322                             .format(apkfilename=apkfilename))
1323             return True, None, False
1324
1325         # Check for debuggable apks...
1326         if common.is_apk_and_debuggable(apkfile):
1327             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1328
1329         if options.rename_apks:
1330             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1331             std_short_name = os.path.join(repodir, n)
1332             if apkfile != std_short_name:
1333                 if os.path.exists(std_short_name):
1334                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1335                     if apkfile != std_long_name:
1336                         if os.path.exists(std_long_name):
1337                             dupdir = os.path.join('duplicates', repodir)
1338                             if not os.path.isdir(dupdir):
1339                                 os.makedirs(dupdir, exist_ok=True)
1340                             dupfile = os.path.join('duplicates', std_long_name)
1341                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1342                             os.rename(apkfile, dupfile)
1343                             return True, None, False
1344                         else:
1345                             os.rename(apkfile, std_long_name)
1346                     apkfile = std_long_name
1347                 else:
1348                     os.rename(apkfile, std_short_name)
1349                     apkfile = std_short_name
1350                 apkfilename = apkfile[len(repodir) + 1:]
1351
1352         apk['apkName'] = apkfilename
1353         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1354         if os.path.exists(os.path.join(repodir, srcfilename)):
1355             apk['srcname'] = srcfilename
1356
1357         # verify the jar signature is correct, allow deprecated
1358         # algorithms only if the APK is in the archive.
1359         skipapk = False
1360         if not common.verify_apk_signature(apkfile):
1361             if repodir == 'archive' or allow_disabled_algorithms:
1362                 if common.verify_old_apk_signature(apkfile):
1363                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1364                 else:
1365                     skipapk = True
1366             else:
1367                 skipapk = True
1368
1369         if skipapk:
1370             if archive_bad_sig:
1371                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1372                                 .format(apkfilename=apkfilename))
1373                 move_apk_between_sections(repodir, 'archive', apk)
1374             else:
1375                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1376                                 .format(apkfilename=apkfilename))
1377             return True, None, False
1378
1379         apkzip = zipfile.ZipFile(apkfile, 'r')
1380
1381         manifest = apkzip.getinfo('AndroidManifest.xml')
1382         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1383         if (1980, 0, 0) != manifest.date_time[0:3]:
1384             try:
1385                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1386             except ValueError as e:
1387                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1388                                 .format(apkfilename=apkfile) + str(e))
1389
1390         # extract icons from APK zip file
1391         iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
1392         try:
1393             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1394         finally:
1395             apkzip.close()  # ensure that APK zip file gets closed
1396
1397         # resize existing icons for densities missing in the APK
1398         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1399
1400         if use_date_from_apk and manifest.date_time[1] != 0:
1401             default_date_param = datetime(*manifest.date_time)
1402         else:
1403             default_date_param = None
1404
1405         # Record in known apks, getting the added date at the same time..
1406         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1407                                     default_date=default_date_param)
1408         if added:
1409             apk['added'] = added
1410
1411         apkcache[apkfilename] = apk
1412         cachechanged = True
1413
1414     return False, apk, cachechanged
1415
1416
1417 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1418     """Processes the apks in the given repo directory.
1419
1420     This also extracts the icons.
1421
1422     :param apkcache: current apk cache information
1423     :param repodir: repo directory to scan
1424     :param knownapks: known apks info
1425     :param use_date_from_apk: use date from APK (instead of current date)
1426                               for newly added APKs
1427     :returns: (apks, cachechanged) where apks is a list of apk information,
1428               and cachechanged is True if the apkcache got changed.
1429     """
1430
1431     cachechanged = False
1432
1433     for icon_dir in get_all_icon_dirs(repodir):
1434         if os.path.exists(icon_dir):
1435             if options.clean:
1436                 shutil.rmtree(icon_dir)
1437                 os.makedirs(icon_dir)
1438         else:
1439             os.makedirs(icon_dir)
1440
1441     apks = []
1442     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1443         apkfilename = apkfile[len(repodir) + 1:]
1444         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1445         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1446                                              use_date_from_apk, ada, True)
1447         if skip:
1448             continue
1449         apks.append(apk)
1450         cachechanged = cachechanged or cachethis
1451
1452     return apks, cachechanged
1453
1454
1455 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1456     """Extracts PNG icons from an APK with the supported pixel densities
1457
1458     Extracts icons from the given APK zip in various densities, saves
1459     them into given repo directory and stores their names in the APK
1460     metadata dictionary.  If the icon is an XML icon, then this tries
1461     to find PNG icon that can replace it.
1462
1463     :param icon_filename: A string representing the icon's file name
1464     :param apk: A populated dictionary containing APK metadata.
1465                 Needs to have 'icons_src' key
1466     :param apkzip: An opened zipfile.ZipFile of the APK file
1467     :param repo_dir: The directory of the APK's repository
1468     :return: A list of icon densities that are missing
1469
1470     """
1471     res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1472     pngs = dict()
1473     for f in apkzip.namelist():
1474         m = res_name_re.match(f)
1475         if m and m.group(4) == 'png':
1476             density = screen_resolutions[m.group(2)]
1477             pngs[m.group(3) + '/' + density] = m.group(0)
1478
1479     icon_type = None
1480     empty_densities = []
1481     for density in screen_densities:
1482         if density not in apk['icons_src']:
1483             empty_densities.append(density)
1484             continue
1485         icon_src = apk['icons_src'][density]
1486         icon_dir = get_icon_dir(repo_dir, density)
1487         icon_type = '.png'
1488
1489         # Extract the icon files per density
1490         if icon_src.endswith('.xml'):
1491             m = res_name_re.match(icon_src)
1492             if m:
1493                 name = pngs.get(m.group(3) + '/' + str(density))
1494                 if name:
1495                     icon_src = name
1496             if icon_src.endswith('.xml'):
1497                 empty_densities.append(density)
1498                 icon_type = '.xml'
1499         icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
1500
1501         try:
1502             with open(icon_dest, 'wb') as f:
1503                 f.write(get_icon_bytes(apkzip, icon_src))
1504             apk['icons'][density] = icon_filename + icon_type
1505         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1506             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1507             del apk['icons_src'][density]
1508             empty_densities.append(density)
1509
1510     # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
1511     if '-1' in apk['icons_src']:
1512         icon_src = apk['icons_src']['-1']
1513         icon_type = icon_src[-4:]
1514         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
1515         with open(icon_path, 'wb') as f:
1516             f.write(get_icon_bytes(apkzip, icon_src))
1517         if icon_type == '.png':
1518             im = None
1519             try:
1520                 im = Image.open(icon_path)
1521                 dpi = px_to_dpi(im.size[0])
1522                 for density in screen_densities:
1523                     if density in apk['icons']:
1524                         break
1525                     if density == screen_densities[-1] or dpi >= int(density):
1526                         apk['icons'][density] = icon_filename + icon_type
1527                         shutil.move(icon_path,
1528                                     os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
1529                         empty_densities.remove(density)
1530                         break
1531             except Exception as e:
1532                 logging.warning(_("Failed reading {path}: {error}")
1533                                 .format(path=icon_path, error=e))
1534             finally:
1535                 if im and hasattr(im, 'close'):
1536                     im.close()
1537
1538     if apk['icons']:
1539         apk['icon'] = icon_filename + icon_type
1540
1541     return empty_densities
1542
1543
1544 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1545     """
1546     Resize existing PNG icons for densities missing in the APK to ensure all densities are available
1547
1548     :param empty_densities: A list of icon densities that are missing
1549     :param icon_filename: A string representing the icon's file name
1550     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1551     :param repo_dir: The directory of the APK's repository
1552
1553     """
1554     icon_filename += '.png'
1555     # First try resizing down to not lose quality
1556     last_density = None
1557     for density in screen_densities:
1558         if density == '65534':  # not possible to generate 'anydpi' from other densities
1559             continue
1560         if density not in empty_densities:
1561             last_density = density
1562             continue
1563         if last_density is None:
1564             continue
1565         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1566
1567         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1568         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1569         fp = None
1570         try:
1571             fp = open(last_icon_path, 'rb')
1572             im = Image.open(fp)
1573
1574             size = dpi_to_px(density)
1575
1576             im.thumbnail((size, size), Image.ANTIALIAS)
1577             im.save(icon_path, "PNG", optimize=True,
1578                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1579             empty_densities.remove(density)
1580         except Exception as e:
1581             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1582         finally:
1583             if fp:
1584                 fp.close()
1585
1586     # Then just copy from the highest resolution available
1587     last_density = None
1588     for density in reversed(screen_densities):
1589         if density not in empty_densities:
1590             last_density = density
1591             continue
1592
1593         if last_density is None:
1594             continue
1595
1596         shutil.copyfile(
1597             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1598             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1599         )
1600         empty_densities.remove(density)
1601
1602     for density in screen_densities:
1603         icon_dir = get_icon_dir(repo_dir, density)
1604         icon_dest = os.path.join(icon_dir, icon_filename)
1605         resize_icon(icon_dest, density)
1606
1607     # Copy from icons-mdpi to icons since mdpi is the baseline density
1608     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1609     if os.path.isfile(baseline):
1610         apk['icons']['0'] = icon_filename
1611         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1612
1613
1614 def apply_info_from_latest_apk(apps, apks):
1615     """
1616     Some information from the apks needs to be applied up to the application level.
1617     When doing this, we use the info from the most recent version's apk.
1618     We deal with figuring out when the app was added and last updated at the same time.
1619     """
1620     for appid, app in apps.items():
1621         bestver = UNSET_VERSION_CODE
1622         for apk in apks:
1623             if apk['packageName'] == appid:
1624                 if apk['versionCode'] > bestver:
1625                     bestver = apk['versionCode']
1626                     bestapk = apk
1627
1628                 if 'added' in apk:
1629                     if not app.added or apk['added'] < app.added:
1630                         app.added = apk['added']
1631                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1632                         app.lastUpdated = apk['added']
1633
1634         if not app.added:
1635             logging.debug("Don't know when " + appid + " was added")
1636         if not app.lastUpdated:
1637             logging.debug("Don't know when " + appid + " was last updated")
1638
1639         if bestver == UNSET_VERSION_CODE:
1640
1641             if app.Name is None:
1642                 app.Name = app.AutoName or appid
1643             app.icon = None
1644             logging.debug("Application " + appid + " has no packages")
1645         else:
1646             if app.Name is None:
1647                 app.Name = bestapk['name']
1648             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1649             if app.CurrentVersionCode is None:
1650                 app.CurrentVersionCode = str(bestver)
1651
1652
1653 def make_categories_txt(repodir, categories):
1654     '''Write a category list in the repo to allow quick access'''
1655     catdata = ''
1656     for cat in sorted(categories):
1657         catdata += cat + '\n'
1658     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1659         f.write(catdata)
1660
1661
1662 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1663
1664     def filter_apk_list_sorted(apk_list):
1665         res = []
1666         for apk in apk_list:
1667             if apk['packageName'] == appid:
1668                 res.append(apk)
1669
1670         # Sort the apk list by version code. First is highest/newest.
1671         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1672
1673     for appid, app in apps.items():
1674
1675         if app.ArchivePolicy:
1676             keepversions = int(app.ArchivePolicy[:-9])
1677         else:
1678             keepversions = defaultkeepversions
1679
1680         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1681                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1682
1683         current_app_apks = filter_apk_list_sorted(apks)
1684         if len(current_app_apks) > keepversions:
1685             # Move back the ones we don't want.
1686             for apk in current_app_apks[keepversions:]:
1687                 move_apk_between_sections(repodir, archivedir, apk)
1688                 archapks.append(apk)
1689                 apks.remove(apk)
1690
1691         current_app_archapks = filter_apk_list_sorted(archapks)
1692         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1693             kept = 0
1694             # Move forward the ones we want again, except DisableAlgorithm
1695             for apk in current_app_archapks:
1696                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1697                     move_apk_between_sections(archivedir, repodir, apk)
1698                     archapks.remove(apk)
1699                     apks.append(apk)
1700                     kept += 1
1701                 if kept == keepversions:
1702                     break
1703
1704
1705 def move_apk_between_sections(from_dir, to_dir, apk):
1706     """move an APK from repo to archive or vice versa"""
1707
1708     def _move_file(from_dir, to_dir, filename, ignore_missing):
1709         from_path = os.path.join(from_dir, filename)
1710         if ignore_missing and not os.path.exists(from_path):
1711             return
1712         to_path = os.path.join(to_dir, filename)
1713         if not os.path.exists(to_dir):
1714             os.mkdir(to_dir)
1715         shutil.move(from_path, to_path)
1716
1717     if from_dir == to_dir:
1718         return
1719
1720     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1721     _move_file(from_dir, to_dir, apk['apkName'], False)
1722     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1723     for density in all_screen_densities:
1724         from_icon_dir = get_icon_dir(from_dir, density)
1725         to_icon_dir = get_icon_dir(to_dir, density)
1726         if density not in apk.get('icons', []):
1727             continue
1728         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1729     if 'srcname' in apk:
1730         _move_file(from_dir, to_dir, apk['srcname'], False)
1731
1732
1733 def add_apks_to_per_app_repos(repodir, apks):
1734     apks_per_app = dict()
1735     for apk in apks:
1736         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1737         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1738         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1739         apks_per_app[apk['packageName']] = apk
1740
1741         if not os.path.exists(apk['per_app_icons']):
1742             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1743             os.makedirs(apk['per_app_icons'])
1744
1745         apkpath = os.path.join(repodir, apk['apkName'])
1746         shutil.copy(apkpath, apk['per_app_repo'])
1747         apksigpath = apkpath + '.sig'
1748         if os.path.exists(apksigpath):
1749             shutil.copy(apksigpath, apk['per_app_repo'])
1750         apkascpath = apkpath + '.asc'
1751         if os.path.exists(apkascpath):
1752             shutil.copy(apkascpath, apk['per_app_repo'])
1753
1754
1755 def create_metadata_from_template(apk):
1756     '''create a new metadata file using internal or external template
1757
1758     Generate warnings for apk's with no metadata (or create skeleton
1759     metadata files, if requested on the command line).  Though the
1760     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1761     since those impose things on the metadata file made from the
1762     template: field sort order, empty field value, formatting, etc.
1763     '''
1764
1765     import yaml
1766     if os.path.exists('template.yml'):
1767         with open('template.yml') as f:
1768             metatxt = f.read()
1769         if 'name' in apk and apk['name'] != '':
1770             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1771                              r'\1 ' + apk['name'],
1772                              metatxt,
1773                              flags=re.IGNORECASE | re.MULTILINE)
1774         else:
1775             logging.warning(_('{appid} does not have a name! Using package name instead.')
1776                             .format(appid=apk['packageName']))
1777             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1778                              r'\1 ' + apk['packageName'],
1779                              metatxt,
1780                              flags=re.IGNORECASE | re.MULTILINE)
1781         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1782             f.write(metatxt)
1783     else:
1784         app = dict()
1785         app['Categories'] = [os.path.basename(os.getcwd())]
1786         # include some blanks as part of the template
1787         app['AuthorName'] = ''
1788         app['Summary'] = ''
1789         app['WebSite'] = ''
1790         app['IssueTracker'] = ''
1791         app['SourceCode'] = ''
1792         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1793         if 'name' in apk and apk['name'] != '':
1794             app['Name'] = apk['name']
1795         else:
1796             logging.warning(_('{appid} does not have a name! Using package name instead.')
1797                             .format(appid=apk['packageName']))
1798             app['Name'] = apk['packageName']
1799         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1800             yaml.dump(app, f, default_flow_style=False)
1801     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1802
1803
1804 config = None
1805 options = None
1806 start_timestamp = time.gmtime()
1807
1808
1809 def main():
1810
1811     global config, options
1812
1813     # Parse command line...
1814     parser = ArgumentParser()
1815     common.setup_global_opts(parser)
1816     parser.add_argument("--create-key", action="store_true", default=False,
1817                         help=_("Add a repo signing key to an unsigned repo"))
1818     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1819                         help=_("Add skeleton metadata files for APKs that are missing them"))
1820     parser.add_argument("--delete-unknown", action="store_true", default=False,
1821                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1822     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1823                         help=_("Report on build data status"))
1824     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1825                         help=_("Interactively ask about things that need updating."))
1826     parser.add_argument("-I", "--icons", action="store_true", default=False,
1827                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1828     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1829                         help=_("Specify editor to use in interactive mode. Default " +
1830                                "is {path}").format(path='/etc/alternatives/editor'))
1831     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1832                         help=_("Update the wiki"))
1833     parser.add_argument("--pretty", action="store_true", default=False,
1834                         help=_("Produce human-readable XML/JSON for index files"))
1835     parser.add_argument("--clean", action="store_true", default=False,
1836                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1837     parser.add_argument("--nosign", action="store_true", default=False,
1838                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1839     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1840                         help=_("Use date from APK instead of current time for newly added APKs"))
1841     parser.add_argument("--rename-apks", action="store_true", default=False,
1842                         help=_("Rename APK files that do not match package.name_123.apk"))
1843     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1844                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1845     metadata.add_metadata_arguments(parser)
1846     options = parser.parse_args()
1847     metadata.warnings_action = options.W
1848
1849     config = common.read_config(options)
1850
1851     if not ('jarsigner' in config and 'keytool' in config):
1852         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1853
1854     repodirs = ['repo']
1855     if config['archive_older'] != 0:
1856         repodirs.append('archive')
1857         if not os.path.exists('archive'):
1858             os.mkdir('archive')
1859
1860     if options.icons:
1861         resize_all_icons(repodirs)
1862         sys.exit(0)
1863
1864     if options.rename_apks:
1865         options.clean = True
1866
1867     # check that icons exist now, rather than fail at the end of `fdroid update`
1868     for k in ['repo_icon', 'archive_icon']:
1869         if k in config:
1870             if not os.path.exists(config[k]):
1871                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1872                                  .format(name=k, path=config[k]))
1873                 sys.exit(1)
1874
1875     # if the user asks to create a keystore, do it now, reusing whatever it can
1876     if options.create_key:
1877         if os.path.exists(config['keystore']):
1878             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1879             logging.critical("\t'" + config['keystore'] + "'")
1880             sys.exit(1)
1881
1882         if 'repo_keyalias' not in config:
1883             config['repo_keyalias'] = socket.getfqdn()
1884             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1885         if 'keydname' not in config:
1886             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1887             common.write_to_config(config, 'keydname', config['keydname'])
1888         if 'keystore' not in config:
1889             config['keystore'] = common.default_config['keystore']
1890             common.write_to_config(config, 'keystore', config['keystore'])
1891
1892         password = common.genpassword()
1893         if 'keystorepass' not in config:
1894             config['keystorepass'] = password
1895             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1896         if 'keypass' not in config:
1897             config['keypass'] = password
1898             common.write_to_config(config, 'keypass', config['keypass'])
1899         common.genkeystore(config)
1900
1901     # Get all apps...
1902     apps = metadata.read_metadata()
1903
1904     # Generate a list of categories...
1905     categories = set()
1906     for app in apps.values():
1907         categories.update(app.Categories)
1908
1909     # Read known apks data (will be updated and written back when we've finished)
1910     knownapks = common.KnownApks()
1911
1912     # Get APK cache
1913     apkcache = get_cache()
1914
1915     # Delete builds for disabled apps
1916     delete_disabled_builds(apps, apkcache, repodirs)
1917
1918     # Scan all apks in the main repo
1919     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1920
1921     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1922                                            options.use_date_from_apk)
1923     cachechanged = cachechanged or fcachechanged
1924     apks += files
1925     for apk in apks:
1926         if apk['packageName'] not in apps:
1927             if options.create_metadata:
1928                 create_metadata_from_template(apk)
1929                 apps = metadata.read_metadata()
1930             else:
1931                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1932                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1933                 if options.delete_unknown:
1934                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1935                                  .format(apkfilename=apk['apkName']))
1936                     rmf = os.path.join(repodirs[0], apk['apkName'])
1937                     if not os.path.exists(rmf):
1938                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1939                     else:
1940                         os.remove(rmf)
1941                 else:
1942                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1943
1944     copy_triple_t_store_metadata(apps)
1945     insert_obbs(repodirs[0], apps, apks)
1946     insert_localized_app_metadata(apps)
1947     translate_per_build_anti_features(apps, apks)
1948
1949     # Scan the archive repo for apks as well
1950     if len(repodirs) > 1:
1951         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1952         if cc:
1953             cachechanged = True
1954     else:
1955         archapks = []
1956
1957     # Apply information from latest apks to the application and update dates
1958     apply_info_from_latest_apk(apps, apks + archapks)
1959
1960     # Sort the app list by name, then the web site doesn't have to by default.
1961     # (we had to wait until we'd scanned the apks to do this, because mostly the
1962     # name comes from there!)
1963     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1964
1965     # APKs are placed into multiple repos based on the app package, providing
1966     # per-app subscription feeds for nightly builds and things like it
1967     if config['per_app_repos']:
1968         add_apks_to_per_app_repos(repodirs[0], apks)
1969         for appid, app in apps.items():
1970             repodir = os.path.join(appid, 'fdroid', 'repo')
1971             appdict = dict()
1972             appdict[appid] = app
1973             if os.path.isdir(repodir):
1974                 index.make(appdict, [appid], apks, repodir, False)
1975             else:
1976                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1977         return
1978
1979     if len(repodirs) > 1:
1980         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1981
1982     # Make the index for the main repo...
1983     index.make(apps, sortedids, apks, repodirs[0], False)
1984     make_categories_txt(repodirs[0], categories)
1985
1986     # If there's an archive repo,  make the index for it. We already scanned it
1987     # earlier on.
1988     if len(repodirs) > 1:
1989         index.make(apps, sortedids, archapks, repodirs[1], True)
1990
1991     git_remote = config.get('binary_transparency_remote')
1992     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1993         from . import btlog
1994         btlog.make_binary_transparency_log(repodirs)
1995
1996     if config['update_stats']:
1997         # Update known apks info...
1998         knownapks.writeifchanged()
1999
2000         # Generate latest apps data for widget
2001         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
2002             data = ''
2003             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
2004                 for line in f:
2005                     appid = line.rstrip()
2006                     data += appid + "\t"
2007                     app = apps[appid]
2008                     data += app.Name + "\t"
2009                     if app.icon is not None:
2010                         data += app.icon + "\t"
2011                     data += app.License + "\n"
2012             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
2013                 f.write(data)
2014
2015     if cachechanged:
2016         write_cache(apkcache)
2017
2018     # Update the wiki...
2019     if options.wiki:
2020         update_wiki(apps, sortedids, apks + archapks)
2021
2022     logging.info(_("Finished"))
2023
2024
2025 if __name__ == "__main__":
2026     main()