fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0':
 100         return os.path.join(repodir, "icons")
 101     return os.path.join(repodir, "icons-%s" % density)
 102
 103
 104 def get_icon_dirs(repodir):
 105     for density in screen_densities:
 106         yield get_icon_dir(repodir, density)
 107
 108
 109 def get_all_icon_dirs(repodir):
 110     for density in all_screen_densities:
 111         yield get_icon_dir(repodir, density)
 112
 113
 114 def update_wiki(apps, sortedids, apks):
 115     """Update the wiki
 116
 117     :param apps: fully populated list of all applications
 118     :param apks: all apks, except...
 119     """
 120     logging.info("Updating wiki")
 121     wikicat = 'Apps'
 122     wikiredircat = 'App Redirects'
 123     import mwclient
 124     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 125                          path=config['wiki_path'])
 126     site.login(config['wiki_user'], config['wiki_password'])
 127     generated_pages = {}
 128     generated_redirects = {}
 129
 130     for appid in sortedids:
 131         app = metadata.App(apps[appid])
 132
 133         wikidata = ''
 134         if app.Disabled:
 135             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 136         if app.AntiFeatures:
 137             for af in sorted(app.AntiFeatures):
 138                 wikidata += '{{AntiFeature|' + af + '}}\n'
 139         if app.RequiresRoot:
 140             requiresroot = 'Yes'
 141         else:
 142             requiresroot = 'No'
 143         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 144             appid,
 145             app.Name,
 146             app.added.strftime('%Y-%m-%d') if app.added else '',
 147             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 148             app.SourceCode,
 149             app.IssueTracker,
 150             app.WebSite,
 151             app.Changelog,
 152             app.Donate,
 153             app.FlattrID,
 154             app.LiberapayID,
 155             app.Bitcoin,
 156             app.Litecoin,
 157             app.License,
 158             requiresroot,
 159             app.AuthorName,
 160             app.AuthorEmail)
 161
 162         if app.Provides:
 163             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 164
 165         wikidata += app.Summary
 166         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 167
 168         wikidata += "=Description=\n"
 169         wikidata += metadata.description_wiki(app.Description) + "\n"
 170
 171         wikidata += "=Maintainer Notes=\n"
 172         if app.MaintainerNotes:
 173             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 174         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 175
 176         # Get a list of all packages for this application...
 177         apklist = []
 178         gotcurrentver = False
 179         cantupdate = False
 180         buildfails = False
 181         for apk in apks:
 182             if apk['packageName'] == appid:
 183                 if str(apk['versionCode']) == app.CurrentVersionCode:
 184                     gotcurrentver = True
 185                 apklist.append(apk)
 186         # Include ones we can't build, as a special case...
 187         for build in app.builds:
 188             if build.disable:
 189                 if build.versionCode == app.CurrentVersionCode:
 190                     cantupdate = True
 191                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 192                 apklist.append({'versionCode': int(build.versionCode),
 193                                 'versionName': build.versionName,
 194                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 195                                 })
 196             else:
 197                 builtit = False
 198                 for apk in apklist:
 199                     if apk['versionCode'] == int(build.versionCode):
 200                         builtit = True
 201                         break
 202                 if not builtit:
 203                     buildfails = True
 204                     apklist.append({'versionCode': int(build.versionCode),
 205                                     'versionName': build.versionName,
 206                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 207                                     })
 208         if app.CurrentVersionCode == '0':
 209             cantupdate = True
 210         # Sort with most recent first...
 211         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 212
 213         wikidata += "=Versions=\n"
 214         if len(apklist) == 0:
 215             wikidata += "We currently have no versions of this app available."
 216         elif not gotcurrentver:
 217             wikidata += "We don't have the current version of this app."
 218         else:
 219             wikidata += "We have the current version of this app."
 220         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 221         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 222         if len(app.NoSourceSince) > 0:
 223             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 224         if len(app.CurrentVersion) > 0:
 225             wikidata += "The current (recommended) version is " + app.CurrentVersion
 226             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 227         validapks = 0
 228         for apk in apklist:
 229             wikidata += "==" + apk['versionName'] + "==\n"
 230
 231             if 'buildproblem' in apk:
 232                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 233             else:
 234                 validapks += 1
 235                 wikidata += "This version is built and signed by "
 236                 if 'srcname' in apk:
 237                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 238                 else:
 239                     wikidata += "the original developer.\n\n"
 240             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 241
 242         wikidata += '\n[[Category:' + wikicat + ']]\n'
 243         if len(app.NoSourceSince) > 0:
 244             wikidata += '\n[[Category:Apps missing source code]]\n'
 245         if validapks == 0 and not app.Disabled:
 246             wikidata += '\n[[Category:Apps with no packages]]\n'
 247         if cantupdate and not app.Disabled:
 248             wikidata += "\n[[Category:Apps we cannot update]]\n"
 249         if buildfails and not app.Disabled:
 250             wikidata += "\n[[Category:Apps with failing builds]]\n"
 251         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 252             wikidata += '\n[[Category:Apps to Update]]\n'
 253         if app.Disabled:
 254             wikidata += '\n[[Category:Apps that are disabled]]\n'
 255         if app.UpdateCheckMode == 'None' and not app.Disabled:
 256             wikidata += '\n[[Category:Apps with no update check]]\n'
 257         for appcat in app.Categories:
 258             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 259
 260         # We can't have underscores in the page name, even if they're in
 261         # the package ID, because MediaWiki messes with them...
 262         pagename = appid.replace('_', ' ')
 263
 264         # Drop a trailing newline, because mediawiki is going to drop it anyway
 265         # and it we don't we'll think the page has changed when it hasn't...
 266         if wikidata.endswith('\n'):
 267             wikidata = wikidata[:-1]
 268
 269         generated_pages[pagename] = wikidata
 270
 271         # Make a redirect from the name to the ID too, unless there's
 272         # already an existing page with the name and it isn't a redirect.
 273         noclobber = False
 274         apppagename = app.Name.replace('_', ' ')
 275         apppagename = apppagename.replace('{', '')
 276         apppagename = apppagename.replace('}', ' ')
 277         apppagename = apppagename.replace(':', ' ')
 278         apppagename = apppagename.replace('[', ' ')
 279         apppagename = apppagename.replace(']', ' ')
 280         # Drop double spaces caused mostly by replacing ':' above
 281         apppagename = apppagename.replace('  ', ' ')
 282         for expagename in site.allpages(prefix=apppagename,
 283                                         filterredir='nonredirects',
 284                                         generator=False):
 285             if expagename == apppagename:
 286                 noclobber = True
 287         # Another reason not to make the redirect page is if the app name
 288         # is the same as it's ID, because that will overwrite the real page
 289         # with an redirect to itself! (Although it seems like an odd
 290         # scenario this happens a lot, e.g. where there is metadata but no
 291         # builds or binaries to extract a name from.
 292         if apppagename == pagename:
 293             noclobber = True
 294         if not noclobber:
 295             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 296
 297     for tcat, genp in [(wikicat, generated_pages),
 298                        (wikiredircat, generated_redirects)]:
 299         catpages = site.Pages['Category:' + tcat]
 300         existingpages = []
 301         for page in catpages:
 302             existingpages.append(page.name)
 303             if page.name in genp:
 304                 pagetxt = page.edit()
 305                 if pagetxt != genp[page.name]:
 306                     logging.debug("Updating modified page " + page.name)
 307                     page.save(genp[page.name], summary='Auto-updated')
 308                 else:
 309                     logging.debug("Page " + page.name + " is unchanged")
 310             else:
 311                 logging.warn("Deleting page " + page.name)
 312                 page.delete('No longer published')
 313         for pagename, text in genp.items():
 314             logging.debug("Checking " + pagename)
 315             if pagename not in existingpages:
 316                 logging.debug("Creating page " + pagename)
 317                 try:
 318                     newpage = site.Pages[pagename]
 319                     newpage.save(text, summary='Auto-created')
 320                 except Exception as e:
 321                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 322
 323     # Purge server cache to ensure counts are up to date
 324     site.Pages['Repository Maintenance'].purge()
 325
 326     # Write a page with the last build log for this version code
 327     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 328     newpage = site.Pages[wiki_page_path]
 329     txt = ''
 330     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 331     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 332     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 333     txt += "\n\n"
 334     txt += common.get_android_tools_version_log()
 335     newpage.save(txt, summary='Run log')
 336     newpage = site.Pages['update']
 337     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 338
 339
 340 def delete_disabled_builds(apps, apkcache, repodirs):
 341     """Delete disabled build outputs.
 342
 343     :param apps: list of all applications, as per metadata.read_metadata
 344     :param apkcache: current apk cache information
 345     :param repodirs: the repo directories to process
 346     """
 347     for appid, app in apps.items():
 348         for build in app['builds']:
 349             if not build.disable:
 350                 continue
 351             apkfilename = common.get_release_filename(app, build)
 352             iconfilename = "%s.%s.png" % (
 353                 appid,
 354                 build.versionCode)
 355             for repodir in repodirs:
 356                 files = [
 357                     os.path.join(repodir, apkfilename),
 358                     os.path.join(repodir, apkfilename + '.asc'),
 359                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 360                 ]
 361                 for density in all_screen_densities:
 362                     repo_dir = get_icon_dir(repodir, density)
 363                     files.append(os.path.join(repo_dir, iconfilename))
 364
 365                 for f in files:
 366                     if os.path.exists(f):
 367                         logging.info("Deleting disabled build output " + f)
 368                         os.remove(f)
 369             if apkfilename in apkcache:
 370                 del apkcache[apkfilename]
 371
 372
 373 def resize_icon(iconpath, density):
 374
 375     if not os.path.isfile(iconpath):
 376         return
 377
 378     fp = None
 379     try:
 380         fp = open(iconpath, 'rb')
 381         im = Image.open(fp)
 382         size = dpi_to_px(density)
 383
 384         if any(length > size for length in im.size):
 385             oldsize = im.size
 386             im.thumbnail((size, size), Image.ANTIALIAS)
 387             logging.debug("%s was too large at %s - new size is %s" % (
 388                 iconpath, oldsize, im.size))
 389             im.save(iconpath, "PNG", optimize=True,
 390                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 391
 392     except Exception as e:
 393         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 394
 395     finally:
 396         if fp:
 397             fp.close()
 398
 399
 400 def resize_all_icons(repodirs):
 401     """Resize all icons that exceed the max size
 402
 403     :param repodirs: the repo directories to process
 404     """
 405     for repodir in repodirs:
 406         for density in screen_densities:
 407             icon_dir = get_icon_dir(repodir, density)
 408             icon_glob = os.path.join(icon_dir, '*.png')
 409             for iconpath in glob.glob(icon_glob):
 410                 resize_icon(iconpath, density)
 411
 412
 413 def getsig(apkpath):
 414     """ Get the signing certificate of an apk. To get the same md5 has that
 415     Android gets, we encode the .RSA certificate in a specific format and pass
 416     it hex-encoded to the md5 digest algorithm.
 417
 418     :param apkpath: path to the apk
 419     :returns: A string containing the md5 of the signature of the apk or None
 420               if an error occurred.
 421     """
 422
 423     with zipfile.ZipFile(apkpath, 'r') as apk:
 424         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 425
 426         if len(certs) < 1:
 427             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 428             return None
 429         if len(certs) > 1:
 430             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 431             return None
 432
 433         cert = apk.read(certs[0])
 434
 435     cert_encoded = common.get_certificate(cert)
 436
 437     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 438
 439
 440 def get_cache_file():
 441     return os.path.join('tmp', 'apkcache')
 442
 443
 444 def get_cache():
 445     """Get the cached dict of the APK index
 446
 447     Gather information about all the apk files in the repo directory,
 448     using cached data if possible. Some of the index operations take a
 449     long time, like calculating the SHA-256 and verifying the APK
 450     signature.
 451
 452     The cache is invalidated if the metadata version is different, or
 453     the 'allow_disabled_algorithms' config/option is different.  In
 454     those cases, there is no easy way to know what has changed from
 455     the cache, so just rerun the whole thing.
 456
 457     :return: apkcache
 458
 459     """
 460     apkcachefile = get_cache_file()
 461     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 462     if not options.clean and os.path.exists(apkcachefile):
 463         with open(apkcachefile, 'rb') as cf:
 464             apkcache = pickle.load(cf, encoding='utf-8')
 465         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 466            or apkcache.get('allow_disabled_algorithms') != ada:
 467             apkcache = {}
 468     else:
 469         apkcache = {}
 470
 471     apkcache["METADATA_VERSION"] = METADATA_VERSION
 472     apkcache['allow_disabled_algorithms'] = ada
 473
 474     return apkcache
 475
 476
 477 def write_cache(apkcache):
 478     apkcachefile = get_cache_file()
 479     cache_path = os.path.dirname(apkcachefile)
 480     if not os.path.exists(cache_path):
 481         os.makedirs(cache_path)
 482     with open(apkcachefile, 'wb') as cf:
 483         pickle.dump(apkcache, cf)
 484
 485
 486 def get_icon_bytes(apkzip, iconsrc):
 487     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 488     try:
 489         return apkzip.read(iconsrc)
 490     except KeyError:
 491         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 492
 493
 494 def sha256sum(filename):
 495     '''Calculate the sha256 of the given file'''
 496     sha = hashlib.sha256()
 497     with open(filename, 'rb') as f:
 498         while True:
 499             t = f.read(16384)
 500             if len(t) == 0:
 501                 break
 502             sha.update(t)
 503     return sha.hexdigest()
 504
 505
 506 def has_known_vulnerability(filename):
 507     """checks for known vulnerabilities in the APK
 508
 509     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 510     version.  Google also enforces this:
 511     https://support.google.com/faqs/answer/6376725?hl=en
 512
 513     Checks whether there are more than one classes.dex or AndroidManifest.xml
 514     files, which is invalid and an essential part of the "Master Key" attack.
 515     http://www.saurik.com/id/17
 516
 517     Janus is similar to Master Key but is perhaps easier to scan for.
 518     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 519     """
 520
 521     found_vuln = False
 522
 523     # statically load this pattern
 524     if not hasattr(has_known_vulnerability, "pattern"):
 525         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 526
 527     with open(filename.encode(), 'rb') as fp:
 528         first4 = fp.read(4)
 529     if first4 != b'\x50\x4b\x03\x04':
 530         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 531                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 532                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 533
 534     files_in_apk = set()
 535     with zipfile.ZipFile(filename) as zf:
 536         for name in zf.namelist():
 537             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 538                 lib = zf.open(name)
 539                 while True:
 540                     chunk = lib.read(4096)
 541                     if chunk == b'':
 542                         break
 543                     m = has_known_vulnerability.pattern.search(chunk)
 544                     if m:
 545                         version = m.group(1).decode('ascii')
 546                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 547                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 548                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 549                             logging.debug(_('"{path}" contains recent {name} ({version})')
 550                                           .format(path=filename, name=name, version=version))
 551                         else:
 552                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 553                                             .format(path=filename, name=name, version=version))
 554                             found_vuln = True
 555                         break
 556             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 557                 if name in files_in_apk:
 558                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 559                                     .format(apkfilename=filename, name=name))
 560                     found_vuln = True
 561                 files_in_apk.add(name)
 562     return found_vuln
 563
 564
 565 def insert_obbs(repodir, apps, apks):
 566     """Scans the .obb files in a given repo directory and adds them to the
 567     relevant APK instances.  OBB files have versionCodes like APK
 568     files, and they are loosely associated.  If there is an OBB file
 569     present, then any APK with the same or higher versionCode will use
 570     that OBB file.  There are two OBB types: main and patch, each APK
 571     can only have only have one of each.
 572
 573     https://developer.android.com/google/play/expansion-files.html
 574
 575     :param repodir: repo directory to scan
 576     :param apps: list of current, valid apps
 577     :param apks: current information on all APKs
 578
 579     """
 580
 581     def obbWarnDelete(f, msg):
 582         logging.warning(msg + ' ' + f)
 583         if options.delete_unknown:
 584             logging.error(_("Deleting unknown file: {path}").format(path=f))
 585             os.remove(f)
 586
 587     obbs = []
 588     java_Integer_MIN_VALUE = -pow(2, 31)
 589     currentPackageNames = apps.keys()
 590     for f in glob.glob(os.path.join(repodir, '*.obb')):
 591         obbfile = os.path.basename(f)
 592         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 593         chunks = obbfile.split('.')
 594         if chunks[0] != 'main' and chunks[0] != 'patch':
 595             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 596             continue
 597         if not re.match(r'^-?[0-9]+$', chunks[1]):
 598             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 599                           .format(name=chunks[0]))
 600             continue
 601         versionCode = int(chunks[1])
 602         packagename = ".".join(chunks[2:-1])
 603
 604         highestVersionCode = java_Integer_MIN_VALUE
 605         if packagename not in currentPackageNames:
 606             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 607             continue
 608         for apk in apks:
 609             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 610                 highestVersionCode = apk['versionCode']
 611         if versionCode > highestVersionCode:
 612             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 613                           .format(integer=str(versionCode)))
 614             continue
 615         obbsha256 = sha256sum(f)
 616         obbs.append((packagename, versionCode, obbfile, obbsha256))
 617
 618     for apk in apks:
 619         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 620             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 621                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 622                     apk['obbMainFile'] = obbfile
 623                     apk['obbMainFileSha256'] = obbsha256
 624                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 625                     apk['obbPatchFile'] = obbfile
 626                     apk['obbPatchFileSha256'] = obbsha256
 627             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 628                 break
 629
 630
 631 def translate_per_build_anti_features(apps, apks):
 632     """Grab the anti-features list from the build metadata
 633
 634     For most Anti-Features, they are really most applicable per-APK,
 635     not for an app.  An app can fix a vulnerability, add/remove
 636     tracking, etc.  This reads the 'antifeatures' list from the Build
 637     entries in the fdroiddata metadata file, then transforms it into
 638     the 'antiFeatures' list of unique items for the index.
 639
 640     The field key is all lower case in the metadata file to match the
 641     rest of the Build fields.  It is 'antiFeatures' camel case in the
 642     implementation, index, and fdroidclient since it is translated
 643     from the build 'antifeatures' field, not directly included.
 644
 645     """
 646
 647     antiFeatures = dict()
 648     for packageName, app in apps.items():
 649         d = dict()
 650         for build in app['builds']:
 651             afl = build.get('antifeatures')
 652             if afl:
 653                 d[int(build.versionCode)] = afl
 654         if len(d) > 0:
 655             antiFeatures[packageName] = d
 656
 657     for apk in apks:
 658         d = antiFeatures.get(apk['packageName'])
 659         if d:
 660             afl = d.get(apk['versionCode'])
 661             if afl:
 662                 apk['antiFeatures'].update(afl)
 663
 664
 665 def _get_localized_dict(app, locale):
 666     '''get the dict to add localized store metadata to'''
 667     if 'localized' not in app:
 668         app['localized'] = collections.OrderedDict()
 669     if locale not in app['localized']:
 670         app['localized'][locale] = collections.OrderedDict()
 671     return app['localized'][locale]
 672
 673
 674 def _set_localized_text_entry(app, locale, key, f):
 675     limit = config['char_limits'][key]
 676     localized = _get_localized_dict(app, locale)
 677     with open(f) as fp:
 678         text = fp.read()[:limit]
 679         if len(text) > 0:
 680             localized[key] = text
 681
 682
 683 def _set_author_entry(app, key, f):
 684     limit = config['char_limits']['author']
 685     with open(f) as fp:
 686         text = fp.read()[:limit]
 687         if len(text) > 0:
 688             app[key] = text
 689
 690
 691 def _strip_and_copy_image(inpath, outpath):
 692     """Remove any metadata from image and copy it to new path
 693
 694     Sadly, image metadata like EXIF can be used to exploit devices.
 695     It is not used at all in the F-Droid ecosystem, so its much safer
 696     just to remove it entirely.
 697
 698     """
 699
 700     extension = common.get_extension(inpath)[1]
 701     if os.path.isdir(outpath):
 702         outpath = os.path.join(outpath, os.path.basename(inpath))
 703     if extension == 'png':
 704         with open(inpath, 'rb') as fp:
 705             in_image = Image.open(fp)
 706             in_image.save(outpath, "PNG", optimize=True,
 707                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 708     elif extension == 'jpg' or extension == 'jpeg':
 709         with open(inpath, 'rb') as fp:
 710             in_image = Image.open(fp)
 711             data = list(in_image.getdata())
 712             out_image = Image.new(in_image.mode, in_image.size)
 713         out_image.putdata(data)
 714         out_image.save(outpath, "JPEG", optimize=True)
 715     else:
 716         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 717                               .format(extension=extension))
 718
 719
 720 def copy_triple_t_store_metadata(apps):
 721     """Include store metadata from the app's source repo
 722
 723     The Triple-T Gradle Play Publisher is a plugin that has a standard
 724     file layout for all of the metadata and graphics that the Google
 725     Play Store accepts.  Since F-Droid has the git repo, it can just
 726     pluck those files directly.  This method reads any text files into
 727     the app dict, then copies any graphics into the fdroid repo
 728     directory structure.
 729
 730     This needs to be run before insert_localized_app_metadata() so that
 731     the graphics files that are copied into the fdroid repo get
 732     properly indexed.
 733
 734     https://github.com/Triple-T/gradle-play-publisher#upload-images
 735     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 736
 737     """
 738
 739     if not os.path.isdir('build'):
 740         return  # nothing to do
 741
 742     for packageName, app in apps.items():
 743         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 744             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 745             for root, dirs, files in os.walk(d):
 746                 segments = root.split('/')
 747                 locale = segments[-2]
 748                 for f in files:
 749                     if f == 'fulldescription':
 750                         _set_localized_text_entry(app, locale, 'description',
 751                                                   os.path.join(root, f))
 752                         continue
 753                     elif f == 'shortdescription':
 754                         _set_localized_text_entry(app, locale, 'summary',
 755                                                   os.path.join(root, f))
 756                         continue
 757                     elif f == 'title':
 758                         _set_localized_text_entry(app, locale, 'name',
 759                                                   os.path.join(root, f))
 760                         continue
 761                     elif f == 'video':
 762                         _set_localized_text_entry(app, locale, 'video',
 763                                                   os.path.join(root, f))
 764                         continue
 765                     elif f == 'whatsnew':
 766                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 767                                                   os.path.join(root, f))
 768                         continue
 769                     elif f == 'contactEmail':
 770                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 771                         continue
 772                     elif f == 'contactPhone':
 773                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 774                         continue
 775                     elif f == 'contactWebsite':
 776                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 777                         continue
 778
 779                     base, extension = common.get_extension(f)
 780                     dirname = os.path.basename(root)
 781                     if extension in ALLOWED_EXTENSIONS \
 782                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 783                         if segments[-2] == 'listing':
 784                             locale = segments[-3]
 785                         else:
 786                             locale = segments[-2]
 787                         destdir = os.path.join('repo', packageName, locale, dirname)
 788                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 789                         sourcefile = os.path.join(root, f)
 790                         destfile = os.path.join(destdir, os.path.basename(f))
 791                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 792                         _strip_and_copy_image(sourcefile, destfile)
 793
 794
 795 def insert_localized_app_metadata(apps):
 796     """scans standard locations for graphics and localized text
 797
 798     Scans for localized description files, store graphics, and
 799     screenshot PNG files in statically defined screenshots directory
 800     and adds them to the app metadata.  The screenshots and graphic
 801     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 802     and must be in the following layout:
 803     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 804
 805     repo/packageName/locale/featureGraphic.png
 806     repo/packageName/locale/phoneScreenshots/1.png
 807     repo/packageName/locale/phoneScreenshots/2.png
 808
 809     The changelog files must be text files named with the versionCode
 810     ending with ".txt" and must be in the following layout:
 811     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 812
 813     repo/packageName/locale/changelogs/12345.txt
 814
 815     This will scan the each app's source repo then the metadata/ dir
 816     for these standard locations of changelog files.  If it finds
 817     them, they will be added to the dict of all packages, with the
 818     versions in the metadata/ folder taking precendence over the what
 819     is in the app's source repo.
 820
 821     Where "packageName" is the app's packageName and "locale" is the locale
 822     of the graphics, e.g. what language they are in, using the IETF RFC5646
 823     format (en-US, fr-CA, es-MX, etc).
 824
 825     This will also scan the app's git for a fastlane folder, and the
 826     metadata/ folder and the apps' source repos for standard locations
 827     of graphic and screenshot files.  If it finds them, it will copy
 828     them into the repo.  The fastlane files follow this pattern:
 829     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 830
 831     """
 832
 833     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 834     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 837
 838     for srcd in sorted(sourcedirs):
 839         if not os.path.isdir(srcd):
 840             continue
 841         for root, dirs, files in os.walk(srcd):
 842             segments = root.split('/')
 843             packageName = segments[1]
 844             if packageName not in apps:
 845                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 846                 continue
 847             locale = segments[-1]
 848             destdir = os.path.join('repo', packageName, locale)
 849
 850             # flavours specified in build receipt
 851             build_flavours = ""
 852             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 853                     and 'gradle' in apps[packageName].builds[-1]:
 854                 build_flavours = apps[packageName].builds[-1].gradle
 855
 856             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 857                 logging.debug("ignoring due to wrong flavour")
 858                 continue
 859
 860             for f in files:
 861                 if f in ('description.txt', 'full_description.txt'):
 862                     _set_localized_text_entry(apps[packageName], locale, 'description',
 863                                               os.path.join(root, f))
 864                     continue
 865                 elif f in ('summary.txt', 'short_description.txt'):
 866                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 867                                               os.path.join(root, f))
 868                     continue
 869                 elif f in ('name.txt', 'title.txt'):
 870                     _set_localized_text_entry(apps[packageName], locale, 'name',
 871                                               os.path.join(root, f))
 872                     continue
 873                 elif f == 'video.txt':
 874                     _set_localized_text_entry(apps[packageName], locale, 'video',
 875                                               os.path.join(root, f))
 876                     continue
 877                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 878                     locale = segments[-2]
 879                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 880                                               os.path.join(root, f))
 881                     continue
 882
 883                 base, extension = common.get_extension(f)
 884                 if locale == 'images':
 885                     locale = segments[-2]
 886                     destdir = os.path.join('repo', packageName, locale)
 887                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 888                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 889                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 890                     _strip_and_copy_image(os.path.join(root, f), destdir)
 891             for d in dirs:
 892                 if d in SCREENSHOT_DIRS:
 893                     if locale == 'images':
 894                         locale = segments[-2]
 895                         destdir = os.path.join('repo', packageName, locale)
 896                     for f in glob.glob(os.path.join(root, d, '*.*')):
 897                         _ignored, extension = common.get_extension(f)
 898                         if extension in ALLOWED_EXTENSIONS:
 899                             screenshotdestdir = os.path.join(destdir, d)
 900                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 901                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 902                             _strip_and_copy_image(f, screenshotdestdir)
 903
 904     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 905     for d in repofiles:
 906         if not os.path.isdir(d):
 907             continue
 908         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 909             if not os.path.isfile(f):
 910                 continue
 911             segments = f.split('/')
 912             packageName = segments[1]
 913             locale = segments[2]
 914             screenshotdir = segments[3]
 915             filename = os.path.basename(f)
 916             base, extension = common.get_extension(filename)
 917
 918             if packageName not in apps:
 919                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 920                                 .format(path=filename, name=packageName))
 921                 continue
 922             graphics = _get_localized_dict(apps[packageName], locale)
 923
 924             if extension not in ALLOWED_EXTENSIONS:
 925                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 926             elif base in GRAPHIC_NAMES:
 927                 # there can only be zero or one of these per locale
 928                 graphics[base] = filename
 929             elif screenshotdir in SCREENSHOT_DIRS:
 930                 # there can any number of these per locale
 931                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 932                 if screenshotdir not in graphics:
 933                     graphics[screenshotdir] = []
 934                 graphics[screenshotdir].append(filename)
 935             else:
 936                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 937
 938
 939 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 940     """Scan a repo for all files with an extension except APK/OBB
 941
 942     :param apkcache: current cached info about all repo files
 943     :param repodir: repo directory to scan
 944     :param knownapks: list of all known files, as per metadata.read_metadata
 945     :param use_date_from_file: use date from file (instead of current date)
 946                                for newly added files
 947     """
 948
 949     cachechanged = False
 950     repo_files = []
 951     repodir = repodir.encode('utf-8')
 952     for name in os.listdir(repodir):
 953         file_extension = common.get_file_extension(name)
 954         if file_extension == 'apk' or file_extension == 'obb':
 955             continue
 956         filename = os.path.join(repodir, name)
 957         name_utf8 = name.decode('utf-8')
 958         if filename.endswith(b'_src.tar.gz'):
 959             logging.debug(_('skipping source tarball: {path}')
 960                           .format(path=filename.decode('utf-8')))
 961             continue
 962         if not common.is_repo_file(filename):
 963             continue
 964         stat = os.stat(filename)
 965         if stat.st_size == 0:
 966             raise FDroidException(_('{path} is zero size!')
 967                                   .format(path=filename))
 968
 969         shasum = sha256sum(filename)
 970         usecache = False
 971         if name in apkcache:
 972             repo_file = apkcache[name]
 973             # added time is cached as tuple but used here as datetime instance
 974             if 'added' in repo_file:
 975                 a = repo_file['added']
 976                 if isinstance(a, datetime):
 977                     repo_file['added'] = a
 978                 else:
 979                     repo_file['added'] = datetime(*a[:6])
 980             if repo_file.get('hash') == shasum:
 981                 logging.debug(_("Reading {apkfilename} from cache")
 982                               .format(apkfilename=name_utf8))
 983                 usecache = True
 984             else:
 985                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 986                               .format(apkfilename=name_utf8))
 987
 988         if not usecache:
 989             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 990             repo_file = collections.OrderedDict()
 991             repo_file['name'] = os.path.splitext(name_utf8)[0]
 992             # TODO rename apkname globally to something more generic
 993             repo_file['apkName'] = name_utf8
 994             repo_file['hash'] = shasum
 995             repo_file['hashType'] = 'sha256'
 996             repo_file['versionCode'] = 0
 997             repo_file['versionName'] = shasum
 998             # the static ID is the SHA256 unless it is set in the metadata
 999             repo_file['packageName'] = shasum
1000
1001             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1002             if m:
1003                 repo_file['packageName'] = m.group(1)
1004                 repo_file['versionCode'] = int(m.group(2))
1005             srcfilename = name + b'_src.tar.gz'
1006             if os.path.exists(os.path.join(repodir, srcfilename)):
1007                 repo_file['srcname'] = srcfilename.decode('utf-8')
1008             repo_file['size'] = stat.st_size
1009
1010             apkcache[name] = repo_file
1011             cachechanged = True
1012
1013         if use_date_from_file:
1014             timestamp = stat.st_ctime
1015             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1016         else:
1017             default_date_param = None
1018
1019         # Record in knownapks, getting the added date at the same time..
1020         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1021                                     default_date=default_date_param)
1022         if added:
1023             repo_file['added'] = added
1024
1025         repo_files.append(repo_file)
1026
1027     return repo_files, cachechanged
1028
1029
1030 def scan_apk(apk_file):
1031     """
1032     Scans an APK file and returns dictionary with metadata of the APK.
1033
1034     Attention: This does *not* verify that the APK signature is correct.
1035
1036     :param apk_file: The (ideally absolute) path to the APK file
1037     :raises BuildException
1038     :return A dict containing APK metadata
1039     """
1040     apk = {
1041         'hash': sha256sum(apk_file),
1042         'hashType': 'sha256',
1043         'uses-permission': [],
1044         'uses-permission-sdk-23': [],
1045         'features': [],
1046         'icons_src': {},
1047         'icons': {},
1048         'antiFeatures': set(),
1049     }
1050
1051     if SdkToolsPopen(['aapt', 'version'], output=False):
1052         scan_apk_aapt(apk, apk_file)
1053     else:
1054         scan_apk_androguard(apk, apk_file)
1055
1056     # Get the signature, or rather the signing key fingerprints
1057     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1058     apk['sig'] = getsig(apk_file)
1059     if not apk['sig']:
1060         raise BuildException("Failed to get apk signature")
1061     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1062                                                                apk_file))
1063     if not apk.get('signer'):
1064         raise BuildException("Failed to get apk signing key fingerprint")
1065
1066     # Get size of the APK
1067     apk['size'] = os.path.getsize(apk_file)
1068
1069     if 'minSdkVersion' not in apk:
1070         logging.warning("No SDK version information found in {0}".format(apk_file))
1071         apk['minSdkVersion'] = 1
1072
1073     # Check for known vulnerabilities
1074     if has_known_vulnerability(apk_file):
1075         apk['antiFeatures'].add('KnownVuln')
1076
1077     return apk
1078
1079
1080 def scan_apk_aapt(apk, apkfile):
1081     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1082     if p.returncode != 0:
1083         if options.delete_unknown:
1084             if os.path.exists(apkfile):
1085                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1086                 os.remove(apkfile)
1087             else:
1088                 logging.error("Could not find {0} to remove it".format(apkfile))
1089         else:
1090             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1091         raise BuildException(_("Invalid APK"))
1092     for line in p.output.splitlines():
1093         if line.startswith("package:"):
1094             try:
1095                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1096                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1097                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1098             except Exception as e:
1099                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1100         elif line.startswith("application:"):
1101             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1102             # Keep path to non-dpi icon in case we need it
1103             match = re.match(APK_ICON_PAT_NODPI, line)
1104             if match:
1105                 apk['icons_src']['-1'] = match.group(1)
1106         elif line.startswith("launchable-activity:"):
1107             # Only use launchable-activity as fallback to application
1108             if not apk['name']:
1109                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1110             if '-1' not in apk['icons_src']:
1111                 match = re.match(APK_ICON_PAT_NODPI, line)
1112                 if match:
1113                     apk['icons_src']['-1'] = match.group(1)
1114         elif line.startswith("application-icon-"):
1115             match = re.match(APK_ICON_PAT, line)
1116             if match:
1117                 density = match.group(1)
1118                 path = match.group(2)
1119                 apk['icons_src'][density] = path
1120         elif line.startswith("sdkVersion:"):
1121             m = re.match(APK_SDK_VERSION_PAT, line)
1122             if m is None:
1123                 logging.error(line.replace('sdkVersion:', '')
1124                               + ' is not a valid minSdkVersion!')
1125             else:
1126                 apk['minSdkVersion'] = m.group(1)
1127                 # if target not set, default to min
1128                 if 'targetSdkVersion' not in apk:
1129                     apk['targetSdkVersion'] = m.group(1)
1130         elif line.startswith("targetSdkVersion:"):
1131             m = re.match(APK_SDK_VERSION_PAT, line)
1132             if m is None:
1133                 logging.error(line.replace('targetSdkVersion:', '')
1134                               + ' is not a valid targetSdkVersion!')
1135             else:
1136                 apk['targetSdkVersion'] = m.group(1)
1137         elif line.startswith("maxSdkVersion:"):
1138             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1139         elif line.startswith("native-code:"):
1140             apk['nativecode'] = []
1141             for arch in line[13:].split(' '):
1142                 apk['nativecode'].append(arch[1:-1])
1143         elif line.startswith('uses-permission:'):
1144             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1145             if perm_match['maxSdkVersion']:
1146                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1147             permission = UsesPermission(
1148                 perm_match['name'],
1149                 perm_match['maxSdkVersion']
1150             )
1151
1152             apk['uses-permission'].append(permission)
1153         elif line.startswith('uses-permission-sdk-23:'):
1154             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1155             if perm_match['maxSdkVersion']:
1156                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1157             permission_sdk_23 = UsesPermissionSdk23(
1158                 perm_match['name'],
1159                 perm_match['maxSdkVersion']
1160             )
1161
1162             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1163
1164         elif line.startswith('uses-feature:'):
1165             feature = re.match(APK_FEATURE_PAT, line).group(1)
1166             # Filter out this, it's only added with the latest SDK tools and
1167             # causes problems for lots of apps.
1168             if feature != "android.hardware.screen.portrait" \
1169                     and feature != "android.hardware.screen.landscape":
1170                 if feature.startswith("android.feature."):
1171                     feature = feature[16:]
1172                 apk['features'].add(feature)
1173
1174
1175 def scan_apk_androguard(apk, apkfile):
1176     try:
1177         from androguard.core.bytecodes.apk import APK
1178         apkobject = APK(apkfile)
1179         if apkobject.is_valid_APK():
1180             arsc = apkobject.get_android_resources()
1181         else:
1182             if options.delete_unknown:
1183                 if os.path.exists(apkfile):
1184                     logging.error(_("Failed to get apk information, deleting {path}")
1185                                   .format(path=apkfile))
1186                     os.remove(apkfile)
1187                 else:
1188                     logging.error(_("Could not find {path} to remove it")
1189                                   .format(path=apkfile))
1190             else:
1191                 logging.error(_("Failed to get apk information, skipping {path}")
1192                               .format(path=apkfile))
1193             raise BuildException(_("Invalid APK"))
1194     except ImportError:
1195         raise FDroidException("androguard library is not installed and aapt not present")
1196     except FileNotFoundError:
1197         logging.error(_("Could not open apk file for analysis"))
1198         raise BuildException(_("Invalid APK"))
1199
1200     apk['packageName'] = apkobject.get_package()
1201     apk['versionCode'] = int(apkobject.get_androidversion_code())
1202     apk['versionName'] = apkobject.get_androidversion_name()
1203     if apk['versionName'][0] == "@":
1204         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1205         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1206         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1207     apk['name'] = apkobject.get_app_name()
1208
1209     if apkobject.get_max_sdk_version() is not None:
1210         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1211     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1212     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1213
1214     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1215     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1216
1217     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1218
1219     for file in apkobject.get_files():
1220         d_re = density_re.match(file)
1221         if d_re:
1222             folder = d_re.group(1).split('-')
1223             if len(folder) > 1:
1224                 resolution = folder[1]
1225             else:
1226                 resolution = 'mdpi'
1227             density = screen_resolutions[resolution]
1228             apk['icons_src'][density] = d_re.group(0)
1229
1230     if apk['icons_src'].get('-1') is None:
1231         apk['icons_src']['-1'] = apk['icons_src']['160']
1232
1233     arch_re = re.compile("^lib/(.*)/.*$")
1234     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1235     if len(arch) >= 1:
1236         apk['nativecode'] = []
1237         apk['nativecode'].extend(sorted(list(arch)))
1238
1239     xml = apkobject.get_android_manifest_xml()
1240
1241     for item in xml.getElementsByTagName('uses-permission'):
1242         name = str(item.getAttribute("android:name"))
1243         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1244         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1245         permission = UsesPermission(
1246             name,
1247             maxSdkVersion
1248         )
1249         apk['uses-permission'].append(permission)
1250
1251     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1252         name = str(item.getAttribute("android:name"))
1253         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1254         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1255         permission_sdk_23 = UsesPermissionSdk23(
1256             name,
1257             maxSdkVersion
1258         )
1259         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1260
1261     for item in xml.getElementsByTagName('uses-feature'):
1262         feature = str(item.getAttribute("android:name"))
1263         if feature != "android.hardware.screen.portrait" \
1264                 and feature != "android.hardware.screen.landscape":
1265             if feature.startswith("android.feature."):
1266                 feature = feature[16:]
1267         apk['features'].append(feature)
1268
1269
1270 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1271                 allow_disabled_algorithms=False, archive_bad_sig=False):
1272     """Processes the apk with the given filename in the given repo directory.
1273
1274     This also extracts the icons.
1275
1276     :param apkcache: current apk cache information
1277     :param apkfilename: the filename of the apk to scan
1278     :param repodir: repo directory to scan
1279     :param knownapks: known apks info
1280     :param use_date_from_apk: use date from APK (instead of current date)
1281                               for newly added APKs
1282     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1283                                       disabled algorithms in the signature (e.g. MD5)
1284     :param archive_bad_sig: move APKs with a bad signature to the archive
1285     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1286      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1287     """
1288
1289     apk = {}
1290     apkfile = os.path.join(repodir, apkfilename)
1291
1292     cachechanged = False
1293     usecache = False
1294     if apkfilename in apkcache:
1295         apk = apkcache[apkfilename]
1296         if apk.get('hash') == sha256sum(apkfile):
1297             logging.debug(_("Reading {apkfilename} from cache")
1298                           .format(apkfilename=apkfilename))
1299             usecache = True
1300         else:
1301             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1302                           .format(apkfilename=apkfilename))
1303
1304     if not usecache:
1305         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1306
1307         try:
1308             apk = scan_apk(apkfile)
1309         except BuildException:
1310             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1311                             .format(apkfilename=apkfilename))
1312             return True, None, False
1313
1314         # Check for debuggable apks...
1315         if common.isApkAndDebuggable(apkfile):
1316             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1317
1318         if options.rename_apks:
1319             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1320             std_short_name = os.path.join(repodir, n)
1321             if apkfile != std_short_name:
1322                 if os.path.exists(std_short_name):
1323                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1324                     if apkfile != std_long_name:
1325                         if os.path.exists(std_long_name):
1326                             dupdir = os.path.join('duplicates', repodir)
1327                             if not os.path.isdir(dupdir):
1328                                 os.makedirs(dupdir, exist_ok=True)
1329                             dupfile = os.path.join('duplicates', std_long_name)
1330                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1331                             os.rename(apkfile, dupfile)
1332                             return True, None, False
1333                         else:
1334                             os.rename(apkfile, std_long_name)
1335                     apkfile = std_long_name
1336                 else:
1337                     os.rename(apkfile, std_short_name)
1338                     apkfile = std_short_name
1339                 apkfilename = apkfile[len(repodir) + 1:]
1340
1341         apk['apkName'] = apkfilename
1342         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1343         if os.path.exists(os.path.join(repodir, srcfilename)):
1344             apk['srcname'] = srcfilename
1345
1346         # verify the jar signature is correct, allow deprecated
1347         # algorithms only if the APK is in the archive.
1348         skipapk = False
1349         if not common.verify_apk_signature(apkfile):
1350             if repodir == 'archive' or allow_disabled_algorithms:
1351                 if common.verify_old_apk_signature(apkfile):
1352                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1353                 else:
1354                     skipapk = True
1355             else:
1356                 skipapk = True
1357
1358         if skipapk:
1359             if archive_bad_sig:
1360                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1361                                 .format(apkfilename=apkfilename))
1362                 move_apk_between_sections(repodir, 'archive', apk)
1363             else:
1364                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1365                                 .format(apkfilename=apkfilename))
1366             return True, None, False
1367
1368         apkzip = zipfile.ZipFile(apkfile, 'r')
1369
1370         manifest = apkzip.getinfo('AndroidManifest.xml')
1371         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1372         if (1980, 0, 0) != manifest.date_time[0:3]:
1373             try:
1374                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1375             except ValueError as e:
1376                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1377                                 .format(apkfilename=apkfile) + str(e))
1378
1379         # extract icons from APK zip file
1380         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1381         try:
1382             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1383         finally:
1384             apkzip.close()  # ensure that APK zip file gets closed
1385
1386         # resize existing icons for densities missing in the APK
1387         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1388
1389         if use_date_from_apk and manifest.date_time[1] != 0:
1390             default_date_param = datetime(*manifest.date_time)
1391         else:
1392             default_date_param = None
1393
1394         # Record in known apks, getting the added date at the same time..
1395         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1396                                     default_date=default_date_param)
1397         if added:
1398             apk['added'] = added
1399
1400         apkcache[apkfilename] = apk
1401         cachechanged = True
1402
1403     return False, apk, cachechanged
1404
1405
1406 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1407     """Processes the apks in the given repo directory.
1408
1409     This also extracts the icons.
1410
1411     :param apkcache: current apk cache information
1412     :param repodir: repo directory to scan
1413     :param knownapks: known apks info
1414     :param use_date_from_apk: use date from APK (instead of current date)
1415                               for newly added APKs
1416     :returns: (apks, cachechanged) where apks is a list of apk information,
1417               and cachechanged is True if the apkcache got changed.
1418     """
1419
1420     cachechanged = False
1421
1422     for icon_dir in get_all_icon_dirs(repodir):
1423         if os.path.exists(icon_dir):
1424             if options.clean:
1425                 shutil.rmtree(icon_dir)
1426                 os.makedirs(icon_dir)
1427         else:
1428             os.makedirs(icon_dir)
1429
1430     apks = []
1431     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1432         apkfilename = apkfile[len(repodir) + 1:]
1433         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1434         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1435                                              use_date_from_apk, ada, True)
1436         if skip:
1437             continue
1438         apks.append(apk)
1439         cachechanged = cachechanged or cachethis
1440
1441     return apks, cachechanged
1442
1443
1444 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1445     """
1446     Extracts icons from the given APK zip in various densities,
1447     saves them into given repo directory
1448     and stores their names in the APK metadata dictionary.
1449
1450     :param icon_filename: A string representing the icon's file name
1451     :param apk: A populated dictionary containing APK metadata.
1452                 Needs to have 'icons_src' key
1453     :param apkzip: An opened zipfile.ZipFile of the APK file
1454     :param repo_dir: The directory of the APK's repository
1455     :return: A list of icon densities that are missing
1456     """
1457     empty_densities = []
1458     for density in screen_densities:
1459         if density not in apk['icons_src']:
1460             empty_densities.append(density)
1461             continue
1462         icon_src = apk['icons_src'][density]
1463         icon_dir = get_icon_dir(repo_dir, density)
1464         icon_dest = os.path.join(icon_dir, icon_filename)
1465
1466         # Extract the icon files per density
1467         if icon_src.endswith('.xml'):
1468             png = os.path.basename(icon_src)[:-4] + '.png'
1469             for f in apkzip.namelist():
1470                 if f.endswith(png):
1471                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1472                     if m and screen_resolutions[m.group(2)] == density:
1473                         icon_src = f
1474             if icon_src.endswith('.xml'):
1475                 empty_densities.append(density)
1476                 continue
1477         try:
1478             with open(icon_dest, 'wb') as f:
1479                 f.write(get_icon_bytes(apkzip, icon_src))
1480             apk['icons'][density] = icon_filename
1481         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1482             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1483             del apk['icons_src'][density]
1484             empty_densities.append(density)
1485
1486     if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
1487         icon_src = apk['icons_src']['-1']
1488         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1489         with open(icon_path, 'wb') as f:
1490             f.write(get_icon_bytes(apkzip, icon_src))
1491         im = None
1492         try:
1493             im = Image.open(icon_path)
1494             dpi = px_to_dpi(im.size[0])
1495             for density in screen_densities:
1496                 if density in apk['icons']:
1497                     break
1498                 if density == screen_densities[-1] or dpi >= int(density):
1499                     apk['icons'][density] = icon_filename
1500                     shutil.move(icon_path,
1501                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1502                     empty_densities.remove(density)
1503                     break
1504         except Exception as e:
1505             logging.warning(_("Failed reading {path}: {error}")
1506                             .format(path=icon_path, error=e))
1507         finally:
1508             if im and hasattr(im, 'close'):
1509                 im.close()
1510
1511     if apk['icons']:
1512         apk['icon'] = icon_filename
1513
1514     return empty_densities
1515
1516
1517 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1518     """
1519     Resize existing icons for densities missing in the APK to ensure all densities are available
1520
1521     :param empty_densities: A list of icon densities that are missing
1522     :param icon_filename: A string representing the icon's file name
1523     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1524     :param repo_dir: The directory of the APK's repository
1525     """
1526     # First try resizing down to not lose quality
1527     last_density = None
1528     for density in screen_densities:
1529         if density not in empty_densities:
1530             last_density = density
1531             continue
1532         if last_density is None:
1533             continue
1534         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1535
1536         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1537         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1538         fp = None
1539         try:
1540             fp = open(last_icon_path, 'rb')
1541             im = Image.open(fp)
1542
1543             size = dpi_to_px(density)
1544
1545             im.thumbnail((size, size), Image.ANTIALIAS)
1546             im.save(icon_path, "PNG", optimize=True,
1547                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1548             empty_densities.remove(density)
1549         except Exception as e:
1550             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1551         finally:
1552             if fp:
1553                 fp.close()
1554
1555     # Then just copy from the highest resolution available
1556     last_density = None
1557     for density in reversed(screen_densities):
1558         if density not in empty_densities:
1559             last_density = density
1560             continue
1561
1562         if last_density is None:
1563             continue
1564
1565         shutil.copyfile(
1566             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1567             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1568         )
1569         empty_densities.remove(density)
1570
1571     for density in screen_densities:
1572         icon_dir = get_icon_dir(repo_dir, density)
1573         icon_dest = os.path.join(icon_dir, icon_filename)
1574         resize_icon(icon_dest, density)
1575
1576     # Copy from icons-mdpi to icons since mdpi is the baseline density
1577     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1578     if os.path.isfile(baseline):
1579         apk['icons']['0'] = icon_filename
1580         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1581
1582
1583 def apply_info_from_latest_apk(apps, apks):
1584     """
1585     Some information from the apks needs to be applied up to the application level.
1586     When doing this, we use the info from the most recent version's apk.
1587     We deal with figuring out when the app was added and last updated at the same time.
1588     """
1589     for appid, app in apps.items():
1590         bestver = UNSET_VERSION_CODE
1591         for apk in apks:
1592             if apk['packageName'] == appid:
1593                 if apk['versionCode'] > bestver:
1594                     bestver = apk['versionCode']
1595                     bestapk = apk
1596
1597                 if 'added' in apk:
1598                     if not app.added or apk['added'] < app.added:
1599                         app.added = apk['added']
1600                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1601                         app.lastUpdated = apk['added']
1602
1603         if not app.added:
1604             logging.debug("Don't know when " + appid + " was added")
1605         if not app.lastUpdated:
1606             logging.debug("Don't know when " + appid + " was last updated")
1607
1608         if bestver == UNSET_VERSION_CODE:
1609
1610             if app.Name is None:
1611                 app.Name = app.AutoName or appid
1612             app.icon = None
1613             logging.debug("Application " + appid + " has no packages")
1614         else:
1615             if app.Name is None:
1616                 app.Name = bestapk['name']
1617             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1618             if app.CurrentVersionCode is None:
1619                 app.CurrentVersionCode = str(bestver)
1620
1621
1622 def make_categories_txt(repodir, categories):
1623     '''Write a category list in the repo to allow quick access'''
1624     catdata = ''
1625     for cat in sorted(categories):
1626         catdata += cat + '\n'
1627     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1628         f.write(catdata)
1629
1630
1631 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1632
1633     def filter_apk_list_sorted(apk_list):
1634         res = []
1635         for apk in apk_list:
1636             if apk['packageName'] == appid:
1637                 res.append(apk)
1638
1639         # Sort the apk list by version code. First is highest/newest.
1640         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1641
1642     for appid, app in apps.items():
1643
1644         if app.ArchivePolicy:
1645             keepversions = int(app.ArchivePolicy[:-9])
1646         else:
1647             keepversions = defaultkeepversions
1648
1649         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1650                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1651
1652         current_app_apks = filter_apk_list_sorted(apks)
1653         if len(current_app_apks) > keepversions:
1654             # Move back the ones we don't want.
1655             for apk in current_app_apks[keepversions:]:
1656                 move_apk_between_sections(repodir, archivedir, apk)
1657                 archapks.append(apk)
1658                 apks.remove(apk)
1659
1660         current_app_archapks = filter_apk_list_sorted(archapks)
1661         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1662             kept = 0
1663             # Move forward the ones we want again, except DisableAlgorithm
1664             for apk in current_app_archapks:
1665                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1666                     move_apk_between_sections(archivedir, repodir, apk)
1667                     archapks.remove(apk)
1668                     apks.append(apk)
1669                     kept += 1
1670                 if kept == keepversions:
1671                     break
1672
1673
1674 def move_apk_between_sections(from_dir, to_dir, apk):
1675     """move an APK from repo to archive or vice versa"""
1676
1677     def _move_file(from_dir, to_dir, filename, ignore_missing):
1678         from_path = os.path.join(from_dir, filename)
1679         if ignore_missing and not os.path.exists(from_path):
1680             return
1681         to_path = os.path.join(to_dir, filename)
1682         if not os.path.exists(to_dir):
1683             os.mkdir(to_dir)
1684         shutil.move(from_path, to_path)
1685
1686     if from_dir == to_dir:
1687         return
1688
1689     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1690     _move_file(from_dir, to_dir, apk['apkName'], False)
1691     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1692     for density in all_screen_densities:
1693         from_icon_dir = get_icon_dir(from_dir, density)
1694         to_icon_dir = get_icon_dir(to_dir, density)
1695         if density not in apk.get('icons', []):
1696             continue
1697         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1698     if 'srcname' in apk:
1699         _move_file(from_dir, to_dir, apk['srcname'], False)
1700
1701
1702 def add_apks_to_per_app_repos(repodir, apks):
1703     apks_per_app = dict()
1704     for apk in apks:
1705         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1706         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1707         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1708         apks_per_app[apk['packageName']] = apk
1709
1710         if not os.path.exists(apk['per_app_icons']):
1711             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1712             os.makedirs(apk['per_app_icons'])
1713
1714         apkpath = os.path.join(repodir, apk['apkName'])
1715         shutil.copy(apkpath, apk['per_app_repo'])
1716         apksigpath = apkpath + '.sig'
1717         if os.path.exists(apksigpath):
1718             shutil.copy(apksigpath, apk['per_app_repo'])
1719         apkascpath = apkpath + '.asc'
1720         if os.path.exists(apkascpath):
1721             shutil.copy(apkascpath, apk['per_app_repo'])
1722
1723
1724 def create_metadata_from_template(apk):
1725     '''create a new metadata file using internal or external template
1726
1727     Generate warnings for apk's with no metadata (or create skeleton
1728     metadata files, if requested on the command line).  Though the
1729     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1730     since those impose things on the metadata file made from the
1731     template: field sort order, empty field value, formatting, etc.
1732     '''
1733
1734     import yaml
1735     if os.path.exists('template.yml'):
1736         with open('template.yml') as f:
1737             metatxt = f.read()
1738         if 'name' in apk and apk['name'] != '':
1739             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1740                              r'\1 ' + apk['name'],
1741                              metatxt,
1742                              flags=re.IGNORECASE | re.MULTILINE)
1743         else:
1744             logging.warning(_('{appid} does not have a name! Using package name instead.')
1745                             .format(appid=apk['packageName']))
1746             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1747                              r'\1 ' + apk['packageName'],
1748                              metatxt,
1749                              flags=re.IGNORECASE | re.MULTILINE)
1750         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1751             f.write(metatxt)
1752     else:
1753         app = dict()
1754         app['Categories'] = [os.path.basename(os.getcwd())]
1755         # include some blanks as part of the template
1756         app['AuthorName'] = ''
1757         app['Summary'] = ''
1758         app['WebSite'] = ''
1759         app['IssueTracker'] = ''
1760         app['SourceCode'] = ''
1761         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1762         if 'name' in apk and apk['name'] != '':
1763             app['Name'] = apk['name']
1764         else:
1765             logging.warning(_('{appid} does not have a name! Using package name instead.')
1766                             .format(appid=apk['packageName']))
1767             app['Name'] = apk['packageName']
1768         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1769             yaml.dump(app, f, default_flow_style=False)
1770     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1771
1772
1773 config = None
1774 options = None
1775 start_timestamp = time.gmtime()
1776
1777
1778 def main():
1779
1780     global config, options
1781
1782     # Parse command line...
1783     parser = ArgumentParser()
1784     common.setup_global_opts(parser)
1785     parser.add_argument("--create-key", action="store_true", default=False,
1786                         help=_("Add a repo signing key to an unsigned repo"))
1787     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1788                         help=_("Add skeleton metadata files for APKs that are missing them"))
1789     parser.add_argument("--delete-unknown", action="store_true", default=False,
1790                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1791     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1792                         help=_("Report on build data status"))
1793     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1794                         help=_("Interactively ask about things that need updating."))
1795     parser.add_argument("-I", "--icons", action="store_true", default=False,
1796                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1797     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1798                         help=_("Specify editor to use in interactive mode. Default " +
1799                                "is {path}").format(path='/etc/alternatives/editor'))
1800     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1801                         help=_("Update the wiki"))
1802     parser.add_argument("--pretty", action="store_true", default=False,
1803                         help=_("Produce human-readable XML/JSON for index files"))
1804     parser.add_argument("--clean", action="store_true", default=False,
1805                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1806     parser.add_argument("--nosign", action="store_true", default=False,
1807                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1808     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1809                         help=_("Use date from APK instead of current time for newly added APKs"))
1810     parser.add_argument("--rename-apks", action="store_true", default=False,
1811                         help=_("Rename APK files that do not match package.name_123.apk"))
1812     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1813                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1814     metadata.add_metadata_arguments(parser)
1815     options = parser.parse_args()
1816     metadata.warnings_action = options.W
1817
1818     config = common.read_config(options)
1819
1820     if not ('jarsigner' in config and 'keytool' in config):
1821         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1822
1823     repodirs = ['repo']
1824     if config['archive_older'] != 0:
1825         repodirs.append('archive')
1826         if not os.path.exists('archive'):
1827             os.mkdir('archive')
1828
1829     if options.icons:
1830         resize_all_icons(repodirs)
1831         sys.exit(0)
1832
1833     if options.rename_apks:
1834         options.clean = True
1835
1836     # check that icons exist now, rather than fail at the end of `fdroid update`
1837     for k in ['repo_icon', 'archive_icon']:
1838         if k in config:
1839             if not os.path.exists(config[k]):
1840                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1841                                  .format(name=k, path=config[k]))
1842                 sys.exit(1)
1843
1844     # if the user asks to create a keystore, do it now, reusing whatever it can
1845     if options.create_key:
1846         if os.path.exists(config['keystore']):
1847             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1848             logging.critical("\t'" + config['keystore'] + "'")
1849             sys.exit(1)
1850
1851         if 'repo_keyalias' not in config:
1852             config['repo_keyalias'] = socket.getfqdn()
1853             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1854         if 'keydname' not in config:
1855             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1856             common.write_to_config(config, 'keydname', config['keydname'])
1857         if 'keystore' not in config:
1858             config['keystore'] = common.default_config['keystore']
1859             common.write_to_config(config, 'keystore', config['keystore'])
1860
1861         password = common.genpassword()
1862         if 'keystorepass' not in config:
1863             config['keystorepass'] = password
1864             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1865         if 'keypass' not in config:
1866             config['keypass'] = password
1867             common.write_to_config(config, 'keypass', config['keypass'])
1868         common.genkeystore(config)
1869
1870     # Get all apps...
1871     apps = metadata.read_metadata()
1872
1873     # Generate a list of categories...
1874     categories = set()
1875     for app in apps.values():
1876         categories.update(app.Categories)
1877
1878     # Read known apks data (will be updated and written back when we've finished)
1879     knownapks = common.KnownApks()
1880
1881     # Get APK cache
1882     apkcache = get_cache()
1883
1884     # Delete builds for disabled apps
1885     delete_disabled_builds(apps, apkcache, repodirs)
1886
1887     # Scan all apks in the main repo
1888     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1889
1890     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1891                                            options.use_date_from_apk)
1892     cachechanged = cachechanged or fcachechanged
1893     apks += files
1894     for apk in apks:
1895         if apk['packageName'] not in apps:
1896             if options.create_metadata:
1897                 create_metadata_from_template(apk)
1898                 apps = metadata.read_metadata()
1899             else:
1900                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1901                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1902                 if options.delete_unknown:
1903                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1904                                  .format(apkfilename=apk['apkName']))
1905                     rmf = os.path.join(repodirs[0], apk['apkName'])
1906                     if not os.path.exists(rmf):
1907                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1908                     else:
1909                         os.remove(rmf)
1910                 else:
1911                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1912
1913     copy_triple_t_store_metadata(apps)
1914     insert_obbs(repodirs[0], apps, apks)
1915     insert_localized_app_metadata(apps)
1916     translate_per_build_anti_features(apps, apks)
1917
1918     # Scan the archive repo for apks as well
1919     if len(repodirs) > 1:
1920         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1921         if cc:
1922             cachechanged = True
1923     else:
1924         archapks = []
1925
1926     # Apply information from latest apks to the application and update dates
1927     apply_info_from_latest_apk(apps, apks + archapks)
1928
1929     # Sort the app list by name, then the web site doesn't have to by default.
1930     # (we had to wait until we'd scanned the apks to do this, because mostly the
1931     # name comes from there!)
1932     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1933
1934     # APKs are placed into multiple repos based on the app package, providing
1935     # per-app subscription feeds for nightly builds and things like it
1936     if config['per_app_repos']:
1937         add_apks_to_per_app_repos(repodirs[0], apks)
1938         for appid, app in apps.items():
1939             repodir = os.path.join(appid, 'fdroid', 'repo')
1940             appdict = dict()
1941             appdict[appid] = app
1942             if os.path.isdir(repodir):
1943                 index.make(appdict, [appid], apks, repodir, False)
1944             else:
1945                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1946         return
1947
1948     if len(repodirs) > 1:
1949         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1950
1951     # Make the index for the main repo...
1952     index.make(apps, sortedids, apks, repodirs[0], False)
1953     make_categories_txt(repodirs[0], categories)
1954
1955     # If there's an archive repo,  make the index for it. We already scanned it
1956     # earlier on.
1957     if len(repodirs) > 1:
1958         index.make(apps, sortedids, archapks, repodirs[1], True)
1959
1960     git_remote = config.get('binary_transparency_remote')
1961     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1962         from . import btlog
1963         btlog.make_binary_transparency_log(repodirs)
1964
1965     if config['update_stats']:
1966         # Update known apks info...
1967         knownapks.writeifchanged()
1968
1969         # Generate latest apps data for widget
1970         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1971             data = ''
1972             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1973                 for line in f:
1974                     appid = line.rstrip()
1975                     data += appid + "\t"
1976                     app = apps[appid]
1977                     data += app.Name + "\t"
1978                     if app.icon is not None:
1979                         data += app.icon + "\t"
1980                     data += app.License + "\n"
1981             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1982                 f.write(data)
1983
1984     if cachechanged:
1985         write_cache(apkcache)
1986
1987     # Update the wiki...
1988     if options.wiki:
1989         update_wiki(apps, sortedids, apks + archapks)
1990
1991     logging.info(_("Finished"))
1992
1993
1994 if __name__ == "__main__":
1995     main()