fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0':
 100         return os.path.join(repodir, "icons")
 101     return os.path.join(repodir, "icons-%s" % density)
 102
 103
 104 def get_icon_dirs(repodir):
 105     for density in screen_densities:
 106         yield get_icon_dir(repodir, density)
 107
 108
 109 def get_all_icon_dirs(repodir):
 110     for density in all_screen_densities:
 111         yield get_icon_dir(repodir, density)
 112
 113
 114 def update_wiki(apps, sortedids, apks):
 115     """Update the wiki
 116
 117     :param apps: fully populated list of all applications
 118     :param apks: all apks, except...
 119     """
 120     logging.info("Updating wiki")
 121     wikicat = 'Apps'
 122     wikiredircat = 'App Redirects'
 123     import mwclient
 124     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 125                          path=config['wiki_path'])
 126     site.login(config['wiki_user'], config['wiki_password'])
 127     generated_pages = {}
 128     generated_redirects = {}
 129
 130     for appid in sortedids:
 131         app = metadata.App(apps[appid])
 132
 133         wikidata = ''
 134         if app.Disabled:
 135             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 136         if app.AntiFeatures:
 137             for af in sorted(app.AntiFeatures):
 138                 wikidata += '{{AntiFeature|' + af + '}}\n'
 139         if app.RequiresRoot:
 140             requiresroot = 'Yes'
 141         else:
 142             requiresroot = 'No'
 143         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s|activity=%s}}\n' % (
 144             appid,
 145             app.Name,
 146             app.added.strftime('%Y-%m-%d') if app.added else '',
 147             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 148             app.SourceCode,
 149             app.IssueTracker,
 150             app.WebSite,
 151             app.Changelog,
 152             app.Donate,
 153             app.FlattrID,
 154             app.LiberapayID,
 155             app.Bitcoin,
 156             app.Litecoin,
 157             app.License,
 158             requiresroot,
 159             app.AuthorName,
 160             app.AuthorEmail,
 161             'https://gitlab.com/search?group_id=28397&scope=issues&search=' + appid,
 162         )
 163
 164         if app.Provides:
 165             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 166
 167         wikidata += app.Summary
 168         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 169
 170         wikidata += "=Description=\n"
 171         wikidata += metadata.description_wiki(app.Description) + "\n"
 172
 173         wikidata += "=Maintainer Notes=\n"
 174         if app.MaintainerNotes:
 175             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 176         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 177
 178         # Get a list of all packages for this application...
 179         apklist = []
 180         gotcurrentver = False
 181         cantupdate = False
 182         buildfails = False
 183         for apk in apks:
 184             if apk['packageName'] == appid:
 185                 if str(apk['versionCode']) == app.CurrentVersionCode:
 186                     gotcurrentver = True
 187                 apklist.append(apk)
 188         # Include ones we can't build, as a special case...
 189         for build in app.builds:
 190             if build.disable:
 191                 if build.versionCode == app.CurrentVersionCode:
 192                     cantupdate = True
 193                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 194                 apklist.append({'versionCode': int(build.versionCode),
 195                                 'versionName': build.versionName,
 196                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 197                                 })
 198             else:
 199                 builtit = False
 200                 for apk in apklist:
 201                     if apk['versionCode'] == int(build.versionCode):
 202                         builtit = True
 203                         break
 204                 if not builtit:
 205                     buildfails = True
 206                     apklist.append({'versionCode': int(build.versionCode),
 207                                     'versionName': build.versionName,
 208                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 209                                     })
 210         if app.CurrentVersionCode == '0':
 211             cantupdate = True
 212         # Sort with most recent first...
 213         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 214
 215         wikidata += "=Versions=\n"
 216         if len(apklist) == 0:
 217             wikidata += "We currently have no versions of this app available."
 218         elif not gotcurrentver:
 219             wikidata += "We don't have the current version of this app."
 220         else:
 221             wikidata += "We have the current version of this app."
 222         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 223         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 224         if len(app.NoSourceSince) > 0:
 225             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 226         if len(app.CurrentVersion) > 0:
 227             wikidata += "The current (recommended) version is " + app.CurrentVersion
 228             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 229         validapks = 0
 230         for apk in apklist:
 231             wikidata += "==" + apk['versionName'] + "==\n"
 232
 233             if 'buildproblem' in apk:
 234                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 235             else:
 236                 validapks += 1
 237                 wikidata += "This version is built and signed by "
 238                 if 'srcname' in apk:
 239                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 240                 else:
 241                     wikidata += "the original developer.\n\n"
 242             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 243
 244         wikidata += '\n[[Category:' + wikicat + ']]\n'
 245         if len(app.NoSourceSince) > 0:
 246             wikidata += '\n[[Category:Apps missing source code]]\n'
 247         if validapks == 0 and not app.Disabled:
 248             wikidata += '\n[[Category:Apps with no packages]]\n'
 249         if cantupdate and not app.Disabled:
 250             wikidata += "\n[[Category:Apps we cannot update]]\n"
 251         if buildfails and not app.Disabled:
 252             wikidata += "\n[[Category:Apps with failing builds]]\n"
 253         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 254             wikidata += '\n[[Category:Apps to Update]]\n'
 255         if app.Disabled:
 256             wikidata += '\n[[Category:Apps that are disabled]]\n'
 257         if app.UpdateCheckMode == 'None' and not app.Disabled:
 258             wikidata += '\n[[Category:Apps with no update check]]\n'
 259         for appcat in app.Categories:
 260             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 261
 262         # We can't have underscores in the page name, even if they're in
 263         # the package ID, because MediaWiki messes with them...
 264         pagename = appid.replace('_', ' ')
 265
 266         # Drop a trailing newline, because mediawiki is going to drop it anyway
 267         # and it we don't we'll think the page has changed when it hasn't...
 268         if wikidata.endswith('\n'):
 269             wikidata = wikidata[:-1]
 270
 271         generated_pages[pagename] = wikidata
 272
 273         # Make a redirect from the name to the ID too, unless there's
 274         # already an existing page with the name and it isn't a redirect.
 275         noclobber = False
 276         apppagename = app.Name.replace('_', ' ')
 277         apppagename = apppagename.replace('{', '')
 278         apppagename = apppagename.replace('}', ' ')
 279         apppagename = apppagename.replace(':', ' ')
 280         apppagename = apppagename.replace('[', ' ')
 281         apppagename = apppagename.replace(']', ' ')
 282         # Drop double spaces caused mostly by replacing ':' above
 283         apppagename = apppagename.replace('  ', ' ')
 284         for expagename in site.allpages(prefix=apppagename,
 285                                         filterredir='nonredirects',
 286                                         generator=False):
 287             if expagename == apppagename:
 288                 noclobber = True
 289         # Another reason not to make the redirect page is if the app name
 290         # is the same as it's ID, because that will overwrite the real page
 291         # with an redirect to itself! (Although it seems like an odd
 292         # scenario this happens a lot, e.g. where there is metadata but no
 293         # builds or binaries to extract a name from.
 294         if apppagename == pagename:
 295             noclobber = True
 296         if not noclobber:
 297             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 298
 299     for tcat, genp in [(wikicat, generated_pages),
 300                        (wikiredircat, generated_redirects)]:
 301         catpages = site.Pages['Category:' + tcat]
 302         existingpages = []
 303         for page in catpages:
 304             existingpages.append(page.name)
 305             if page.name in genp:
 306                 pagetxt = page.edit()
 307                 if pagetxt != genp[page.name]:
 308                     logging.debug("Updating modified page " + page.name)
 309                     page.save(genp[page.name], summary='Auto-updated')
 310                 else:
 311                     logging.debug("Page " + page.name + " is unchanged")
 312             else:
 313                 logging.warn("Deleting page " + page.name)
 314                 page.delete('No longer published')
 315         for pagename, text in genp.items():
 316             logging.debug("Checking " + pagename)
 317             if pagename not in existingpages:
 318                 logging.debug("Creating page " + pagename)
 319                 try:
 320                     newpage = site.Pages[pagename]
 321                     newpage.save(text, summary='Auto-created')
 322                 except Exception as e:
 323                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 324
 325     # Purge server cache to ensure counts are up to date
 326     site.Pages['Repository Maintenance'].purge()
 327
 328     # Write a page with the last build log for this version code
 329     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 330     newpage = site.Pages[wiki_page_path]
 331     txt = ''
 332     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 333     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 334     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 335     txt += "\n\n"
 336     txt += common.get_android_tools_version_log()
 337     newpage.save(txt, summary='Run log')
 338     newpage = site.Pages['update']
 339     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 340
 341
 342 def delete_disabled_builds(apps, apkcache, repodirs):
 343     """Delete disabled build outputs.
 344
 345     :param apps: list of all applications, as per metadata.read_metadata
 346     :param apkcache: current apk cache information
 347     :param repodirs: the repo directories to process
 348     """
 349     for appid, app in apps.items():
 350         for build in app['builds']:
 351             if not build.disable:
 352                 continue
 353             apkfilename = common.get_release_filename(app, build)
 354             iconfilename = "%s.%s.png" % (
 355                 appid,
 356                 build.versionCode)
 357             for repodir in repodirs:
 358                 files = [
 359                     os.path.join(repodir, apkfilename),
 360                     os.path.join(repodir, apkfilename + '.asc'),
 361                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 362                 ]
 363                 for density in all_screen_densities:
 364                     repo_dir = get_icon_dir(repodir, density)
 365                     files.append(os.path.join(repo_dir, iconfilename))
 366
 367                 for f in files:
 368                     if os.path.exists(f):
 369                         logging.info("Deleting disabled build output " + f)
 370                         os.remove(f)
 371             if apkfilename in apkcache:
 372                 del apkcache[apkfilename]
 373
 374
 375 def resize_icon(iconpath, density):
 376
 377     if not os.path.isfile(iconpath):
 378         return
 379
 380     fp = None
 381     try:
 382         fp = open(iconpath, 'rb')
 383         im = Image.open(fp)
 384         size = dpi_to_px(density)
 385
 386         if any(length > size for length in im.size):
 387             oldsize = im.size
 388             im.thumbnail((size, size), Image.ANTIALIAS)
 389             logging.debug("%s was too large at %s - new size is %s" % (
 390                 iconpath, oldsize, im.size))
 391             im.save(iconpath, "PNG", optimize=True,
 392                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 393
 394     except Exception as e:
 395         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 396
 397     finally:
 398         if fp:
 399             fp.close()
 400
 401
 402 def resize_all_icons(repodirs):
 403     """Resize all icons that exceed the max size
 404
 405     :param repodirs: the repo directories to process
 406     """
 407     for repodir in repodirs:
 408         for density in screen_densities:
 409             icon_dir = get_icon_dir(repodir, density)
 410             icon_glob = os.path.join(icon_dir, '*.png')
 411             for iconpath in glob.glob(icon_glob):
 412                 resize_icon(iconpath, density)
 413
 414
 415 def getsig(apkpath):
 416     """ Get the signing certificate of an apk. To get the same md5 has that
 417     Android gets, we encode the .RSA certificate in a specific format and pass
 418     it hex-encoded to the md5 digest algorithm.
 419
 420     :param apkpath: path to the apk
 421     :returns: A string containing the md5 of the signature of the apk or None
 422               if an error occurred.
 423     """
 424
 425     with zipfile.ZipFile(apkpath, 'r') as apk:
 426         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 427
 428         if len(certs) < 1:
 429             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 430             return None
 431         if len(certs) > 1:
 432             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 433             return None
 434
 435         cert = apk.read(certs[0])
 436
 437     cert_encoded = common.get_certificate(cert)
 438
 439     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 440
 441
 442 def get_cache_file():
 443     return os.path.join('tmp', 'apkcache')
 444
 445
 446 def get_cache():
 447     """Get the cached dict of the APK index
 448
 449     Gather information about all the apk files in the repo directory,
 450     using cached data if possible. Some of the index operations take a
 451     long time, like calculating the SHA-256 and verifying the APK
 452     signature.
 453
 454     The cache is invalidated if the metadata version is different, or
 455     the 'allow_disabled_algorithms' config/option is different.  In
 456     those cases, there is no easy way to know what has changed from
 457     the cache, so just rerun the whole thing.
 458
 459     :return: apkcache
 460
 461     """
 462     apkcachefile = get_cache_file()
 463     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 464     if not options.clean and os.path.exists(apkcachefile):
 465         with open(apkcachefile, 'rb') as cf:
 466             apkcache = pickle.load(cf, encoding='utf-8')
 467         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 468            or apkcache.get('allow_disabled_algorithms') != ada:
 469             apkcache = {}
 470     else:
 471         apkcache = {}
 472
 473     apkcache["METADATA_VERSION"] = METADATA_VERSION
 474     apkcache['allow_disabled_algorithms'] = ada
 475
 476     return apkcache
 477
 478
 479 def write_cache(apkcache):
 480     apkcachefile = get_cache_file()
 481     cache_path = os.path.dirname(apkcachefile)
 482     if not os.path.exists(cache_path):
 483         os.makedirs(cache_path)
 484     with open(apkcachefile, 'wb') as cf:
 485         pickle.dump(apkcache, cf)
 486
 487
 488 def get_icon_bytes(apkzip, iconsrc):
 489     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 490     try:
 491         return apkzip.read(iconsrc)
 492     except KeyError:
 493         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 494
 495
 496 def sha256sum(filename):
 497     '''Calculate the sha256 of the given file'''
 498     sha = hashlib.sha256()
 499     with open(filename, 'rb') as f:
 500         while True:
 501             t = f.read(16384)
 502             if len(t) == 0:
 503                 break
 504             sha.update(t)
 505     return sha.hexdigest()
 506
 507
 508 def has_known_vulnerability(filename):
 509     """checks for known vulnerabilities in the APK
 510
 511     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 512     version.  Google also enforces this:
 513     https://support.google.com/faqs/answer/6376725?hl=en
 514
 515     Checks whether there are more than one classes.dex or AndroidManifest.xml
 516     files, which is invalid and an essential part of the "Master Key" attack.
 517     http://www.saurik.com/id/17
 518
 519     Janus is similar to Master Key but is perhaps easier to scan for.
 520     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 521     """
 522
 523     found_vuln = False
 524
 525     # statically load this pattern
 526     if not hasattr(has_known_vulnerability, "pattern"):
 527         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 528
 529     with open(filename.encode(), 'rb') as fp:
 530         first4 = fp.read(4)
 531     if first4 != b'\x50\x4b\x03\x04':
 532         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 533                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 534                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 535
 536     files_in_apk = set()
 537     with zipfile.ZipFile(filename) as zf:
 538         for name in zf.namelist():
 539             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 540                 lib = zf.open(name)
 541                 while True:
 542                     chunk = lib.read(4096)
 543                     if chunk == b'':
 544                         break
 545                     m = has_known_vulnerability.pattern.search(chunk)
 546                     if m:
 547                         version = m.group(1).decode('ascii')
 548                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 549                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 550                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 551                             logging.debug(_('"{path}" contains recent {name} ({version})')
 552                                           .format(path=filename, name=name, version=version))
 553                         else:
 554                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 555                                             .format(path=filename, name=name, version=version))
 556                             found_vuln = True
 557                         break
 558             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 559                 if name in files_in_apk:
 560                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 561                                     .format(apkfilename=filename, name=name))
 562                     found_vuln = True
 563                 files_in_apk.add(name)
 564     return found_vuln
 565
 566
 567 def insert_obbs(repodir, apps, apks):
 568     """Scans the .obb files in a given repo directory and adds them to the
 569     relevant APK instances.  OBB files have versionCodes like APK
 570     files, and they are loosely associated.  If there is an OBB file
 571     present, then any APK with the same or higher versionCode will use
 572     that OBB file.  There are two OBB types: main and patch, each APK
 573     can only have only have one of each.
 574
 575     https://developer.android.com/google/play/expansion-files.html
 576
 577     :param repodir: repo directory to scan
 578     :param apps: list of current, valid apps
 579     :param apks: current information on all APKs
 580
 581     """
 582
 583     def obbWarnDelete(f, msg):
 584         logging.warning(msg + ' ' + f)
 585         if options.delete_unknown:
 586             logging.error(_("Deleting unknown file: {path}").format(path=f))
 587             os.remove(f)
 588
 589     obbs = []
 590     java_Integer_MIN_VALUE = -pow(2, 31)
 591     currentPackageNames = apps.keys()
 592     for f in glob.glob(os.path.join(repodir, '*.obb')):
 593         obbfile = os.path.basename(f)
 594         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 595         chunks = obbfile.split('.')
 596         if chunks[0] != 'main' and chunks[0] != 'patch':
 597             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 598             continue
 599         if not re.match(r'^-?[0-9]+$', chunks[1]):
 600             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 601                           .format(name=chunks[0]))
 602             continue
 603         versionCode = int(chunks[1])
 604         packagename = ".".join(chunks[2:-1])
 605
 606         highestVersionCode = java_Integer_MIN_VALUE
 607         if packagename not in currentPackageNames:
 608             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 609             continue
 610         for apk in apks:
 611             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 612                 highestVersionCode = apk['versionCode']
 613         if versionCode > highestVersionCode:
 614             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 615                           .format(integer=str(versionCode)))
 616             continue
 617         obbsha256 = sha256sum(f)
 618         obbs.append((packagename, versionCode, obbfile, obbsha256))
 619
 620     for apk in apks:
 621         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 622             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 623                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 624                     apk['obbMainFile'] = obbfile
 625                     apk['obbMainFileSha256'] = obbsha256
 626                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 627                     apk['obbPatchFile'] = obbfile
 628                     apk['obbPatchFileSha256'] = obbsha256
 629             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 630                 break
 631
 632
 633 def translate_per_build_anti_features(apps, apks):
 634     """Grab the anti-features list from the build metadata
 635
 636     For most Anti-Features, they are really most applicable per-APK,
 637     not for an app.  An app can fix a vulnerability, add/remove
 638     tracking, etc.  This reads the 'antifeatures' list from the Build
 639     entries in the fdroiddata metadata file, then transforms it into
 640     the 'antiFeatures' list of unique items for the index.
 641
 642     The field key is all lower case in the metadata file to match the
 643     rest of the Build fields.  It is 'antiFeatures' camel case in the
 644     implementation, index, and fdroidclient since it is translated
 645     from the build 'antifeatures' field, not directly included.
 646
 647     """
 648
 649     antiFeatures = dict()
 650     for packageName, app in apps.items():
 651         d = dict()
 652         for build in app['builds']:
 653             afl = build.get('antifeatures')
 654             if afl:
 655                 d[int(build.versionCode)] = afl
 656         if len(d) > 0:
 657             antiFeatures[packageName] = d
 658
 659     for apk in apks:
 660         d = antiFeatures.get(apk['packageName'])
 661         if d:
 662             afl = d.get(apk['versionCode'])
 663             if afl:
 664                 apk['antiFeatures'].update(afl)
 665
 666
 667 def _get_localized_dict(app, locale):
 668     '''get the dict to add localized store metadata to'''
 669     if 'localized' not in app:
 670         app['localized'] = collections.OrderedDict()
 671     if locale not in app['localized']:
 672         app['localized'][locale] = collections.OrderedDict()
 673     return app['localized'][locale]
 674
 675
 676 def _set_localized_text_entry(app, locale, key, f):
 677     limit = config['char_limits'][key]
 678     localized = _get_localized_dict(app, locale)
 679     with open(f) as fp:
 680         text = fp.read()[:limit]
 681         if len(text) > 0:
 682             localized[key] = text
 683
 684
 685 def _set_author_entry(app, key, f):
 686     limit = config['char_limits']['author']
 687     with open(f) as fp:
 688         text = fp.read()[:limit]
 689         if len(text) > 0:
 690             app[key] = text
 691
 692
 693 def _strip_and_copy_image(inpath, outpath):
 694     """Remove any metadata from image and copy it to new path
 695
 696     Sadly, image metadata like EXIF can be used to exploit devices.
 697     It is not used at all in the F-Droid ecosystem, so its much safer
 698     just to remove it entirely.
 699
 700     """
 701
 702     extension = common.get_extension(inpath)[1]
 703     if os.path.isdir(outpath):
 704         outpath = os.path.join(outpath, os.path.basename(inpath))
 705     if extension == 'png':
 706         with open(inpath, 'rb') as fp:
 707             in_image = Image.open(fp)
 708             in_image.save(outpath, "PNG", optimize=True,
 709                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 710     elif extension == 'jpg' or extension == 'jpeg':
 711         with open(inpath, 'rb') as fp:
 712             in_image = Image.open(fp)
 713             data = list(in_image.getdata())
 714             out_image = Image.new(in_image.mode, in_image.size)
 715         out_image.putdata(data)
 716         out_image.save(outpath, "JPEG", optimize=True)
 717     else:
 718         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 719                               .format(extension=extension))
 720
 721
 722 def copy_triple_t_store_metadata(apps):
 723     """Include store metadata from the app's source repo
 724
 725     The Triple-T Gradle Play Publisher is a plugin that has a standard
 726     file layout for all of the metadata and graphics that the Google
 727     Play Store accepts.  Since F-Droid has the git repo, it can just
 728     pluck those files directly.  This method reads any text files into
 729     the app dict, then copies any graphics into the fdroid repo
 730     directory structure.
 731
 732     This needs to be run before insert_localized_app_metadata() so that
 733     the graphics files that are copied into the fdroid repo get
 734     properly indexed.
 735
 736     https://github.com/Triple-T/gradle-play-publisher#upload-images
 737     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 738
 739     """
 740
 741     if not os.path.isdir('build'):
 742         return  # nothing to do
 743
 744     for packageName, app in apps.items():
 745         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 746             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 747             for root, dirs, files in os.walk(d):
 748                 segments = root.split('/')
 749                 locale = segments[-2]
 750                 for f in files:
 751                     if f == 'fulldescription':
 752                         _set_localized_text_entry(app, locale, 'description',
 753                                                   os.path.join(root, f))
 754                         continue
 755                     elif f == 'shortdescription':
 756                         _set_localized_text_entry(app, locale, 'summary',
 757                                                   os.path.join(root, f))
 758                         continue
 759                     elif f == 'title':
 760                         _set_localized_text_entry(app, locale, 'name',
 761                                                   os.path.join(root, f))
 762                         continue
 763                     elif f == 'video':
 764                         _set_localized_text_entry(app, locale, 'video',
 765                                                   os.path.join(root, f))
 766                         continue
 767                     elif f == 'whatsnew':
 768                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 769                                                   os.path.join(root, f))
 770                         continue
 771                     elif f == 'contactEmail':
 772                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 773                         continue
 774                     elif f == 'contactPhone':
 775                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 776                         continue
 777                     elif f == 'contactWebsite':
 778                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 779                         continue
 780
 781                     base, extension = common.get_extension(f)
 782                     dirname = os.path.basename(root)
 783                     if extension in ALLOWED_EXTENSIONS \
 784                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 785                         if segments[-2] == 'listing':
 786                             locale = segments[-3]
 787                         else:
 788                             locale = segments[-2]
 789                         destdir = os.path.join('repo', packageName, locale, dirname)
 790                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 791                         sourcefile = os.path.join(root, f)
 792                         destfile = os.path.join(destdir, os.path.basename(f))
 793                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 794                         _strip_and_copy_image(sourcefile, destfile)
 795
 796
 797 def insert_localized_app_metadata(apps):
 798     """scans standard locations for graphics and localized text
 799
 800     Scans for localized description files, store graphics, and
 801     screenshot PNG files in statically defined screenshots directory
 802     and adds them to the app metadata.  The screenshots and graphic
 803     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 804     and must be in the following layout:
 805     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 806
 807     repo/packageName/locale/featureGraphic.png
 808     repo/packageName/locale/phoneScreenshots/1.png
 809     repo/packageName/locale/phoneScreenshots/2.png
 810
 811     The changelog files must be text files named with the versionCode
 812     ending with ".txt" and must be in the following layout:
 813     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 814
 815     repo/packageName/locale/changelogs/12345.txt
 816
 817     This will scan the each app's source repo then the metadata/ dir
 818     for these standard locations of changelog files.  If it finds
 819     them, they will be added to the dict of all packages, with the
 820     versions in the metadata/ folder taking precendence over the what
 821     is in the app's source repo.
 822
 823     Where "packageName" is the app's packageName and "locale" is the locale
 824     of the graphics, e.g. what language they are in, using the IETF RFC5646
 825     format (en-US, fr-CA, es-MX, etc).
 826
 827     This will also scan the app's git for a fastlane folder, and the
 828     metadata/ folder and the apps' source repos for standard locations
 829     of graphic and screenshot files.  If it finds them, it will copy
 830     them into the repo.  The fastlane files follow this pattern:
 831     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 832
 833     """
 834
 835     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 837     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 838     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 839
 840     for srcd in sorted(sourcedirs):
 841         if not os.path.isdir(srcd):
 842             continue
 843         for root, dirs, files in os.walk(srcd):
 844             segments = root.split('/')
 845             packageName = segments[1]
 846             if packageName not in apps:
 847                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 848                 continue
 849             locale = segments[-1]
 850             destdir = os.path.join('repo', packageName, locale)
 851
 852             # flavours specified in build receipt
 853             build_flavours = ""
 854             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 855                     and 'gradle' in apps[packageName].builds[-1]:
 856                 build_flavours = apps[packageName].builds[-1].gradle
 857
 858             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 859                 logging.debug("ignoring due to wrong flavour")
 860                 continue
 861
 862             for f in files:
 863                 if f in ('description.txt', 'full_description.txt'):
 864                     _set_localized_text_entry(apps[packageName], locale, 'description',
 865                                               os.path.join(root, f))
 866                     continue
 867                 elif f in ('summary.txt', 'short_description.txt'):
 868                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 869                                               os.path.join(root, f))
 870                     continue
 871                 elif f in ('name.txt', 'title.txt'):
 872                     _set_localized_text_entry(apps[packageName], locale, 'name',
 873                                               os.path.join(root, f))
 874                     continue
 875                 elif f == 'video.txt':
 876                     _set_localized_text_entry(apps[packageName], locale, 'video',
 877                                               os.path.join(root, f))
 878                     continue
 879                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 880                     locale = segments[-2]
 881                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 882                                               os.path.join(root, f))
 883                     continue
 884
 885                 base, extension = common.get_extension(f)
 886                 if locale == 'images':
 887                     locale = segments[-2]
 888                     destdir = os.path.join('repo', packageName, locale)
 889                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 890                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 891                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 892                     _strip_and_copy_image(os.path.join(root, f), destdir)
 893             for d in dirs:
 894                 if d in SCREENSHOT_DIRS:
 895                     if locale == 'images':
 896                         locale = segments[-2]
 897                         destdir = os.path.join('repo', packageName, locale)
 898                     for f in glob.glob(os.path.join(root, d, '*.*')):
 899                         _ignored, extension = common.get_extension(f)
 900                         if extension in ALLOWED_EXTENSIONS:
 901                             screenshotdestdir = os.path.join(destdir, d)
 902                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 903                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 904                             _strip_and_copy_image(f, screenshotdestdir)
 905
 906     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 907     for d in repofiles:
 908         if not os.path.isdir(d):
 909             continue
 910         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 911             if not os.path.isfile(f):
 912                 continue
 913             segments = f.split('/')
 914             packageName = segments[1]
 915             locale = segments[2]
 916             screenshotdir = segments[3]
 917             filename = os.path.basename(f)
 918             base, extension = common.get_extension(filename)
 919
 920             if packageName not in apps:
 921                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 922                                 .format(path=filename, name=packageName))
 923                 continue
 924             graphics = _get_localized_dict(apps[packageName], locale)
 925
 926             if extension not in ALLOWED_EXTENSIONS:
 927                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 928             elif base in GRAPHIC_NAMES:
 929                 # there can only be zero or one of these per locale
 930                 graphics[base] = filename
 931             elif screenshotdir in SCREENSHOT_DIRS:
 932                 # there can any number of these per locale
 933                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 934                 if screenshotdir not in graphics:
 935                     graphics[screenshotdir] = []
 936                 graphics[screenshotdir].append(filename)
 937             else:
 938                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 939
 940
 941 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 942     """Scan a repo for all files with an extension except APK/OBB
 943
 944     :param apkcache: current cached info about all repo files
 945     :param repodir: repo directory to scan
 946     :param knownapks: list of all known files, as per metadata.read_metadata
 947     :param use_date_from_file: use date from file (instead of current date)
 948                                for newly added files
 949     """
 950
 951     cachechanged = False
 952     repo_files = []
 953     repodir = repodir.encode('utf-8')
 954     for name in os.listdir(repodir):
 955         file_extension = common.get_file_extension(name)
 956         if file_extension == 'apk' or file_extension == 'obb':
 957             continue
 958         filename = os.path.join(repodir, name)
 959         name_utf8 = name.decode('utf-8')
 960         if filename.endswith(b'_src.tar.gz'):
 961             logging.debug(_('skipping source tarball: {path}')
 962                           .format(path=filename.decode('utf-8')))
 963             continue
 964         if not common.is_repo_file(filename):
 965             continue
 966         stat = os.stat(filename)
 967         if stat.st_size == 0:
 968             raise FDroidException(_('{path} is zero size!')
 969                                   .format(path=filename))
 970
 971         shasum = sha256sum(filename)
 972         usecache = False
 973         if name in apkcache:
 974             repo_file = apkcache[name]
 975             # added time is cached as tuple but used here as datetime instance
 976             if 'added' in repo_file:
 977                 a = repo_file['added']
 978                 if isinstance(a, datetime):
 979                     repo_file['added'] = a
 980                 else:
 981                     repo_file['added'] = datetime(*a[:6])
 982             if repo_file.get('hash') == shasum:
 983                 logging.debug(_("Reading {apkfilename} from cache")
 984                               .format(apkfilename=name_utf8))
 985                 usecache = True
 986             else:
 987                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 988                               .format(apkfilename=name_utf8))
 989
 990         if not usecache:
 991             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 992             repo_file = collections.OrderedDict()
 993             repo_file['name'] = os.path.splitext(name_utf8)[0]
 994             # TODO rename apkname globally to something more generic
 995             repo_file['apkName'] = name_utf8
 996             repo_file['hash'] = shasum
 997             repo_file['hashType'] = 'sha256'
 998             repo_file['versionCode'] = 0
 999             repo_file['versionName'] = shasum
1000             # the static ID is the SHA256 unless it is set in the metadata
1001             repo_file['packageName'] = shasum
1002
1003             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1004             if m:
1005                 repo_file['packageName'] = m.group(1)
1006                 repo_file['versionCode'] = int(m.group(2))
1007             srcfilename = name + b'_src.tar.gz'
1008             if os.path.exists(os.path.join(repodir, srcfilename)):
1009                 repo_file['srcname'] = srcfilename.decode('utf-8')
1010             repo_file['size'] = stat.st_size
1011
1012             apkcache[name] = repo_file
1013             cachechanged = True
1014
1015         if use_date_from_file:
1016             timestamp = stat.st_ctime
1017             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1018         else:
1019             default_date_param = None
1020
1021         # Record in knownapks, getting the added date at the same time..
1022         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1023                                     default_date=default_date_param)
1024         if added:
1025             repo_file['added'] = added
1026
1027         repo_files.append(repo_file)
1028
1029     return repo_files, cachechanged
1030
1031
1032 def scan_apk(apk_file):
1033     """
1034     Scans an APK file and returns dictionary with metadata of the APK.
1035
1036     Attention: This does *not* verify that the APK signature is correct.
1037
1038     :param apk_file: The (ideally absolute) path to the APK file
1039     :raises BuildException
1040     :return A dict containing APK metadata
1041     """
1042     apk = {
1043         'hash': sha256sum(apk_file),
1044         'hashType': 'sha256',
1045         'uses-permission': [],
1046         'uses-permission-sdk-23': [],
1047         'features': [],
1048         'icons_src': {},
1049         'icons': {},
1050         'antiFeatures': set(),
1051     }
1052
1053     if SdkToolsPopen(['aapt', 'version'], output=False):
1054         scan_apk_aapt(apk, apk_file)
1055     else:
1056         scan_apk_androguard(apk, apk_file)
1057
1058     # Get the signature, or rather the signing key fingerprints
1059     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1060     apk['sig'] = getsig(apk_file)
1061     if not apk['sig']:
1062         raise BuildException("Failed to get apk signature")
1063     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1064                                                                apk_file))
1065     if not apk.get('signer'):
1066         raise BuildException("Failed to get apk signing key fingerprint")
1067
1068     # Get size of the APK
1069     apk['size'] = os.path.getsize(apk_file)
1070
1071     if 'minSdkVersion' not in apk:
1072         logging.warning("No SDK version information found in {0}".format(apk_file))
1073         apk['minSdkVersion'] = 1
1074
1075     # Check for known vulnerabilities
1076     if has_known_vulnerability(apk_file):
1077         apk['antiFeatures'].add('KnownVuln')
1078
1079     return apk
1080
1081
1082 def scan_apk_aapt(apk, apkfile):
1083     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1084     if p.returncode != 0:
1085         if options.delete_unknown:
1086             if os.path.exists(apkfile):
1087                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1088                 os.remove(apkfile)
1089             else:
1090                 logging.error("Could not find {0} to remove it".format(apkfile))
1091         else:
1092             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1093         raise BuildException(_("Invalid APK"))
1094     for line in p.output.splitlines():
1095         if line.startswith("package:"):
1096             try:
1097                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1098                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1099                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1100             except Exception as e:
1101                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1102         elif line.startswith("application:"):
1103             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1104             # Keep path to non-dpi icon in case we need it
1105             match = re.match(APK_ICON_PAT_NODPI, line)
1106             if match:
1107                 apk['icons_src']['-1'] = match.group(1)
1108         elif line.startswith("launchable-activity:"):
1109             # Only use launchable-activity as fallback to application
1110             if not apk['name']:
1111                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1112             if '-1' not in apk['icons_src']:
1113                 match = re.match(APK_ICON_PAT_NODPI, line)
1114                 if match:
1115                     apk['icons_src']['-1'] = match.group(1)
1116         elif line.startswith("application-icon-"):
1117             match = re.match(APK_ICON_PAT, line)
1118             if match:
1119                 density = match.group(1)
1120                 path = match.group(2)
1121                 apk['icons_src'][density] = path
1122         elif line.startswith("sdkVersion:"):
1123             m = re.match(APK_SDK_VERSION_PAT, line)
1124             if m is None:
1125                 logging.error(line.replace('sdkVersion:', '')
1126                               + ' is not a valid minSdkVersion!')
1127             else:
1128                 apk['minSdkVersion'] = m.group(1)
1129                 # if target not set, default to min
1130                 if 'targetSdkVersion' not in apk:
1131                     apk['targetSdkVersion'] = m.group(1)
1132         elif line.startswith("targetSdkVersion:"):
1133             m = re.match(APK_SDK_VERSION_PAT, line)
1134             if m is None:
1135                 logging.error(line.replace('targetSdkVersion:', '')
1136                               + ' is not a valid targetSdkVersion!')
1137             else:
1138                 apk['targetSdkVersion'] = m.group(1)
1139         elif line.startswith("maxSdkVersion:"):
1140             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1141         elif line.startswith("native-code:"):
1142             apk['nativecode'] = []
1143             for arch in line[13:].split(' '):
1144                 apk['nativecode'].append(arch[1:-1])
1145         elif line.startswith('uses-permission:'):
1146             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1147             if perm_match['maxSdkVersion']:
1148                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1149             permission = UsesPermission(
1150                 perm_match['name'],
1151                 perm_match['maxSdkVersion']
1152             )
1153
1154             apk['uses-permission'].append(permission)
1155         elif line.startswith('uses-permission-sdk-23:'):
1156             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1157             if perm_match['maxSdkVersion']:
1158                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1159             permission_sdk_23 = UsesPermissionSdk23(
1160                 perm_match['name'],
1161                 perm_match['maxSdkVersion']
1162             )
1163
1164             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1165
1166         elif line.startswith('uses-feature:'):
1167             feature = re.match(APK_FEATURE_PAT, line).group(1)
1168             # Filter out this, it's only added with the latest SDK tools and
1169             # causes problems for lots of apps.
1170             if feature != "android.hardware.screen.portrait" \
1171                     and feature != "android.hardware.screen.landscape":
1172                 if feature.startswith("android.feature."):
1173                     feature = feature[16:]
1174                 apk['features'].add(feature)
1175
1176
1177 def scan_apk_androguard(apk, apkfile):
1178     try:
1179         from androguard.core.bytecodes.apk import APK
1180         apkobject = APK(apkfile)
1181         if apkobject.is_valid_APK():
1182             arsc = apkobject.get_android_resources()
1183         else:
1184             if options.delete_unknown:
1185                 if os.path.exists(apkfile):
1186                     logging.error(_("Failed to get apk information, deleting {path}")
1187                                   .format(path=apkfile))
1188                     os.remove(apkfile)
1189                 else:
1190                     logging.error(_("Could not find {path} to remove it")
1191                                   .format(path=apkfile))
1192             else:
1193                 logging.error(_("Failed to get apk information, skipping {path}")
1194                               .format(path=apkfile))
1195             raise BuildException(_("Invalid APK"))
1196     except ImportError:
1197         raise FDroidException("androguard library is not installed and aapt not present")
1198     except FileNotFoundError:
1199         logging.error(_("Could not open apk file for analysis"))
1200         raise BuildException(_("Invalid APK"))
1201
1202     apk['packageName'] = apkobject.get_package()
1203     apk['versionCode'] = int(apkobject.get_androidversion_code())
1204     apk['versionName'] = apkobject.get_androidversion_name()
1205     if apk['versionName'][0] == "@":
1206         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1207         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1208         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1209     apk['name'] = apkobject.get_app_name()
1210
1211     if apkobject.get_max_sdk_version() is not None:
1212         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1213     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1214     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1215
1216     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1217     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1218
1219     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1220
1221     for file in apkobject.get_files():
1222         d_re = density_re.match(file)
1223         if d_re:
1224             folder = d_re.group(1).split('-')
1225             if len(folder) > 1:
1226                 resolution = folder[1]
1227             else:
1228                 resolution = 'mdpi'
1229             density = screen_resolutions[resolution]
1230             apk['icons_src'][density] = d_re.group(0)
1231
1232     if apk['icons_src'].get('-1') is None:
1233         apk['icons_src']['-1'] = apk['icons_src']['160']
1234
1235     arch_re = re.compile("^lib/(.*)/.*$")
1236     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1237     if len(arch) >= 1:
1238         apk['nativecode'] = []
1239         apk['nativecode'].extend(sorted(list(arch)))
1240
1241     xml = apkobject.get_android_manifest_xml()
1242
1243     for item in xml.getElementsByTagName('uses-permission'):
1244         name = str(item.getAttribute("android:name"))
1245         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1246         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1247         permission = UsesPermission(
1248             name,
1249             maxSdkVersion
1250         )
1251         apk['uses-permission'].append(permission)
1252
1253     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1254         name = str(item.getAttribute("android:name"))
1255         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1256         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1257         permission_sdk_23 = UsesPermissionSdk23(
1258             name,
1259             maxSdkVersion
1260         )
1261         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1262
1263     for item in xml.getElementsByTagName('uses-feature'):
1264         feature = str(item.getAttribute("android:name"))
1265         if feature != "android.hardware.screen.portrait" \
1266                 and feature != "android.hardware.screen.landscape":
1267             if feature.startswith("android.feature."):
1268                 feature = feature[16:]
1269         apk['features'].append(feature)
1270
1271
1272 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1273                 allow_disabled_algorithms=False, archive_bad_sig=False):
1274     """Processes the apk with the given filename in the given repo directory.
1275
1276     This also extracts the icons.
1277
1278     :param apkcache: current apk cache information
1279     :param apkfilename: the filename of the apk to scan
1280     :param repodir: repo directory to scan
1281     :param knownapks: known apks info
1282     :param use_date_from_apk: use date from APK (instead of current date)
1283                               for newly added APKs
1284     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1285                                       disabled algorithms in the signature (e.g. MD5)
1286     :param archive_bad_sig: move APKs with a bad signature to the archive
1287     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1288      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1289     """
1290
1291     apk = {}
1292     apkfile = os.path.join(repodir, apkfilename)
1293
1294     cachechanged = False
1295     usecache = False
1296     if apkfilename in apkcache:
1297         apk = apkcache[apkfilename]
1298         if apk.get('hash') == sha256sum(apkfile):
1299             logging.debug(_("Reading {apkfilename} from cache")
1300                           .format(apkfilename=apkfilename))
1301             usecache = True
1302         else:
1303             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1304                           .format(apkfilename=apkfilename))
1305
1306     if not usecache:
1307         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1308
1309         try:
1310             apk = scan_apk(apkfile)
1311         except BuildException:
1312             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1313                             .format(apkfilename=apkfilename))
1314             return True, None, False
1315
1316         # Check for debuggable apks...
1317         if common.isApkAndDebuggable(apkfile):
1318             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1319
1320         if options.rename_apks:
1321             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1322             std_short_name = os.path.join(repodir, n)
1323             if apkfile != std_short_name:
1324                 if os.path.exists(std_short_name):
1325                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1326                     if apkfile != std_long_name:
1327                         if os.path.exists(std_long_name):
1328                             dupdir = os.path.join('duplicates', repodir)
1329                             if not os.path.isdir(dupdir):
1330                                 os.makedirs(dupdir, exist_ok=True)
1331                             dupfile = os.path.join('duplicates', std_long_name)
1332                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1333                             os.rename(apkfile, dupfile)
1334                             return True, None, False
1335                         else:
1336                             os.rename(apkfile, std_long_name)
1337                     apkfile = std_long_name
1338                 else:
1339                     os.rename(apkfile, std_short_name)
1340                     apkfile = std_short_name
1341                 apkfilename = apkfile[len(repodir) + 1:]
1342
1343         apk['apkName'] = apkfilename
1344         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1345         if os.path.exists(os.path.join(repodir, srcfilename)):
1346             apk['srcname'] = srcfilename
1347
1348         # verify the jar signature is correct, allow deprecated
1349         # algorithms only if the APK is in the archive.
1350         skipapk = False
1351         if not common.verify_apk_signature(apkfile):
1352             if repodir == 'archive' or allow_disabled_algorithms:
1353                 if common.verify_old_apk_signature(apkfile):
1354                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1355                 else:
1356                     skipapk = True
1357             else:
1358                 skipapk = True
1359
1360         if skipapk:
1361             if archive_bad_sig:
1362                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1363                                 .format(apkfilename=apkfilename))
1364                 move_apk_between_sections(repodir, 'archive', apk)
1365             else:
1366                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1367                                 .format(apkfilename=apkfilename))
1368             return True, None, False
1369
1370         apkzip = zipfile.ZipFile(apkfile, 'r')
1371
1372         manifest = apkzip.getinfo('AndroidManifest.xml')
1373         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1374         if (1980, 0, 0) != manifest.date_time[0:3]:
1375             try:
1376                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1377             except ValueError as e:
1378                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1379                                 .format(apkfilename=apkfile) + str(e))
1380
1381         # extract icons from APK zip file
1382         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1383         try:
1384             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1385         finally:
1386             apkzip.close()  # ensure that APK zip file gets closed
1387
1388         # resize existing icons for densities missing in the APK
1389         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1390
1391         if use_date_from_apk and manifest.date_time[1] != 0:
1392             default_date_param = datetime(*manifest.date_time)
1393         else:
1394             default_date_param = None
1395
1396         # Record in known apks, getting the added date at the same time..
1397         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1398                                     default_date=default_date_param)
1399         if added:
1400             apk['added'] = added
1401
1402         apkcache[apkfilename] = apk
1403         cachechanged = True
1404
1405     return False, apk, cachechanged
1406
1407
1408 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1409     """Processes the apks in the given repo directory.
1410
1411     This also extracts the icons.
1412
1413     :param apkcache: current apk cache information
1414     :param repodir: repo directory to scan
1415     :param knownapks: known apks info
1416     :param use_date_from_apk: use date from APK (instead of current date)
1417                               for newly added APKs
1418     :returns: (apks, cachechanged) where apks is a list of apk information,
1419               and cachechanged is True if the apkcache got changed.
1420     """
1421
1422     cachechanged = False
1423
1424     for icon_dir in get_all_icon_dirs(repodir):
1425         if os.path.exists(icon_dir):
1426             if options.clean:
1427                 shutil.rmtree(icon_dir)
1428                 os.makedirs(icon_dir)
1429         else:
1430             os.makedirs(icon_dir)
1431
1432     apks = []
1433     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1434         apkfilename = apkfile[len(repodir) + 1:]
1435         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1436         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1437                                              use_date_from_apk, ada, True)
1438         if skip:
1439             continue
1440         apks.append(apk)
1441         cachechanged = cachechanged or cachethis
1442
1443     return apks, cachechanged
1444
1445
1446 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1447     """
1448     Extracts icons from the given APK zip in various densities,
1449     saves them into given repo directory
1450     and stores their names in the APK metadata dictionary.
1451
1452     :param icon_filename: A string representing the icon's file name
1453     :param apk: A populated dictionary containing APK metadata.
1454                 Needs to have 'icons_src' key
1455     :param apkzip: An opened zipfile.ZipFile of the APK file
1456     :param repo_dir: The directory of the APK's repository
1457     :return: A list of icon densities that are missing
1458     """
1459     empty_densities = []
1460     for density in screen_densities:
1461         if density not in apk['icons_src']:
1462             empty_densities.append(density)
1463             continue
1464         icon_src = apk['icons_src'][density]
1465         icon_dir = get_icon_dir(repo_dir, density)
1466         icon_dest = os.path.join(icon_dir, icon_filename)
1467
1468         # Extract the icon files per density
1469         if icon_src.endswith('.xml'):
1470             png = os.path.basename(icon_src)[:-4] + '.png'
1471             for f in apkzip.namelist():
1472                 if f.endswith(png):
1473                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1474                     if m and screen_resolutions[m.group(2)] == density:
1475                         icon_src = f
1476             if icon_src.endswith('.xml'):
1477                 empty_densities.append(density)
1478                 continue
1479         try:
1480             with open(icon_dest, 'wb') as f:
1481                 f.write(get_icon_bytes(apkzip, icon_src))
1482             apk['icons'][density] = icon_filename
1483         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1484             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1485             del apk['icons_src'][density]
1486             empty_densities.append(density)
1487
1488     if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
1489         icon_src = apk['icons_src']['-1']
1490         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1491         with open(icon_path, 'wb') as f:
1492             f.write(get_icon_bytes(apkzip, icon_src))
1493         im = None
1494         try:
1495             im = Image.open(icon_path)
1496             dpi = px_to_dpi(im.size[0])
1497             for density in screen_densities:
1498                 if density in apk['icons']:
1499                     break
1500                 if density == screen_densities[-1] or dpi >= int(density):
1501                     apk['icons'][density] = icon_filename
1502                     shutil.move(icon_path,
1503                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1504                     empty_densities.remove(density)
1505                     break
1506         except Exception as e:
1507             logging.warning(_("Failed reading {path}: {error}")
1508                             .format(path=icon_path, error=e))
1509         finally:
1510             if im and hasattr(im, 'close'):
1511                 im.close()
1512
1513     if apk['icons']:
1514         apk['icon'] = icon_filename
1515
1516     return empty_densities
1517
1518
1519 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1520     """
1521     Resize existing icons for densities missing in the APK to ensure all densities are available
1522
1523     :param empty_densities: A list of icon densities that are missing
1524     :param icon_filename: A string representing the icon's file name
1525     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1526     :param repo_dir: The directory of the APK's repository
1527     """
1528     # First try resizing down to not lose quality
1529     last_density = None
1530     for density in screen_densities:
1531         if density not in empty_densities:
1532             last_density = density
1533             continue
1534         if last_density is None:
1535             continue
1536         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1537
1538         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1539         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1540         fp = None
1541         try:
1542             fp = open(last_icon_path, 'rb')
1543             im = Image.open(fp)
1544
1545             size = dpi_to_px(density)
1546
1547             im.thumbnail((size, size), Image.ANTIALIAS)
1548             im.save(icon_path, "PNG", optimize=True,
1549                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1550             empty_densities.remove(density)
1551         except Exception as e:
1552             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1553         finally:
1554             if fp:
1555                 fp.close()
1556
1557     # Then just copy from the highest resolution available
1558     last_density = None
1559     for density in reversed(screen_densities):
1560         if density not in empty_densities:
1561             last_density = density
1562             continue
1563
1564         if last_density is None:
1565             continue
1566
1567         shutil.copyfile(
1568             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1569             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1570         )
1571         empty_densities.remove(density)
1572
1573     for density in screen_densities:
1574         icon_dir = get_icon_dir(repo_dir, density)
1575         icon_dest = os.path.join(icon_dir, icon_filename)
1576         resize_icon(icon_dest, density)
1577
1578     # Copy from icons-mdpi to icons since mdpi is the baseline density
1579     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1580     if os.path.isfile(baseline):
1581         apk['icons']['0'] = icon_filename
1582         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1583
1584
1585 def apply_info_from_latest_apk(apps, apks):
1586     """
1587     Some information from the apks needs to be applied up to the application level.
1588     When doing this, we use the info from the most recent version's apk.
1589     We deal with figuring out when the app was added and last updated at the same time.
1590     """
1591     for appid, app in apps.items():
1592         bestver = UNSET_VERSION_CODE
1593         for apk in apks:
1594             if apk['packageName'] == appid:
1595                 if apk['versionCode'] > bestver:
1596                     bestver = apk['versionCode']
1597                     bestapk = apk
1598
1599                 if 'added' in apk:
1600                     if not app.added or apk['added'] < app.added:
1601                         app.added = apk['added']
1602                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1603                         app.lastUpdated = apk['added']
1604
1605         if not app.added:
1606             logging.debug("Don't know when " + appid + " was added")
1607         if not app.lastUpdated:
1608             logging.debug("Don't know when " + appid + " was last updated")
1609
1610         if bestver == UNSET_VERSION_CODE:
1611
1612             if app.Name is None:
1613                 app.Name = app.AutoName or appid
1614             app.icon = None
1615             logging.debug("Application " + appid + " has no packages")
1616         else:
1617             if app.Name is None:
1618                 app.Name = bestapk['name']
1619             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1620             if app.CurrentVersionCode is None:
1621                 app.CurrentVersionCode = str(bestver)
1622
1623
1624 def make_categories_txt(repodir, categories):
1625     '''Write a category list in the repo to allow quick access'''
1626     catdata = ''
1627     for cat in sorted(categories):
1628         catdata += cat + '\n'
1629     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1630         f.write(catdata)
1631
1632
1633 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1634
1635     def filter_apk_list_sorted(apk_list):
1636         res = []
1637         for apk in apk_list:
1638             if apk['packageName'] == appid:
1639                 res.append(apk)
1640
1641         # Sort the apk list by version code. First is highest/newest.
1642         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1643
1644     for appid, app in apps.items():
1645
1646         if app.ArchivePolicy:
1647             keepversions = int(app.ArchivePolicy[:-9])
1648         else:
1649             keepversions = defaultkeepversions
1650
1651         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1652                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1653
1654         current_app_apks = filter_apk_list_sorted(apks)
1655         if len(current_app_apks) > keepversions:
1656             # Move back the ones we don't want.
1657             for apk in current_app_apks[keepversions:]:
1658                 move_apk_between_sections(repodir, archivedir, apk)
1659                 archapks.append(apk)
1660                 apks.remove(apk)
1661
1662         current_app_archapks = filter_apk_list_sorted(archapks)
1663         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1664             kept = 0
1665             # Move forward the ones we want again, except DisableAlgorithm
1666             for apk in current_app_archapks:
1667                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1668                     move_apk_between_sections(archivedir, repodir, apk)
1669                     archapks.remove(apk)
1670                     apks.append(apk)
1671                     kept += 1
1672                 if kept == keepversions:
1673                     break
1674
1675
1676 def move_apk_between_sections(from_dir, to_dir, apk):
1677     """move an APK from repo to archive or vice versa"""
1678
1679     def _move_file(from_dir, to_dir, filename, ignore_missing):
1680         from_path = os.path.join(from_dir, filename)
1681         if ignore_missing and not os.path.exists(from_path):
1682             return
1683         to_path = os.path.join(to_dir, filename)
1684         if not os.path.exists(to_dir):
1685             os.mkdir(to_dir)
1686         shutil.move(from_path, to_path)
1687
1688     if from_dir == to_dir:
1689         return
1690
1691     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1692     _move_file(from_dir, to_dir, apk['apkName'], False)
1693     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1694     for density in all_screen_densities:
1695         from_icon_dir = get_icon_dir(from_dir, density)
1696         to_icon_dir = get_icon_dir(to_dir, density)
1697         if density not in apk.get('icons', []):
1698             continue
1699         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1700     if 'srcname' in apk:
1701         _move_file(from_dir, to_dir, apk['srcname'], False)
1702
1703
1704 def add_apks_to_per_app_repos(repodir, apks):
1705     apks_per_app = dict()
1706     for apk in apks:
1707         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1708         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1709         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1710         apks_per_app[apk['packageName']] = apk
1711
1712         if not os.path.exists(apk['per_app_icons']):
1713             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1714             os.makedirs(apk['per_app_icons'])
1715
1716         apkpath = os.path.join(repodir, apk['apkName'])
1717         shutil.copy(apkpath, apk['per_app_repo'])
1718         apksigpath = apkpath + '.sig'
1719         if os.path.exists(apksigpath):
1720             shutil.copy(apksigpath, apk['per_app_repo'])
1721         apkascpath = apkpath + '.asc'
1722         if os.path.exists(apkascpath):
1723             shutil.copy(apkascpath, apk['per_app_repo'])
1724
1725
1726 def create_metadata_from_template(apk):
1727     '''create a new metadata file using internal or external template
1728
1729     Generate warnings for apk's with no metadata (or create skeleton
1730     metadata files, if requested on the command line).  Though the
1731     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1732     since those impose things on the metadata file made from the
1733     template: field sort order, empty field value, formatting, etc.
1734     '''
1735
1736     import yaml
1737     if os.path.exists('template.yml'):
1738         with open('template.yml') as f:
1739             metatxt = f.read()
1740         if 'name' in apk and apk['name'] != '':
1741             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1742                              r'\1 ' + apk['name'],
1743                              metatxt,
1744                              flags=re.IGNORECASE | re.MULTILINE)
1745         else:
1746             logging.warning(_('{appid} does not have a name! Using package name instead.')
1747                             .format(appid=apk['packageName']))
1748             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1749                              r'\1 ' + apk['packageName'],
1750                              metatxt,
1751                              flags=re.IGNORECASE | re.MULTILINE)
1752         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1753             f.write(metatxt)
1754     else:
1755         app = dict()
1756         app['Categories'] = [os.path.basename(os.getcwd())]
1757         # include some blanks as part of the template
1758         app['AuthorName'] = ''
1759         app['Summary'] = ''
1760         app['WebSite'] = ''
1761         app['IssueTracker'] = ''
1762         app['SourceCode'] = ''
1763         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1764         if 'name' in apk and apk['name'] != '':
1765             app['Name'] = apk['name']
1766         else:
1767             logging.warning(_('{appid} does not have a name! Using package name instead.')
1768                             .format(appid=apk['packageName']))
1769             app['Name'] = apk['packageName']
1770         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1771             yaml.dump(app, f, default_flow_style=False)
1772     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1773
1774
1775 config = None
1776 options = None
1777 start_timestamp = time.gmtime()
1778
1779
1780 def main():
1781
1782     global config, options
1783
1784     # Parse command line...
1785     parser = ArgumentParser()
1786     common.setup_global_opts(parser)
1787     parser.add_argument("--create-key", action="store_true", default=False,
1788                         help=_("Add a repo signing key to an unsigned repo"))
1789     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1790                         help=_("Add skeleton metadata files for APKs that are missing them"))
1791     parser.add_argument("--delete-unknown", action="store_true", default=False,
1792                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1793     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1794                         help=_("Report on build data status"))
1795     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1796                         help=_("Interactively ask about things that need updating."))
1797     parser.add_argument("-I", "--icons", action="store_true", default=False,
1798                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1799     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1800                         help=_("Specify editor to use in interactive mode. Default " +
1801                                "is {path}").format(path='/etc/alternatives/editor'))
1802     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1803                         help=_("Update the wiki"))
1804     parser.add_argument("--pretty", action="store_true", default=False,
1805                         help=_("Produce human-readable XML/JSON for index files"))
1806     parser.add_argument("--clean", action="store_true", default=False,
1807                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1808     parser.add_argument("--nosign", action="store_true", default=False,
1809                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1810     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1811                         help=_("Use date from APK instead of current time for newly added APKs"))
1812     parser.add_argument("--rename-apks", action="store_true", default=False,
1813                         help=_("Rename APK files that do not match package.name_123.apk"))
1814     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1815                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1816     metadata.add_metadata_arguments(parser)
1817     options = parser.parse_args()
1818     metadata.warnings_action = options.W
1819
1820     config = common.read_config(options)
1821
1822     if not ('jarsigner' in config and 'keytool' in config):
1823         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1824
1825     repodirs = ['repo']
1826     if config['archive_older'] != 0:
1827         repodirs.append('archive')
1828         if not os.path.exists('archive'):
1829             os.mkdir('archive')
1830
1831     if options.icons:
1832         resize_all_icons(repodirs)
1833         sys.exit(0)
1834
1835     if options.rename_apks:
1836         options.clean = True
1837
1838     # check that icons exist now, rather than fail at the end of `fdroid update`
1839     for k in ['repo_icon', 'archive_icon']:
1840         if k in config:
1841             if not os.path.exists(config[k]):
1842                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1843                                  .format(name=k, path=config[k]))
1844                 sys.exit(1)
1845
1846     # if the user asks to create a keystore, do it now, reusing whatever it can
1847     if options.create_key:
1848         if os.path.exists(config['keystore']):
1849             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1850             logging.critical("\t'" + config['keystore'] + "'")
1851             sys.exit(1)
1852
1853         if 'repo_keyalias' not in config:
1854             config['repo_keyalias'] = socket.getfqdn()
1855             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1856         if 'keydname' not in config:
1857             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1858             common.write_to_config(config, 'keydname', config['keydname'])
1859         if 'keystore' not in config:
1860             config['keystore'] = common.default_config['keystore']
1861             common.write_to_config(config, 'keystore', config['keystore'])
1862
1863         password = common.genpassword()
1864         if 'keystorepass' not in config:
1865             config['keystorepass'] = password
1866             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1867         if 'keypass' not in config:
1868             config['keypass'] = password
1869             common.write_to_config(config, 'keypass', config['keypass'])
1870         common.genkeystore(config)
1871
1872     # Get all apps...
1873     apps = metadata.read_metadata()
1874
1875     # Generate a list of categories...
1876     categories = set()
1877     for app in apps.values():
1878         categories.update(app.Categories)
1879
1880     # Read known apks data (will be updated and written back when we've finished)
1881     knownapks = common.KnownApks()
1882
1883     # Get APK cache
1884     apkcache = get_cache()
1885
1886     # Delete builds for disabled apps
1887     delete_disabled_builds(apps, apkcache, repodirs)
1888
1889     # Scan all apks in the main repo
1890     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1891
1892     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1893                                            options.use_date_from_apk)
1894     cachechanged = cachechanged or fcachechanged
1895     apks += files
1896     for apk in apks:
1897         if apk['packageName'] not in apps:
1898             if options.create_metadata:
1899                 create_metadata_from_template(apk)
1900                 apps = metadata.read_metadata()
1901             else:
1902                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1903                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1904                 if options.delete_unknown:
1905                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1906                                  .format(apkfilename=apk['apkName']))
1907                     rmf = os.path.join(repodirs[0], apk['apkName'])
1908                     if not os.path.exists(rmf):
1909                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1910                     else:
1911                         os.remove(rmf)
1912                 else:
1913                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1914
1915     copy_triple_t_store_metadata(apps)
1916     insert_obbs(repodirs[0], apps, apks)
1917     insert_localized_app_metadata(apps)
1918     translate_per_build_anti_features(apps, apks)
1919
1920     # Scan the archive repo for apks as well
1921     if len(repodirs) > 1:
1922         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1923         if cc:
1924             cachechanged = True
1925     else:
1926         archapks = []
1927
1928     # Apply information from latest apks to the application and update dates
1929     apply_info_from_latest_apk(apps, apks + archapks)
1930
1931     # Sort the app list by name, then the web site doesn't have to by default.
1932     # (we had to wait until we'd scanned the apks to do this, because mostly the
1933     # name comes from there!)
1934     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1935
1936     # APKs are placed into multiple repos based on the app package, providing
1937     # per-app subscription feeds for nightly builds and things like it
1938     if config['per_app_repos']:
1939         add_apks_to_per_app_repos(repodirs[0], apks)
1940         for appid, app in apps.items():
1941             repodir = os.path.join(appid, 'fdroid', 'repo')
1942             appdict = dict()
1943             appdict[appid] = app
1944             if os.path.isdir(repodir):
1945                 index.make(appdict, [appid], apks, repodir, False)
1946             else:
1947                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1948         return
1949
1950     if len(repodirs) > 1:
1951         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1952
1953     # Make the index for the main repo...
1954     index.make(apps, sortedids, apks, repodirs[0], False)
1955     make_categories_txt(repodirs[0], categories)
1956
1957     # If there's an archive repo,  make the index for it. We already scanned it
1958     # earlier on.
1959     if len(repodirs) > 1:
1960         index.make(apps, sortedids, archapks, repodirs[1], True)
1961
1962     git_remote = config.get('binary_transparency_remote')
1963     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1964         from . import btlog
1965         btlog.make_binary_transparency_log(repodirs)
1966
1967     if config['update_stats']:
1968         # Update known apks info...
1969         knownapks.writeifchanged()
1970
1971         # Generate latest apps data for widget
1972         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1973             data = ''
1974             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1975                 for line in f:
1976                     appid = line.rstrip()
1977                     data += appid + "\t"
1978                     app = apps[appid]
1979                     data += app.Name + "\t"
1980                     if app.icon is not None:
1981                         data += app.icon + "\t"
1982                     data += app.License + "\n"
1983             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1984                 f.write(data)
1985
1986     if cachechanged:
1987         write_cache(apkcache)
1988
1989     # Update the wiki...
1990     if options.wiki:
1991         update_wiki(apps, sortedids, apks + archapks)
1992
1993     logging.info(_("Finished"))
1994
1995
1996 if __name__ == "__main__":
1997     main()