fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*?)'")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
  63 # resolutions must end with 'dpi'
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "tvdpi": '213',
  72     "undefineddpi": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0' or density == '65534':
 100         return os.path.join(repodir, "icons")
 101     else:
 102         return os.path.join(repodir, "icons-%s" % density)
 103
 104
 105 def get_icon_dirs(repodir):
 106     for density in screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def get_all_icon_dirs(repodir):
 111     for density in all_screen_densities:
 112         yield get_icon_dir(repodir, density)
 113
 114
 115 def update_wiki(apps, sortedids, apks):
 116     """Update the wiki
 117
 118     :param apps: fully populated list of all applications
 119     :param apks: all apks, except...
 120     """
 121     logging.info("Updating wiki")
 122     wikicat = 'Apps'
 123     wikiredircat = 'App Redirects'
 124     import mwclient
 125     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 126                          path=config['wiki_path'])
 127     site.login(config['wiki_user'], config['wiki_password'])
 128     generated_pages = {}
 129     generated_redirects = {}
 130
 131     for appid in sortedids:
 132         app = metadata.App(apps[appid])
 133
 134         wikidata = ''
 135         if app.Disabled:
 136             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 137         if app.AntiFeatures:
 138             for af in sorted(app.AntiFeatures):
 139                 wikidata += '{{AntiFeature|' + af + '}}\n'
 140         if app.RequiresRoot:
 141             requiresroot = 'Yes'
 142         else:
 143             requiresroot = 'No'
 144         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 145             appid,
 146             app.Name,
 147             app.added.strftime('%Y-%m-%d') if app.added else '',
 148             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 149             app.SourceCode,
 150             app.IssueTracker,
 151             app.WebSite,
 152             app.Changelog,
 153             app.Donate,
 154             app.FlattrID,
 155             app.LiberapayID,
 156             app.Bitcoin,
 157             app.Litecoin,
 158             app.License,
 159             requiresroot,
 160             app.AuthorName,
 161             app.AuthorEmail)
 162
 163         if app.Provides:
 164             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 165
 166         wikidata += app.Summary
 167         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 168
 169         wikidata += "=Description=\n"
 170         wikidata += metadata.description_wiki(app.Description) + "\n"
 171
 172         wikidata += "=Maintainer Notes=\n"
 173         if app.MaintainerNotes:
 174             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 175         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 176
 177         # Get a list of all packages for this application...
 178         apklist = []
 179         gotcurrentver = False
 180         cantupdate = False
 181         buildfails = False
 182         for apk in apks:
 183             if apk['packageName'] == appid:
 184                 if str(apk['versionCode']) == app.CurrentVersionCode:
 185                     gotcurrentver = True
 186                 apklist.append(apk)
 187         # Include ones we can't build, as a special case...
 188         for build in app.builds:
 189             if build.disable:
 190                 if build.versionCode == app.CurrentVersionCode:
 191                     cantupdate = True
 192                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 193                 apklist.append({'versionCode': int(build.versionCode),
 194                                 'versionName': build.versionName,
 195                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 196                                 })
 197             else:
 198                 builtit = False
 199                 for apk in apklist:
 200                     if apk['versionCode'] == int(build.versionCode):
 201                         builtit = True
 202                         break
 203                 if not builtit:
 204                     buildfails = True
 205                     apklist.append({'versionCode': int(build.versionCode),
 206                                     'versionName': build.versionName,
 207                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 208                                     })
 209         if app.CurrentVersionCode == '0':
 210             cantupdate = True
 211         # Sort with most recent first...
 212         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 213
 214         wikidata += "=Versions=\n"
 215         if len(apklist) == 0:
 216             wikidata += "We currently have no versions of this app available."
 217         elif not gotcurrentver:
 218             wikidata += "We don't have the current version of this app."
 219         else:
 220             wikidata += "We have the current version of this app."
 221         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 222         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 223         if len(app.NoSourceSince) > 0:
 224             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 225         if len(app.CurrentVersion) > 0:
 226             wikidata += "The current (recommended) version is " + app.CurrentVersion
 227             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 228         validapks = 0
 229         for apk in apklist:
 230             wikidata += "==" + apk['versionName'] + "==\n"
 231
 232             if 'buildproblem' in apk:
 233                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 234             else:
 235                 validapks += 1
 236                 wikidata += "This version is built and signed by "
 237                 if 'srcname' in apk:
 238                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 239                 else:
 240                     wikidata += "the original developer.\n\n"
 241             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 242
 243         wikidata += '\n[[Category:' + wikicat + ']]\n'
 244         if len(app.NoSourceSince) > 0:
 245             wikidata += '\n[[Category:Apps missing source code]]\n'
 246         if validapks == 0 and not app.Disabled:
 247             wikidata += '\n[[Category:Apps with no packages]]\n'
 248         if cantupdate and not app.Disabled:
 249             wikidata += "\n[[Category:Apps we cannot update]]\n"
 250         if buildfails and not app.Disabled:
 251             wikidata += "\n[[Category:Apps with failing builds]]\n"
 252         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 253             wikidata += '\n[[Category:Apps to Update]]\n'
 254         if app.Disabled:
 255             wikidata += '\n[[Category:Apps that are disabled]]\n'
 256         if app.UpdateCheckMode == 'None' and not app.Disabled:
 257             wikidata += '\n[[Category:Apps with no update check]]\n'
 258         for appcat in app.Categories:
 259             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 260
 261         # We can't have underscores in the page name, even if they're in
 262         # the package ID, because MediaWiki messes with them...
 263         pagename = appid.replace('_', ' ')
 264
 265         # Drop a trailing newline, because mediawiki is going to drop it anyway
 266         # and it we don't we'll think the page has changed when it hasn't...
 267         if wikidata.endswith('\n'):
 268             wikidata = wikidata[:-1]
 269
 270         generated_pages[pagename] = wikidata
 271
 272         # Make a redirect from the name to the ID too, unless there's
 273         # already an existing page with the name and it isn't a redirect.
 274         noclobber = False
 275         apppagename = app.Name
 276         for ch in '_{}:[]|':
 277             apppagename = apppagename.replace(ch, ' ')
 278         # Drop double spaces caused mostly by replacing ':' above
 279         apppagename = apppagename.replace('  ', ' ')
 280         for expagename in site.allpages(prefix=apppagename,
 281                                         filterredir='nonredirects',
 282                                         generator=False):
 283             if expagename == apppagename:
 284                 noclobber = True
 285         # Another reason not to make the redirect page is if the app name
 286         # is the same as it's ID, because that will overwrite the real page
 287         # with an redirect to itself! (Although it seems like an odd
 288         # scenario this happens a lot, e.g. where there is metadata but no
 289         # builds or binaries to extract a name from.
 290         if apppagename == pagename:
 291             noclobber = True
 292         if not noclobber:
 293             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 294
 295     for tcat, genp in [(wikicat, generated_pages),
 296                        (wikiredircat, generated_redirects)]:
 297         catpages = site.Pages['Category:' + tcat]
 298         existingpages = []
 299         for page in catpages:
 300             existingpages.append(page.name)
 301             if page.name in genp:
 302                 pagetxt = page.text()
 303                 if pagetxt != genp[page.name]:
 304                     logging.debug("Updating modified page " + page.name)
 305                     page.save(genp[page.name], summary='Auto-updated')
 306                 else:
 307                     logging.debug("Page " + page.name + " is unchanged")
 308             else:
 309                 logging.warn("Deleting page " + page.name)
 310                 page.delete('No longer published')
 311         for pagename, text in genp.items():
 312             logging.debug("Checking " + pagename)
 313             if pagename not in existingpages:
 314                 logging.debug("Creating page " + pagename)
 315                 try:
 316                     newpage = site.Pages[pagename]
 317                     newpage.save(text, summary='Auto-created')
 318                 except Exception as e:
 319                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 320
 321     # Purge server cache to ensure counts are up to date
 322     site.Pages['Repository Maintenance'].purge()
 323
 324     # Write a page with the last build log for this version code
 325     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 326     newpage = site.Pages[wiki_page_path]
 327     txt = ''
 328     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 329     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 330     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 331     txt += common.get_git_describe_link()
 332     txt += "\n\n"
 333     txt += common.get_android_tools_version_log()
 334     newpage.save(txt, summary='Run log')
 335     newpage = site.Pages['update']
 336     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 337
 338
 339 def delete_disabled_builds(apps, apkcache, repodirs):
 340     """Delete disabled build outputs.
 341
 342     :param apps: list of all applications, as per metadata.read_metadata
 343     :param apkcache: current apk cache information
 344     :param repodirs: the repo directories to process
 345     """
 346     for appid, app in apps.items():
 347         for build in app['builds']:
 348             if not build.disable:
 349                 continue
 350             apkfilename = common.get_release_filename(app, build)
 351             iconfilename = "%s.%s.png" % (
 352                 appid,
 353                 build.versionCode)
 354             for repodir in repodirs:
 355                 files = [
 356                     os.path.join(repodir, apkfilename),
 357                     os.path.join(repodir, apkfilename + '.asc'),
 358                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 359                 ]
 360                 for density in all_screen_densities:
 361                     repo_dir = get_icon_dir(repodir, density)
 362                     files.append(os.path.join(repo_dir, iconfilename))
 363
 364                 for f in files:
 365                     if os.path.exists(f):
 366                         logging.info("Deleting disabled build output " + f)
 367                         os.remove(f)
 368             if apkfilename in apkcache:
 369                 del apkcache[apkfilename]
 370
 371
 372 def resize_icon(iconpath, density):
 373
 374     if not os.path.isfile(iconpath):
 375         return
 376
 377     fp = None
 378     try:
 379         fp = open(iconpath, 'rb')
 380         im = Image.open(fp)
 381         size = dpi_to_px(density)
 382
 383         if any(length > size for length in im.size):
 384             oldsize = im.size
 385             im.thumbnail((size, size), Image.ANTIALIAS)
 386             logging.debug("%s was too large at %s - new size is %s" % (
 387                 iconpath, oldsize, im.size))
 388             im.save(iconpath, "PNG", optimize=True,
 389                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 390
 391     except Exception as e:
 392         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 393
 394     finally:
 395         if fp:
 396             fp.close()
 397
 398
 399 def resize_all_icons(repodirs):
 400     """Resize all icons that exceed the max size
 401
 402     :param repodirs: the repo directories to process
 403     """
 404     for repodir in repodirs:
 405         for density in screen_densities:
 406             icon_dir = get_icon_dir(repodir, density)
 407             icon_glob = os.path.join(icon_dir, '*.png')
 408             for iconpath in glob.glob(icon_glob):
 409                 resize_icon(iconpath, density)
 410
 411
 412 def getsig(apkpath):
 413     """ Get the signing certificate of an apk. To get the same md5 has that
 414     Android gets, we encode the .RSA certificate in a specific format and pass
 415     it hex-encoded to the md5 digest algorithm.
 416
 417     :param apkpath: path to the apk
 418     :returns: A string containing the md5 of the signature of the apk or None
 419               if an error occurred.
 420     """
 421
 422     with zipfile.ZipFile(apkpath, 'r') as apk:
 423         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 424
 425         if len(certs) < 1:
 426             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 427             return None
 428         if len(certs) > 1:
 429             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 430             return None
 431
 432         cert = apk.read(certs[0])
 433
 434     cert_encoded = common.get_certificate(cert)
 435
 436     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 437
 438
 439 def get_cache_file():
 440     return os.path.join('tmp', 'apkcache')
 441
 442
 443 def get_cache():
 444     """Get the cached dict of the APK index
 445
 446     Gather information about all the apk files in the repo directory,
 447     using cached data if possible. Some of the index operations take a
 448     long time, like calculating the SHA-256 and verifying the APK
 449     signature.
 450
 451     The cache is invalidated if the metadata version is different, or
 452     the 'allow_disabled_algorithms' config/option is different.  In
 453     those cases, there is no easy way to know what has changed from
 454     the cache, so just rerun the whole thing.
 455
 456     :return: apkcache
 457
 458     """
 459     apkcachefile = get_cache_file()
 460     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 461     if not options.clean and os.path.exists(apkcachefile):
 462         with open(apkcachefile, 'rb') as cf:
 463             apkcache = pickle.load(cf, encoding='utf-8')
 464         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 465            or apkcache.get('allow_disabled_algorithms') != ada:
 466             apkcache = {}
 467     else:
 468         apkcache = {}
 469
 470     apkcache["METADATA_VERSION"] = METADATA_VERSION
 471     apkcache['allow_disabled_algorithms'] = ada
 472
 473     return apkcache
 474
 475
 476 def write_cache(apkcache):
 477     apkcachefile = get_cache_file()
 478     cache_path = os.path.dirname(apkcachefile)
 479     if not os.path.exists(cache_path):
 480         os.makedirs(cache_path)
 481     with open(apkcachefile, 'wb') as cf:
 482         pickle.dump(apkcache, cf)
 483
 484
 485 def get_icon_bytes(apkzip, iconsrc):
 486     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 487     try:
 488         return apkzip.read(iconsrc)
 489     except KeyError:
 490         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 491
 492
 493 def sha256sum(filename):
 494     '''Calculate the sha256 of the given file'''
 495     sha = hashlib.sha256()
 496     with open(filename, 'rb') as f:
 497         while True:
 498             t = f.read(16384)
 499             if len(t) == 0:
 500                 break
 501             sha.update(t)
 502     return sha.hexdigest()
 503
 504
 505 def has_known_vulnerability(filename):
 506     """checks for known vulnerabilities in the APK
 507
 508     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 509     version.  Google also enforces this:
 510     https://support.google.com/faqs/answer/6376725?hl=en
 511
 512     Checks whether there are more than one classes.dex or AndroidManifest.xml
 513     files, which is invalid and an essential part of the "Master Key" attack.
 514     http://www.saurik.com/id/17
 515
 516     Janus is similar to Master Key but is perhaps easier to scan for.
 517     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 518     """
 519
 520     found_vuln = False
 521
 522     # statically load this pattern
 523     if not hasattr(has_known_vulnerability, "pattern"):
 524         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 525
 526     with open(filename.encode(), 'rb') as fp:
 527         first4 = fp.read(4)
 528     if first4 != b'\x50\x4b\x03\x04':
 529         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 530                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 531                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 532
 533     files_in_apk = set()
 534     with zipfile.ZipFile(filename) as zf:
 535         for name in zf.namelist():
 536             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 537                 lib = zf.open(name)
 538                 while True:
 539                     chunk = lib.read(4096)
 540                     if chunk == b'':
 541                         break
 542                     m = has_known_vulnerability.pattern.search(chunk)
 543                     if m:
 544                         version = m.group(1).decode('ascii')
 545                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 546                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 547                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 548                             logging.debug(_('"{path}" contains recent {name} ({version})')
 549                                           .format(path=filename, name=name, version=version))
 550                         else:
 551                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 552                                             .format(path=filename, name=name, version=version))
 553                             found_vuln = True
 554                         break
 555             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 556                 if name in files_in_apk:
 557                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 558                                     .format(apkfilename=filename, name=name))
 559                     found_vuln = True
 560                 files_in_apk.add(name)
 561     return found_vuln
 562
 563
 564 def insert_obbs(repodir, apps, apks):
 565     """Scans the .obb files in a given repo directory and adds them to the
 566     relevant APK instances.  OBB files have versionCodes like APK
 567     files, and they are loosely associated.  If there is an OBB file
 568     present, then any APK with the same or higher versionCode will use
 569     that OBB file.  There are two OBB types: main and patch, each APK
 570     can only have only have one of each.
 571
 572     https://developer.android.com/google/play/expansion-files.html
 573
 574     :param repodir: repo directory to scan
 575     :param apps: list of current, valid apps
 576     :param apks: current information on all APKs
 577
 578     """
 579
 580     def obbWarnDelete(f, msg):
 581         logging.warning(msg + ' ' + f)
 582         if options.delete_unknown:
 583             logging.error(_("Deleting unknown file: {path}").format(path=f))
 584             os.remove(f)
 585
 586     obbs = []
 587     java_Integer_MIN_VALUE = -pow(2, 31)
 588     currentPackageNames = apps.keys()
 589     for f in glob.glob(os.path.join(repodir, '*.obb')):
 590         obbfile = os.path.basename(f)
 591         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 592         chunks = obbfile.split('.')
 593         if chunks[0] != 'main' and chunks[0] != 'patch':
 594             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 595             continue
 596         if not re.match(r'^-?[0-9]+$', chunks[1]):
 597             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 598                           .format(name=chunks[0]))
 599             continue
 600         versionCode = int(chunks[1])
 601         packagename = ".".join(chunks[2:-1])
 602
 603         highestVersionCode = java_Integer_MIN_VALUE
 604         if packagename not in currentPackageNames:
 605             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 606             continue
 607         for apk in apks:
 608             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 609                 highestVersionCode = apk['versionCode']
 610         if versionCode > highestVersionCode:
 611             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 612                           .format(integer=str(versionCode)))
 613             continue
 614         obbsha256 = sha256sum(f)
 615         obbs.append((packagename, versionCode, obbfile, obbsha256))
 616
 617     for apk in apks:
 618         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 619             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 620                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 621                     apk['obbMainFile'] = obbfile
 622                     apk['obbMainFileSha256'] = obbsha256
 623                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 624                     apk['obbPatchFile'] = obbfile
 625                     apk['obbPatchFileSha256'] = obbsha256
 626             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 627                 break
 628
 629
 630 def translate_per_build_anti_features(apps, apks):
 631     """Grab the anti-features list from the build metadata
 632
 633     For most Anti-Features, they are really most applicable per-APK,
 634     not for an app.  An app can fix a vulnerability, add/remove
 635     tracking, etc.  This reads the 'antifeatures' list from the Build
 636     entries in the fdroiddata metadata file, then transforms it into
 637     the 'antiFeatures' list of unique items for the index.
 638
 639     The field key is all lower case in the metadata file to match the
 640     rest of the Build fields.  It is 'antiFeatures' camel case in the
 641     implementation, index, and fdroidclient since it is translated
 642     from the build 'antifeatures' field, not directly included.
 643
 644     """
 645
 646     antiFeatures = dict()
 647     for packageName, app in apps.items():
 648         d = dict()
 649         for build in app['builds']:
 650             afl = build.get('antifeatures')
 651             if afl:
 652                 d[int(build.versionCode)] = afl
 653         if len(d) > 0:
 654             antiFeatures[packageName] = d
 655
 656     for apk in apks:
 657         d = antiFeatures.get(apk['packageName'])
 658         if d:
 659             afl = d.get(apk['versionCode'])
 660             if afl:
 661                 apk['antiFeatures'].update(afl)
 662
 663
 664 def _get_localized_dict(app, locale):
 665     '''get the dict to add localized store metadata to'''
 666     if 'localized' not in app:
 667         app['localized'] = collections.OrderedDict()
 668     if locale not in app['localized']:
 669         app['localized'][locale] = collections.OrderedDict()
 670     return app['localized'][locale]
 671
 672
 673 def _set_localized_text_entry(app, locale, key, f):
 674     limit = config['char_limits'][key]
 675     localized = _get_localized_dict(app, locale)
 676     with open(f) as fp:
 677         text = fp.read()[:limit]
 678         if len(text) > 0:
 679             localized[key] = text
 680
 681
 682 def _set_author_entry(app, key, f):
 683     limit = config['char_limits']['author']
 684     with open(f) as fp:
 685         text = fp.read()[:limit]
 686         if len(text) > 0:
 687             app[key] = text
 688
 689
 690 def _strip_and_copy_image(inpath, outpath):
 691     """Remove any metadata from image and copy it to new path
 692
 693     Sadly, image metadata like EXIF can be used to exploit devices.
 694     It is not used at all in the F-Droid ecosystem, so its much safer
 695     just to remove it entirely.
 696
 697     """
 698
 699     extension = common.get_extension(inpath)[1]
 700     if os.path.isdir(outpath):
 701         outpath = os.path.join(outpath, os.path.basename(inpath))
 702     if extension == 'png':
 703         with open(inpath, 'rb') as fp:
 704             in_image = Image.open(fp)
 705             in_image.save(outpath, "PNG", optimize=True,
 706                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 707     elif extension == 'jpg' or extension == 'jpeg':
 708         with open(inpath, 'rb') as fp:
 709             in_image = Image.open(fp)
 710             data = list(in_image.getdata())
 711             out_image = Image.new(in_image.mode, in_image.size)
 712         out_image.putdata(data)
 713         out_image.save(outpath, "JPEG", optimize=True)
 714     else:
 715         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 716                               .format(extension=extension))
 717
 718
 719 def copy_triple_t_store_metadata(apps):
 720     """Include store metadata from the app's source repo
 721
 722     The Triple-T Gradle Play Publisher is a plugin that has a standard
 723     file layout for all of the metadata and graphics that the Google
 724     Play Store accepts.  Since F-Droid has the git repo, it can just
 725     pluck those files directly.  This method reads any text files into
 726     the app dict, then copies any graphics into the fdroid repo
 727     directory structure.
 728
 729     This needs to be run before insert_localized_app_metadata() so that
 730     the graphics files that are copied into the fdroid repo get
 731     properly indexed.
 732
 733     https://github.com/Triple-T/gradle-play-publisher#upload-images
 734     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 735
 736     """
 737
 738     if not os.path.isdir('build'):
 739         return  # nothing to do
 740
 741     for packageName, app in apps.items():
 742         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 743             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 744             for root, dirs, files in os.walk(d):
 745                 segments = root.split('/')
 746                 locale = segments[-2]
 747                 for f in files:
 748                     if f == 'fulldescription':
 749                         _set_localized_text_entry(app, locale, 'description',
 750                                                   os.path.join(root, f))
 751                         continue
 752                     elif f == 'shortdescription':
 753                         _set_localized_text_entry(app, locale, 'summary',
 754                                                   os.path.join(root, f))
 755                         continue
 756                     elif f == 'title':
 757                         _set_localized_text_entry(app, locale, 'name',
 758                                                   os.path.join(root, f))
 759                         continue
 760                     elif f == 'video':
 761                         _set_localized_text_entry(app, locale, 'video',
 762                                                   os.path.join(root, f))
 763                         continue
 764                     elif f == 'whatsnew':
 765                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 766                                                   os.path.join(root, f))
 767                         continue
 768                     elif f == 'contactEmail':
 769                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 770                         continue
 771                     elif f == 'contactPhone':
 772                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 773                         continue
 774                     elif f == 'contactWebsite':
 775                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 776                         continue
 777
 778                     base, extension = common.get_extension(f)
 779                     dirname = os.path.basename(root)
 780                     if extension in ALLOWED_EXTENSIONS \
 781                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 782                         if segments[-2] == 'listing':
 783                             locale = segments[-3]
 784                         else:
 785                             locale = segments[-2]
 786                         destdir = os.path.join('repo', packageName, locale, dirname)
 787                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 788                         sourcefile = os.path.join(root, f)
 789                         destfile = os.path.join(destdir, os.path.basename(f))
 790                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 791                         _strip_and_copy_image(sourcefile, destfile)
 792
 793
 794 def insert_localized_app_metadata(apps):
 795     """scans standard locations for graphics and localized text
 796
 797     Scans for localized description files, store graphics, and
 798     screenshot PNG files in statically defined screenshots directory
 799     and adds them to the app metadata.  The screenshots and graphic
 800     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 801     and must be in the following layout:
 802     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 803
 804     repo/packageName/locale/featureGraphic.png
 805     repo/packageName/locale/phoneScreenshots/1.png
 806     repo/packageName/locale/phoneScreenshots/2.png
 807
 808     The changelog files must be text files named with the versionCode
 809     ending with ".txt" and must be in the following layout:
 810     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 811
 812     repo/packageName/locale/changelogs/12345.txt
 813
 814     This will scan the each app's source repo then the metadata/ dir
 815     for these standard locations of changelog files.  If it finds
 816     them, they will be added to the dict of all packages, with the
 817     versions in the metadata/ folder taking precendence over the what
 818     is in the app's source repo.
 819
 820     Where "packageName" is the app's packageName and "locale" is the locale
 821     of the graphics, e.g. what language they are in, using the IETF RFC5646
 822     format (en-US, fr-CA, es-MX, etc).
 823
 824     This will also scan the app's git for a fastlane folder, and the
 825     metadata/ folder and the apps' source repos for standard locations
 826     of graphic and screenshot files.  If it finds them, it will copy
 827     them into the repo.  The fastlane files follow this pattern:
 828     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 829
 830     """
 831
 832     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 833     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 834     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 836
 837     for srcd in sorted(sourcedirs):
 838         if not os.path.isdir(srcd):
 839             continue
 840         for root, dirs, files in os.walk(srcd):
 841             segments = root.split('/')
 842             packageName = segments[1]
 843             if packageName not in apps:
 844                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 845                 continue
 846             locale = segments[-1]
 847             destdir = os.path.join('repo', packageName, locale)
 848
 849             # flavours specified in build receipt
 850             build_flavours = ""
 851             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 852                     and 'gradle' in apps[packageName].builds[-1]:
 853                 build_flavours = apps[packageName].builds[-1].gradle
 854
 855             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 856                 logging.debug("ignoring due to wrong flavour")
 857                 continue
 858
 859             for f in files:
 860                 if f in ('description.txt', 'full_description.txt'):
 861                     _set_localized_text_entry(apps[packageName], locale, 'description',
 862                                               os.path.join(root, f))
 863                     continue
 864                 elif f in ('summary.txt', 'short_description.txt'):
 865                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 866                                               os.path.join(root, f))
 867                     continue
 868                 elif f in ('name.txt', 'title.txt'):
 869                     _set_localized_text_entry(apps[packageName], locale, 'name',
 870                                               os.path.join(root, f))
 871                     continue
 872                 elif f == 'video.txt':
 873                     _set_localized_text_entry(apps[packageName], locale, 'video',
 874                                               os.path.join(root, f))
 875                     continue
 876                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 877                     locale = segments[-2]
 878                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 879                                               os.path.join(root, f))
 880                     continue
 881
 882                 base, extension = common.get_extension(f)
 883                 if locale == 'images':
 884                     locale = segments[-2]
 885                     destdir = os.path.join('repo', packageName, locale)
 886                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 887                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 888                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 889                     _strip_and_copy_image(os.path.join(root, f), destdir)
 890             for d in dirs:
 891                 if d in SCREENSHOT_DIRS:
 892                     if locale == 'images':
 893                         locale = segments[-2]
 894                         destdir = os.path.join('repo', packageName, locale)
 895                     for f in glob.glob(os.path.join(root, d, '*.*')):
 896                         _ignored, extension = common.get_extension(f)
 897                         if extension in ALLOWED_EXTENSIONS:
 898                             screenshotdestdir = os.path.join(destdir, d)
 899                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 900                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 901                             _strip_and_copy_image(f, screenshotdestdir)
 902
 903     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 904     for d in repofiles:
 905         if not os.path.isdir(d):
 906             continue
 907         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 908             if not os.path.isfile(f):
 909                 continue
 910             segments = f.split('/')
 911             packageName = segments[1]
 912             locale = segments[2]
 913             screenshotdir = segments[3]
 914             filename = os.path.basename(f)
 915             base, extension = common.get_extension(filename)
 916
 917             if packageName not in apps:
 918                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 919                                 .format(path=filename, name=packageName))
 920                 continue
 921             graphics = _get_localized_dict(apps[packageName], locale)
 922
 923             if extension not in ALLOWED_EXTENSIONS:
 924                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 925             elif base in GRAPHIC_NAMES:
 926                 # there can only be zero or one of these per locale
 927                 graphics[base] = filename
 928             elif screenshotdir in SCREENSHOT_DIRS:
 929                 # there can any number of these per locale
 930                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 931                 if screenshotdir not in graphics:
 932                     graphics[screenshotdir] = []
 933                 graphics[screenshotdir].append(filename)
 934             else:
 935                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 936
 937
 938 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 939     """Scan a repo for all files with an extension except APK/OBB
 940
 941     :param apkcache: current cached info about all repo files
 942     :param repodir: repo directory to scan
 943     :param knownapks: list of all known files, as per metadata.read_metadata
 944     :param use_date_from_file: use date from file (instead of current date)
 945                                for newly added files
 946     """
 947
 948     cachechanged = False
 949     repo_files = []
 950     repodir = repodir.encode('utf-8')
 951     for name in os.listdir(repodir):
 952         file_extension = common.get_file_extension(name)
 953         if file_extension == 'apk' or file_extension == 'obb':
 954             continue
 955         filename = os.path.join(repodir, name)
 956         name_utf8 = name.decode('utf-8')
 957         if filename.endswith(b'_src.tar.gz'):
 958             logging.debug(_('skipping source tarball: {path}')
 959                           .format(path=filename.decode('utf-8')))
 960             continue
 961         if not common.is_repo_file(filename):
 962             continue
 963         stat = os.stat(filename)
 964         if stat.st_size == 0:
 965             raise FDroidException(_('{path} is zero size!')
 966                                   .format(path=filename))
 967
 968         shasum = sha256sum(filename)
 969         usecache = False
 970         if name in apkcache:
 971             repo_file = apkcache[name]
 972             # added time is cached as tuple but used here as datetime instance
 973             if 'added' in repo_file:
 974                 a = repo_file['added']
 975                 if isinstance(a, datetime):
 976                     repo_file['added'] = a
 977                 else:
 978                     repo_file['added'] = datetime(*a[:6])
 979             if repo_file.get('hash') == shasum:
 980                 logging.debug(_("Reading {apkfilename} from cache")
 981                               .format(apkfilename=name_utf8))
 982                 usecache = True
 983             else:
 984                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 985                               .format(apkfilename=name_utf8))
 986
 987         if not usecache:
 988             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 989             repo_file = collections.OrderedDict()
 990             repo_file['name'] = os.path.splitext(name_utf8)[0]
 991             # TODO rename apkname globally to something more generic
 992             repo_file['apkName'] = name_utf8
 993             repo_file['hash'] = shasum
 994             repo_file['hashType'] = 'sha256'
 995             repo_file['versionCode'] = 0
 996             repo_file['versionName'] = shasum[0:7]
 997             # the static ID is the SHA256 unless it is set in the metadata
 998             repo_file['packageName'] = shasum
 999
1000             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1001             if m:
1002                 repo_file['packageName'] = m.group(1)
1003                 repo_file['versionCode'] = int(m.group(2))
1004             srcfilename = name + b'_src.tar.gz'
1005             if os.path.exists(os.path.join(repodir, srcfilename)):
1006                 repo_file['srcname'] = srcfilename.decode('utf-8')
1007             repo_file['size'] = stat.st_size
1008
1009             apkcache[name] = repo_file
1010             cachechanged = True
1011
1012         if use_date_from_file:
1013             timestamp = stat.st_ctime
1014             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1015         else:
1016             default_date_param = None
1017
1018         # Record in knownapks, getting the added date at the same time..
1019         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1020                                     default_date=default_date_param)
1021         if added:
1022             repo_file['added'] = added
1023
1024         repo_files.append(repo_file)
1025
1026     return repo_files, cachechanged
1027
1028
1029 def scan_apk(apk_file):
1030     """
1031     Scans an APK file and returns dictionary with metadata of the APK.
1032
1033     Attention: This does *not* verify that the APK signature is correct.
1034
1035     :param apk_file: The (ideally absolute) path to the APK file
1036     :raises BuildException
1037     :return A dict containing APK metadata
1038     """
1039     apk = {
1040         'hash': sha256sum(apk_file),
1041         'hashType': 'sha256',
1042         'uses-permission': [],
1043         'uses-permission-sdk-23': [],
1044         'features': [],
1045         'icons_src': {},
1046         'icons': {},
1047         'antiFeatures': set(),
1048     }
1049
1050     if common.use_androguard():
1051         scan_apk_androguard(apk, apk_file)
1052     else:
1053         scan_apk_aapt(apk, apk_file)
1054
1055     # Get the signature, or rather the signing key fingerprints
1056     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1057     apk['sig'] = getsig(apk_file)
1058     if not apk['sig']:
1059         raise BuildException("Failed to get apk signature")
1060     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1061                                                                apk_file))
1062     if not apk.get('signer'):
1063         raise BuildException("Failed to get apk signing key fingerprint")
1064
1065     # Get size of the APK
1066     apk['size'] = os.path.getsize(apk_file)
1067
1068     if 'minSdkVersion' not in apk:
1069         logging.warning("No SDK version information found in {0}".format(apk_file))
1070         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
1071     if 'targetSdkVersion' not in apk:
1072         apk['targetSdkVersion'] = apk['minSdkVersion']
1073
1074     # Check for known vulnerabilities
1075     if has_known_vulnerability(apk_file):
1076         apk['antiFeatures'].add('KnownVuln')
1077
1078     return apk
1079
1080
1081 def _get_apk_icons_src(apkfile, icon_name):
1082     """Extract the paths to the app icon in all available densities
1083
1084     """
1085     icons_src = dict()
1086     density_re = re.compile('^res/(.*)/{}\.(png|xml)$'.format(icon_name))
1087     with zipfile.ZipFile(apkfile) as zf:
1088         for filename in zf.namelist():
1089             m = density_re.match(filename)
1090             if m:
1091                 folder = m.group(1).split('-')
1092                 if len(folder) > 1 and folder[1].endswith('dpi'):
1093                     density = screen_resolutions[folder[1]]
1094                 else:
1095                     density = '160'
1096                 icons_src[density] = m.group(0)
1097     if icons_src.get('-1') is None and '160' in icons_src:
1098         icons_src['-1'] = icons_src['160']
1099     return icons_src
1100
1101
1102 def scan_apk_aapt(apk, apkfile):
1103     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1104     if p.returncode != 0:
1105         if options.delete_unknown:
1106             if os.path.exists(apkfile):
1107                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1108                 os.remove(apkfile)
1109             else:
1110                 logging.error("Could not find {0} to remove it".format(apkfile))
1111         else:
1112             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1113         raise BuildException(_("Invalid APK"))
1114     icon_name = None
1115     for line in p.output.splitlines():
1116         if line.startswith("package:"):
1117             try:
1118                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1119                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1120                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1121             except Exception as e:
1122                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1123         elif line.startswith("application:"):
1124             m = re.match(APK_LABEL_ICON_PAT, line)
1125             if m:
1126                 apk['name'] = m.group(1)
1127                 icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
1128         elif not apk.get('name') and line.startswith("launchable-activity:"):
1129             # Only use launchable-activity as fallback to application
1130             apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)
1131         elif line.startswith("sdkVersion:"):
1132             m = re.match(APK_SDK_VERSION_PAT, line)
1133             if m is None:
1134                 logging.error(line.replace('sdkVersion:', '')
1135                               + ' is not a valid minSdkVersion!')
1136             else:
1137                 apk['minSdkVersion'] = m.group(1)
1138         elif line.startswith("targetSdkVersion:"):
1139             m = re.match(APK_SDK_VERSION_PAT, line)
1140             if m is None:
1141                 logging.error(line.replace('targetSdkVersion:', '')
1142                               + ' is not a valid targetSdkVersion!')
1143             else:
1144                 apk['targetSdkVersion'] = m.group(1)
1145         elif line.startswith("maxSdkVersion:"):
1146             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1147         elif line.startswith("native-code:"):
1148             apk['nativecode'] = []
1149             for arch in line[13:].split(' '):
1150                 apk['nativecode'].append(arch[1:-1])
1151         elif line.startswith('uses-permission:'):
1152             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1153             if perm_match['maxSdkVersion']:
1154                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1155             permission = UsesPermission(
1156                 perm_match['name'],
1157                 perm_match['maxSdkVersion']
1158             )
1159
1160             apk['uses-permission'].append(permission)
1161         elif line.startswith('uses-permission-sdk-23:'):
1162             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1163             if perm_match['maxSdkVersion']:
1164                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1165             permission_sdk_23 = UsesPermissionSdk23(
1166                 perm_match['name'],
1167                 perm_match['maxSdkVersion']
1168             )
1169
1170             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1171
1172         elif line.startswith('uses-feature:'):
1173             feature = re.match(APK_FEATURE_PAT, line).group(1)
1174             # Filter out this, it's only added with the latest SDK tools and
1175             # causes problems for lots of apps.
1176             if feature != "android.hardware.screen.portrait" \
1177                     and feature != "android.hardware.screen.landscape":
1178                 if feature.startswith("android.feature."):
1179                     feature = feature[16:]
1180                 apk['features'].add(feature)
1181     apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1182
1183
1184 def _ensure_final_value(packageName, arsc, value):
1185     """Ensure incoming value is always the value, not the resid
1186
1187     androguard will sometimes return the Android "resId" aka
1188     Resource ID instead of the actual value.  This checks whether
1189     the value is actually a resId, then performs the Android
1190     Resource lookup as needed.
1191
1192     """
1193     if value:
1194         returnValue = value
1195         if value[0] == '@':
1196             try:  # can be a literal value or a resId
1197                 res_id = int(value.replace("@", "0x"), 16)
1198                 res_id = arsc.get_id(packageName, res_id)[1]
1199                 returnValue = arsc.get_string(packageName, res_id)[1]
1200             except ValueError:
1201                 pass
1202         return returnValue
1203
1204
1205 def _sanitize_sdk_version(value):
1206     """Sanitize the raw values from androguard to handle bad values
1207
1208     minSdkVersion/targetSdkVersion/maxSdkVersion must be integers,
1209     but that doesn't stop devs from doing strange things like
1210     setting them using Android XML strings.
1211
1212     https://gitlab.com/souch/SMSbypass/blob/v0.9/app/src/main/AndroidManifest.xml#L29
1213     https://gitlab.com/souch/SMSbypass/blob/v0.9/app/src/main/res/values/strings.xml#L27
1214     """
1215     try:
1216         sdk_version = int(value)
1217         if sdk_version > 0:
1218             return str(sdk_version)  # heinous, but this is still str in the codebase
1219     except (TypeError, ValueError):
1220         pass
1221     return None
1222
1223
1224 def scan_apk_androguard(apk, apkfile):
1225     try:
1226         from androguard.core.bytecodes.apk import APK
1227         apkobject = APK(apkfile)
1228         if apkobject.is_valid_APK():
1229             arsc = apkobject.get_android_resources()
1230         else:
1231             if options.delete_unknown:
1232                 if os.path.exists(apkfile):
1233                     logging.error(_("Failed to get apk information, deleting {path}")
1234                                   .format(path=apkfile))
1235                     os.remove(apkfile)
1236                 else:
1237                     logging.error(_("Could not find {path} to remove it")
1238                                   .format(path=apkfile))
1239             else:
1240                 logging.error(_("Failed to get apk information, skipping {path}")
1241                               .format(path=apkfile))
1242             raise BuildException(_("Invalid APK"))
1243     except ImportError:
1244         raise FDroidException("androguard library is not installed and aapt not present")
1245     except FileNotFoundError:
1246         logging.error(_("Could not open apk file for analysis"))
1247         raise BuildException(_("Invalid APK"))
1248
1249     apk['packageName'] = apkobject.get_package()
1250     apk['versionCode'] = int(apkobject.get_androidversion_code())
1251     apk['name'] = apkobject.get_app_name()
1252
1253     apk['versionName'] = _ensure_final_value(apk['packageName'], arsc,
1254                                              apkobject.get_androidversion_name())
1255
1256     minSdkVersion = _sanitize_sdk_version(apkobject.get_min_sdk_version())
1257     if minSdkVersion is not None:
1258         apk['minSdkVersion'] = minSdkVersion
1259
1260     targetSdkVersion = _sanitize_sdk_version(apkobject.get_target_sdk_version())
1261     if targetSdkVersion is not None:
1262         apk['targetSdkVersion'] = targetSdkVersion
1263
1264     maxSdkVersion = _sanitize_sdk_version(apkobject.get_max_sdk_version())
1265     if maxSdkVersion is not None:
1266         apk['maxSdkVersion'] = maxSdkVersion
1267
1268     icon_id_str = apkobject.get_element("application", "icon")
1269     if icon_id_str:
1270         icon_id = int(icon_id_str.replace("@", "0x"), 16)
1271         resource_id = arsc.get_id(apk['packageName'], icon_id)
1272         if resource_id:
1273             icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1274         else:
1275             icon_name = os.path.splitext(os.path.basename(apkobject.get_app_icon()))[0]
1276         apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1277
1278     arch_re = re.compile("^lib/(.*)/.*$")
1279     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1280     if len(arch) >= 1:
1281         apk['nativecode'] = []
1282         apk['nativecode'].extend(sorted(list(arch)))
1283
1284     xml = apkobject.get_android_manifest_xml()
1285     xmlns = xml.nsmap.get('android')
1286     if not xmlns:
1287         xmlns = 'http://schemas.android.com/apk/res/android'
1288
1289     for item in xml.findall('uses-permission'):
1290         name = str(item.attrib['{' + xmlns + '}name'])
1291         maxSdkVersion = item.attrib.get('{' + xmlns + '}maxSdkVersion')
1292         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1293         permission = UsesPermission(
1294             name,
1295             maxSdkVersion
1296         )
1297         apk['uses-permission'].append(permission)
1298     for name, maxSdkVersion in apkobject.get_uses_implied_permission_list():
1299         permission = UsesPermission(
1300             name,
1301             maxSdkVersion
1302         )
1303         apk['uses-permission'].append(permission)
1304
1305     for item in xml.findall('uses-permission-sdk-23'):
1306         name = str(item.attrib['{' + xmlns + '}name'])
1307         maxSdkVersion = item.attrib.get('{' + xmlns + '}maxSdkVersion')
1308         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1309         permission_sdk_23 = UsesPermissionSdk23(
1310             name,
1311             maxSdkVersion
1312         )
1313         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1314
1315     for item in xml.findall('uses-feature'):
1316         key = '{' + xmlns + '}name'
1317         if key not in item.attrib:
1318             continue
1319         feature = str(item.attrib[key])
1320         if feature != "android.hardware.screen.portrait" \
1321                 and feature != "android.hardware.screen.landscape":
1322             if feature.startswith("android.feature."):
1323                 feature = feature[16:]
1324         required = item.attrib.get('{' + xmlns + '}required')
1325         if required is None or required == 'true':
1326             apk['features'].append(feature)
1327
1328
1329 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1330                 allow_disabled_algorithms=False, archive_bad_sig=False):
1331     """Processes the apk with the given filename in the given repo directory.
1332
1333     This also extracts the icons.
1334
1335     :param apkcache: current apk cache information
1336     :param apkfilename: the filename of the apk to scan
1337     :param repodir: repo directory to scan
1338     :param knownapks: known apks info
1339     :param use_date_from_apk: use date from APK (instead of current date)
1340                               for newly added APKs
1341     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1342                                       disabled algorithms in the signature (e.g. MD5)
1343     :param archive_bad_sig: move APKs with a bad signature to the archive
1344     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1345      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1346     """
1347
1348     apk = {}
1349     apkfile = os.path.join(repodir, apkfilename)
1350
1351     cachechanged = False
1352     usecache = False
1353     if apkfilename in apkcache:
1354         apk = apkcache[apkfilename]
1355         if apk.get('hash') == sha256sum(apkfile):
1356             logging.debug(_("Reading {apkfilename} from cache")
1357                           .format(apkfilename=apkfilename))
1358             usecache = True
1359         else:
1360             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1361                           .format(apkfilename=apkfilename))
1362
1363     if not usecache:
1364         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1365
1366         try:
1367             apk = scan_apk(apkfile)
1368         except BuildException:
1369             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1370                             .format(apkfilename=apkfilename))
1371             return True, None, False
1372
1373         # Check for debuggable apks...
1374         if common.is_apk_and_debuggable(apkfile):
1375             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1376
1377         if options.rename_apks:
1378             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1379             std_short_name = os.path.join(repodir, n)
1380             if apkfile != std_short_name:
1381                 if os.path.exists(std_short_name):
1382                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1383                     if apkfile != std_long_name:
1384                         if os.path.exists(std_long_name):
1385                             dupdir = os.path.join('duplicates', repodir)
1386                             if not os.path.isdir(dupdir):
1387                                 os.makedirs(dupdir, exist_ok=True)
1388                             dupfile = os.path.join('duplicates', std_long_name)
1389                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1390                             os.rename(apkfile, dupfile)
1391                             return True, None, False
1392                         else:
1393                             os.rename(apkfile, std_long_name)
1394                     apkfile = std_long_name
1395                 else:
1396                     os.rename(apkfile, std_short_name)
1397                     apkfile = std_short_name
1398                 apkfilename = apkfile[len(repodir) + 1:]
1399
1400         apk['apkName'] = apkfilename
1401         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1402         if os.path.exists(os.path.join(repodir, srcfilename)):
1403             apk['srcname'] = srcfilename
1404
1405         # verify the jar signature is correct, allow deprecated
1406         # algorithms only if the APK is in the archive.
1407         skipapk = False
1408         if not common.verify_apk_signature(apkfile):
1409             if repodir == 'archive' or allow_disabled_algorithms:
1410                 if common.verify_old_apk_signature(apkfile):
1411                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1412                 else:
1413                     skipapk = True
1414             else:
1415                 skipapk = True
1416
1417         if skipapk:
1418             if archive_bad_sig:
1419                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1420                                 .format(apkfilename=apkfilename))
1421                 move_apk_between_sections(repodir, 'archive', apk)
1422             else:
1423                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1424                                 .format(apkfilename=apkfilename))
1425             return True, None, False
1426
1427         apkzip = zipfile.ZipFile(apkfile, 'r')
1428
1429         manifest = apkzip.getinfo('AndroidManifest.xml')
1430         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1431         if (1980, 0, 0) != manifest.date_time[0:3]:
1432             try:
1433                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1434             except ValueError as e:
1435                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1436                                 .format(apkfilename=apkfile) + str(e))
1437
1438         # extract icons from APK zip file
1439         iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
1440         try:
1441             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1442         finally:
1443             apkzip.close()  # ensure that APK zip file gets closed
1444
1445         # resize existing icons for densities missing in the APK
1446         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1447
1448         if use_date_from_apk and manifest.date_time[1] != 0:
1449             default_date_param = datetime(*manifest.date_time)
1450         else:
1451             default_date_param = None
1452
1453         # Record in known apks, getting the added date at the same time..
1454         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1455                                     default_date=default_date_param)
1456         if added:
1457             apk['added'] = added
1458
1459         apkcache[apkfilename] = apk
1460         cachechanged = True
1461
1462     return False, apk, cachechanged
1463
1464
1465 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1466     """Processes the apks in the given repo directory.
1467
1468     This also extracts the icons.
1469
1470     :param apkcache: current apk cache information
1471     :param repodir: repo directory to scan
1472     :param knownapks: known apks info
1473     :param use_date_from_apk: use date from APK (instead of current date)
1474                               for newly added APKs
1475     :returns: (apks, cachechanged) where apks is a list of apk information,
1476               and cachechanged is True if the apkcache got changed.
1477     """
1478
1479     cachechanged = False
1480
1481     for icon_dir in get_all_icon_dirs(repodir):
1482         if os.path.exists(icon_dir):
1483             if options.clean:
1484                 shutil.rmtree(icon_dir)
1485                 os.makedirs(icon_dir)
1486         else:
1487             os.makedirs(icon_dir)
1488
1489     apks = []
1490     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1491         apkfilename = apkfile[len(repodir) + 1:]
1492         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1493         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1494                                              use_date_from_apk, ada, True)
1495         if skip:
1496             continue
1497         apks.append(apk)
1498         cachechanged = cachechanged or cachethis
1499
1500     return apks, cachechanged
1501
1502
1503 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1504     """Extracts PNG icons from an APK with the supported pixel densities
1505
1506     Extracts icons from the given APK zip in various densities, saves
1507     them into given repo directory and stores their names in the APK
1508     metadata dictionary.  If the icon is an XML icon, then this tries
1509     to find PNG icon that can replace it.
1510
1511     :param icon_filename: A string representing the icon's file name
1512     :param apk: A populated dictionary containing APK metadata.
1513                 Needs to have 'icons_src' key
1514     :param apkzip: An opened zipfile.ZipFile of the APK file
1515     :param repo_dir: The directory of the APK's repository
1516     :return: A list of icon densities that are missing
1517
1518     """
1519     res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1520     pngs = dict()
1521     for f in apkzip.namelist():
1522         m = res_name_re.match(f)
1523         if m and m.group(4) == 'png':
1524             density = screen_resolutions[m.group(2)]
1525             pngs[m.group(3) + '/' + density] = m.group(0)
1526
1527     icon_type = None
1528     empty_densities = []
1529     for density in screen_densities:
1530         if density not in apk['icons_src']:
1531             empty_densities.append(density)
1532             continue
1533         icon_src = apk['icons_src'][density]
1534         icon_dir = get_icon_dir(repo_dir, density)
1535         icon_type = '.png'
1536
1537         # Extract the icon files per density
1538         if icon_src.endswith('.xml'):
1539             m = res_name_re.match(icon_src)
1540             if m:
1541                 name = pngs.get(m.group(3) + '/' + str(density))
1542                 if name:
1543                     icon_src = name
1544             if icon_src.endswith('.xml'):
1545                 empty_densities.append(density)
1546                 icon_type = '.xml'
1547         icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
1548
1549         try:
1550             with open(icon_dest, 'wb') as f:
1551                 f.write(get_icon_bytes(apkzip, icon_src))
1552             apk['icons'][density] = icon_filename + icon_type
1553         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1554             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1555             del apk['icons_src'][density]
1556             empty_densities.append(density)
1557
1558     # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
1559     if '-1' in apk['icons_src']:
1560         icon_src = apk['icons_src']['-1']
1561         icon_type = icon_src[-4:]
1562         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
1563         with open(icon_path, 'wb') as f:
1564             f.write(get_icon_bytes(apkzip, icon_src))
1565         if icon_type == '.png':
1566             im = None
1567             try:
1568                 im = Image.open(icon_path)
1569                 dpi = px_to_dpi(im.size[0])
1570                 for density in screen_densities:
1571                     if density in apk['icons']:
1572                         break
1573                     if density == screen_densities[-1] or dpi >= int(density):
1574                         apk['icons'][density] = icon_filename + icon_type
1575                         shutil.move(icon_path,
1576                                     os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
1577                         empty_densities.remove(density)
1578                         break
1579             except Exception as e:
1580                 logging.warning(_("Failed reading {path}: {error}")
1581                                 .format(path=icon_path, error=e))
1582             finally:
1583                 if im and hasattr(im, 'close'):
1584                     im.close()
1585
1586     if apk['icons']:
1587         apk['icon'] = icon_filename + icon_type
1588
1589     return empty_densities
1590
1591
1592 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1593     """
1594     Resize existing PNG icons for densities missing in the APK to ensure all densities are available
1595
1596     :param empty_densities: A list of icon densities that are missing
1597     :param icon_filename: A string representing the icon's file name
1598     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1599     :param repo_dir: The directory of the APK's repository
1600
1601     """
1602     icon_filename += '.png'
1603     # First try resizing down to not lose quality
1604     last_density = None
1605     for density in screen_densities:
1606         if density == '65534':  # not possible to generate 'anydpi' from other densities
1607             continue
1608         if density not in empty_densities:
1609             last_density = density
1610             continue
1611         if last_density is None:
1612             continue
1613         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1614
1615         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1616         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1617         fp = None
1618         try:
1619             fp = open(last_icon_path, 'rb')
1620             im = Image.open(fp)
1621
1622             size = dpi_to_px(density)
1623
1624             im.thumbnail((size, size), Image.ANTIALIAS)
1625             im.save(icon_path, "PNG", optimize=True,
1626                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1627             empty_densities.remove(density)
1628         except Exception as e:
1629             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1630         finally:
1631             if fp:
1632                 fp.close()
1633
1634     # Then just copy from the highest resolution available
1635     last_density = None
1636     for density in reversed(screen_densities):
1637         if density not in empty_densities:
1638             last_density = density
1639             continue
1640
1641         if last_density is None:
1642             continue
1643
1644         shutil.copyfile(
1645             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1646             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1647         )
1648         empty_densities.remove(density)
1649
1650     for density in screen_densities:
1651         icon_dir = get_icon_dir(repo_dir, density)
1652         icon_dest = os.path.join(icon_dir, icon_filename)
1653         resize_icon(icon_dest, density)
1654
1655     # Copy from icons-mdpi to icons since mdpi is the baseline density
1656     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1657     if os.path.isfile(baseline):
1658         apk['icons']['0'] = icon_filename
1659         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1660
1661
1662 def apply_info_from_latest_apk(apps, apks):
1663     """
1664     Some information from the apks needs to be applied up to the application level.
1665     When doing this, we use the info from the most recent version's apk.
1666     We deal with figuring out when the app was added and last updated at the same time.
1667     """
1668     for appid, app in apps.items():
1669         bestver = UNSET_VERSION_CODE
1670         for apk in apks:
1671             if apk['packageName'] == appid:
1672                 if apk['versionCode'] > bestver:
1673                     bestver = apk['versionCode']
1674                     bestapk = apk
1675
1676                 if 'added' in apk:
1677                     if not app.added or apk['added'] < app.added:
1678                         app.added = apk['added']
1679                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1680                         app.lastUpdated = apk['added']
1681
1682         if not app.added:
1683             logging.debug("Don't know when " + appid + " was added")
1684         if not app.lastUpdated:
1685             logging.debug("Don't know when " + appid + " was last updated")
1686
1687         if bestver == UNSET_VERSION_CODE:
1688
1689             if app.Name is None:
1690                 app.Name = app.AutoName or appid
1691             app.icon = None
1692             logging.debug("Application " + appid + " has no packages")
1693         else:
1694             if app.Name is None:
1695                 app.Name = bestapk['name']
1696             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1697             if app.CurrentVersionCode is None:
1698                 app.CurrentVersionCode = str(bestver)
1699
1700
1701 def make_categories_txt(repodir, categories):
1702     '''Write a category list in the repo to allow quick access'''
1703     catdata = ''
1704     for cat in sorted(categories):
1705         catdata += cat + '\n'
1706     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1707         f.write(catdata)
1708
1709
1710 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1711
1712     def filter_apk_list_sorted(apk_list):
1713         res = []
1714         for apk in apk_list:
1715             if apk['packageName'] == appid:
1716                 res.append(apk)
1717
1718         # Sort the apk list by version code. First is highest/newest.
1719         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1720
1721     for appid, app in apps.items():
1722
1723         if app.ArchivePolicy:
1724             keepversions = int(app.ArchivePolicy[:-9])
1725         else:
1726             keepversions = defaultkeepversions
1727
1728         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1729                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1730
1731         current_app_apks = filter_apk_list_sorted(apks)
1732         if len(current_app_apks) > keepversions:
1733             # Move back the ones we don't want.
1734             for apk in current_app_apks[keepversions:]:
1735                 move_apk_between_sections(repodir, archivedir, apk)
1736                 archapks.append(apk)
1737                 apks.remove(apk)
1738
1739         current_app_archapks = filter_apk_list_sorted(archapks)
1740         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1741             kept = 0
1742             # Move forward the ones we want again, except DisableAlgorithm
1743             for apk in current_app_archapks:
1744                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1745                     move_apk_between_sections(archivedir, repodir, apk)
1746                     archapks.remove(apk)
1747                     apks.append(apk)
1748                     kept += 1
1749                 if kept == keepversions:
1750                     break
1751
1752
1753 def move_apk_between_sections(from_dir, to_dir, apk):
1754     """move an APK from repo to archive or vice versa"""
1755
1756     def _move_file(from_dir, to_dir, filename, ignore_missing):
1757         from_path = os.path.join(from_dir, filename)
1758         if ignore_missing and not os.path.exists(from_path):
1759             return
1760         to_path = os.path.join(to_dir, filename)
1761         if not os.path.exists(to_dir):
1762             os.mkdir(to_dir)
1763         shutil.move(from_path, to_path)
1764
1765     if from_dir == to_dir:
1766         return
1767
1768     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1769     _move_file(from_dir, to_dir, apk['apkName'], False)
1770     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1771     for density in all_screen_densities:
1772         from_icon_dir = get_icon_dir(from_dir, density)
1773         to_icon_dir = get_icon_dir(to_dir, density)
1774         if density not in apk.get('icons', []):
1775             continue
1776         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1777     if 'srcname' in apk:
1778         _move_file(from_dir, to_dir, apk['srcname'], False)
1779
1780
1781 def add_apks_to_per_app_repos(repodir, apks):
1782     apks_per_app = dict()
1783     for apk in apks:
1784         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1785         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1786         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1787         apks_per_app[apk['packageName']] = apk
1788
1789         if not os.path.exists(apk['per_app_icons']):
1790             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1791             os.makedirs(apk['per_app_icons'])
1792
1793         apkpath = os.path.join(repodir, apk['apkName'])
1794         shutil.copy(apkpath, apk['per_app_repo'])
1795         apksigpath = apkpath + '.sig'
1796         if os.path.exists(apksigpath):
1797             shutil.copy(apksigpath, apk['per_app_repo'])
1798         apkascpath = apkpath + '.asc'
1799         if os.path.exists(apkascpath):
1800             shutil.copy(apkascpath, apk['per_app_repo'])
1801
1802
1803 def create_metadata_from_template(apk):
1804     '''create a new metadata file using internal or external template
1805
1806     Generate warnings for apk's with no metadata (or create skeleton
1807     metadata files, if requested on the command line).  Though the
1808     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1809     since those impose things on the metadata file made from the
1810     template: field sort order, empty field value, formatting, etc.
1811     '''
1812
1813     import yaml
1814     if os.path.exists('template.yml'):
1815         with open('template.yml') as f:
1816             metatxt = f.read()
1817         if 'name' in apk and apk['name'] != '':
1818             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1819                              r'\1 ' + apk['name'],
1820                              metatxt,
1821                              flags=re.IGNORECASE | re.MULTILINE)
1822         else:
1823             logging.warning(_('{appid} does not have a name! Using package name instead.')
1824                             .format(appid=apk['packageName']))
1825             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1826                              r'\1 ' + apk['packageName'],
1827                              metatxt,
1828                              flags=re.IGNORECASE | re.MULTILINE)
1829         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1830             f.write(metatxt)
1831     else:
1832         app = dict()
1833         app['Categories'] = [os.path.basename(os.getcwd())]
1834         # include some blanks as part of the template
1835         app['AuthorName'] = ''
1836         app['Summary'] = ''
1837         app['WebSite'] = ''
1838         app['IssueTracker'] = ''
1839         app['SourceCode'] = ''
1840         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1841         if 'name' in apk and apk['name'] != '':
1842             app['Name'] = apk['name']
1843         else:
1844             logging.warning(_('{appid} does not have a name! Using package name instead.')
1845                             .format(appid=apk['packageName']))
1846             app['Name'] = apk['packageName']
1847         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1848             yaml.dump(app, f, default_flow_style=False)
1849     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1850
1851
1852 config = None
1853 options = None
1854 start_timestamp = time.gmtime()
1855
1856
1857 def main():
1858
1859     global config, options
1860
1861     # Parse command line...
1862     parser = ArgumentParser()
1863     common.setup_global_opts(parser)
1864     parser.add_argument("--create-key", action="store_true", default=False,
1865                         help=_("Add a repo signing key to an unsigned repo"))
1866     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1867                         help=_("Add skeleton metadata files for APKs that are missing them"))
1868     parser.add_argument("--delete-unknown", action="store_true", default=False,
1869                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1870     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1871                         help=_("Report on build data status"))
1872     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1873                         help=_("Interactively ask about things that need updating."))
1874     parser.add_argument("-I", "--icons", action="store_true", default=False,
1875                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1876     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1877                         help=_("Specify editor to use in interactive mode. Default " +
1878                                "is {path}").format(path='/etc/alternatives/editor'))
1879     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1880                         help=_("Update the wiki"))
1881     parser.add_argument("--pretty", action="store_true", default=False,
1882                         help=_("Produce human-readable XML/JSON for index files"))
1883     parser.add_argument("--clean", action="store_true", default=False,
1884                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1885     parser.add_argument("--nosign", action="store_true", default=False,
1886                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1887     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1888                         help=_("Use date from APK instead of current time for newly added APKs"))
1889     parser.add_argument("--rename-apks", action="store_true", default=False,
1890                         help=_("Rename APK files that do not match package.name_123.apk"))
1891     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1892                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1893     metadata.add_metadata_arguments(parser)
1894     options = parser.parse_args()
1895     metadata.warnings_action = options.W
1896
1897     config = common.read_config(options)
1898
1899     if not ('jarsigner' in config and 'keytool' in config):
1900         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1901
1902     repodirs = ['repo']
1903     if config['archive_older'] != 0:
1904         repodirs.append('archive')
1905         if not os.path.exists('archive'):
1906             os.mkdir('archive')
1907
1908     if options.icons:
1909         resize_all_icons(repodirs)
1910         sys.exit(0)
1911
1912     if options.rename_apks:
1913         options.clean = True
1914
1915     # check that icons exist now, rather than fail at the end of `fdroid update`
1916     for k in ['repo_icon', 'archive_icon']:
1917         if k in config:
1918             if not os.path.exists(config[k]):
1919                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1920                                  .format(name=k, path=config[k]))
1921                 sys.exit(1)
1922
1923     # if the user asks to create a keystore, do it now, reusing whatever it can
1924     if options.create_key:
1925         if os.path.exists(config['keystore']):
1926             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1927             logging.critical("\t'" + config['keystore'] + "'")
1928             sys.exit(1)
1929
1930         if 'repo_keyalias' not in config:
1931             config['repo_keyalias'] = socket.getfqdn()
1932             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1933         if 'keydname' not in config:
1934             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1935             common.write_to_config(config, 'keydname', config['keydname'])
1936         if 'keystore' not in config:
1937             config['keystore'] = common.default_config['keystore']
1938             common.write_to_config(config, 'keystore', config['keystore'])
1939
1940         password = common.genpassword()
1941         if 'keystorepass' not in config:
1942             config['keystorepass'] = password
1943             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1944         if 'keypass' not in config:
1945             config['keypass'] = password
1946             common.write_to_config(config, 'keypass', config['keypass'])
1947         common.genkeystore(config)
1948
1949     # Get all apps...
1950     apps = metadata.read_metadata()
1951
1952     # Generate a list of categories...
1953     categories = set()
1954     for app in apps.values():
1955         categories.update(app.Categories)
1956
1957     # Read known apks data (will be updated and written back when we've finished)
1958     knownapks = common.KnownApks()
1959
1960     # Get APK cache
1961     apkcache = get_cache()
1962
1963     # Delete builds for disabled apps
1964     delete_disabled_builds(apps, apkcache, repodirs)
1965
1966     # Scan all apks in the main repo
1967     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1968
1969     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1970                                            options.use_date_from_apk)
1971     cachechanged = cachechanged or fcachechanged
1972     apks += files
1973     for apk in apks:
1974         if apk['packageName'] not in apps:
1975             if options.create_metadata:
1976                 create_metadata_from_template(apk)
1977                 apps = metadata.read_metadata()
1978             else:
1979                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1980                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1981                 if options.delete_unknown:
1982                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1983                                  .format(apkfilename=apk['apkName']))
1984                     rmf = os.path.join(repodirs[0], apk['apkName'])
1985                     if not os.path.exists(rmf):
1986                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1987                     else:
1988                         os.remove(rmf)
1989                 else:
1990                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1991
1992     copy_triple_t_store_metadata(apps)
1993     insert_obbs(repodirs[0], apps, apks)
1994     insert_localized_app_metadata(apps)
1995     translate_per_build_anti_features(apps, apks)
1996
1997     # Scan the archive repo for apks as well
1998     if len(repodirs) > 1:
1999         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
2000         if cc:
2001             cachechanged = True
2002     else:
2003         archapks = []
2004
2005     # Apply information from latest apks to the application and update dates
2006     apply_info_from_latest_apk(apps, apks + archapks)
2007
2008     # Sort the app list by name, then the web site doesn't have to by default.
2009     # (we had to wait until we'd scanned the apks to do this, because mostly the
2010     # name comes from there!)
2011     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
2012
2013     # APKs are placed into multiple repos based on the app package, providing
2014     # per-app subscription feeds for nightly builds and things like it
2015     if config['per_app_repos']:
2016         add_apks_to_per_app_repos(repodirs[0], apks)
2017         for appid, app in apps.items():
2018             repodir = os.path.join(appid, 'fdroid', 'repo')
2019             appdict = dict()
2020             appdict[appid] = app
2021             if os.path.isdir(repodir):
2022                 index.make(appdict, [appid], apks, repodir, False)
2023             else:
2024                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
2025         return
2026
2027     if len(repodirs) > 1:
2028         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
2029
2030     # Make the index for the main repo...
2031     index.make(apps, sortedids, apks, repodirs[0], False)
2032     make_categories_txt(repodirs[0], categories)
2033
2034     # If there's an archive repo,  make the index for it. We already scanned it
2035     # earlier on.
2036     if len(repodirs) > 1:
2037         index.make(apps, sortedids, archapks, repodirs[1], True)
2038
2039     git_remote = config.get('binary_transparency_remote')
2040     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
2041         from . import btlog
2042         btlog.make_binary_transparency_log(repodirs)
2043
2044     if config['update_stats']:
2045         # Update known apks info...
2046         knownapks.writeifchanged()
2047
2048         # Generate latest apps data for widget
2049         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
2050             data = ''
2051             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
2052                 for line in f:
2053                     appid = line.rstrip()
2054                     data += appid + "\t"
2055                     app = apps[appid]
2056                     data += app.Name + "\t"
2057                     if app.icon is not None:
2058                         data += app.icon + "\t"
2059                     data += app.License + "\n"
2060             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
2061                 f.write(data)
2062
2063     if cachechanged:
2064         write_cache(apkcache)
2065
2066     # Update the wiki...
2067     if options.wiki:
2068         update_wiki(apps, sortedids, apks + archapks)
2069
2070     logging.info(_("Finished"))
2071
2072
2073 if __name__ == "__main__":
2074     main()