fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0':
 100         return os.path.join(repodir, "icons")
 101     return os.path.join(repodir, "icons-%s" % density)
 102
 103
 104 def get_icon_dirs(repodir):
 105     for density in screen_densities:
 106         yield get_icon_dir(repodir, density)
 107
 108
 109 def get_all_icon_dirs(repodir):
 110     for density in all_screen_densities:
 111         yield get_icon_dir(repodir, density)
 112
 113
 114 def update_wiki(apps, sortedids, apks):
 115     """Update the wiki
 116
 117     :param apps: fully populated list of all applications
 118     :param apks: all apks, except...
 119     """
 120     logging.info("Updating wiki")
 121     wikicat = 'Apps'
 122     wikiredircat = 'App Redirects'
 123     import mwclient
 124     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 125                          path=config['wiki_path'])
 126     site.login(config['wiki_user'], config['wiki_password'])
 127     generated_pages = {}
 128     generated_redirects = {}
 129
 130     for appid in sortedids:
 131         app = metadata.App(apps[appid])
 132
 133         wikidata = ''
 134         if app.Disabled:
 135             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 136         if app.AntiFeatures:
 137             for af in sorted(app.AntiFeatures):
 138                 wikidata += '{{AntiFeature|' + af + '}}\n'
 139         if app.RequiresRoot:
 140             requiresroot = 'Yes'
 141         else:
 142             requiresroot = 'No'
 143         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 144             appid,
 145             app.Name,
 146             app.added.strftime('%Y-%m-%d') if app.added else '',
 147             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 148             app.SourceCode,
 149             app.IssueTracker,
 150             app.WebSite,
 151             app.Changelog,
 152             app.Donate,
 153             app.FlattrID,
 154             app.LiberapayID,
 155             app.Bitcoin,
 156             app.Litecoin,
 157             app.License,
 158             requiresroot,
 159             app.AuthorName,
 160             app.AuthorEmail)
 161
 162         if app.Provides:
 163             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 164
 165         wikidata += app.Summary
 166         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 167
 168         wikidata += "=Description=\n"
 169         wikidata += metadata.description_wiki(app.Description) + "\n"
 170
 171         wikidata += "=Maintainer Notes=\n"
 172         if app.MaintainerNotes:
 173             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 174         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 175
 176         # Get a list of all packages for this application...
 177         apklist = []
 178         gotcurrentver = False
 179         cantupdate = False
 180         buildfails = False
 181         for apk in apks:
 182             if apk['packageName'] == appid:
 183                 if str(apk['versionCode']) == app.CurrentVersionCode:
 184                     gotcurrentver = True
 185                 apklist.append(apk)
 186         # Include ones we can't build, as a special case...
 187         for build in app.builds:
 188             if build.disable:
 189                 if build.versionCode == app.CurrentVersionCode:
 190                     cantupdate = True
 191                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 192                 apklist.append({'versionCode': int(build.versionCode),
 193                                 'versionName': build.versionName,
 194                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 195                                 })
 196             else:
 197                 builtit = False
 198                 for apk in apklist:
 199                     if apk['versionCode'] == int(build.versionCode):
 200                         builtit = True
 201                         break
 202                 if not builtit:
 203                     buildfails = True
 204                     apklist.append({'versionCode': int(build.versionCode),
 205                                     'versionName': build.versionName,
 206                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 207                                     })
 208         if app.CurrentVersionCode == '0':
 209             cantupdate = True
 210         # Sort with most recent first...
 211         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 212
 213         wikidata += "=Versions=\n"
 214         if len(apklist) == 0:
 215             wikidata += "We currently have no versions of this app available."
 216         elif not gotcurrentver:
 217             wikidata += "We don't have the current version of this app."
 218         else:
 219             wikidata += "We have the current version of this app."
 220         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 221         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 222         if len(app.NoSourceSince) > 0:
 223             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 224         if len(app.CurrentVersion) > 0:
 225             wikidata += "The current (recommended) version is " + app.CurrentVersion
 226             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 227         validapks = 0
 228         for apk in apklist:
 229             wikidata += "==" + apk['versionName'] + "==\n"
 230
 231             if 'buildproblem' in apk:
 232                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 233             else:
 234                 validapks += 1
 235                 wikidata += "This version is built and signed by "
 236                 if 'srcname' in apk:
 237                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 238                 else:
 239                     wikidata += "the original developer.\n\n"
 240             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 241
 242         wikidata += '\n[[Category:' + wikicat + ']]\n'
 243         if len(app.NoSourceSince) > 0:
 244             wikidata += '\n[[Category:Apps missing source code]]\n'
 245         if validapks == 0 and not app.Disabled:
 246             wikidata += '\n[[Category:Apps with no packages]]\n'
 247         if cantupdate and not app.Disabled:
 248             wikidata += "\n[[Category:Apps we cannot update]]\n"
 249         if buildfails and not app.Disabled:
 250             wikidata += "\n[[Category:Apps with failing builds]]\n"
 251         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 252             wikidata += '\n[[Category:Apps to Update]]\n'
 253         if app.Disabled:
 254             wikidata += '\n[[Category:Apps that are disabled]]\n'
 255         if app.UpdateCheckMode == 'None' and not app.Disabled:
 256             wikidata += '\n[[Category:Apps with no update check]]\n'
 257         for appcat in app.Categories:
 258             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 259
 260         # We can't have underscores in the page name, even if they're in
 261         # the package ID, because MediaWiki messes with them...
 262         pagename = appid.replace('_', ' ')
 263
 264         # Drop a trailing newline, because mediawiki is going to drop it anyway
 265         # and it we don't we'll think the page has changed when it hasn't...
 266         if wikidata.endswith('\n'):
 267             wikidata = wikidata[:-1]
 268
 269         generated_pages[pagename] = wikidata
 270
 271         # Make a redirect from the name to the ID too, unless there's
 272         # already an existing page with the name and it isn't a redirect.
 273         noclobber = False
 274         apppagename = app.Name.replace('_', ' ')
 275         apppagename = apppagename.replace('{', '')
 276         apppagename = apppagename.replace('}', ' ')
 277         apppagename = apppagename.replace(':', ' ')
 278         apppagename = apppagename.replace('[', ' ')
 279         apppagename = apppagename.replace(']', ' ')
 280         # Drop double spaces caused mostly by replacing ':' above
 281         apppagename = apppagename.replace('  ', ' ')
 282         for expagename in site.allpages(prefix=apppagename,
 283                                         filterredir='nonredirects',
 284                                         generator=False):
 285             if expagename == apppagename:
 286                 noclobber = True
 287         # Another reason not to make the redirect page is if the app name
 288         # is the same as it's ID, because that will overwrite the real page
 289         # with an redirect to itself! (Although it seems like an odd
 290         # scenario this happens a lot, e.g. where there is metadata but no
 291         # builds or binaries to extract a name from.
 292         if apppagename == pagename:
 293             noclobber = True
 294         if not noclobber:
 295             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 296
 297     for tcat, genp in [(wikicat, generated_pages),
 298                        (wikiredircat, generated_redirects)]:
 299         catpages = site.Pages['Category:' + tcat]
 300         existingpages = []
 301         for page in catpages:
 302             existingpages.append(page.name)
 303             if page.name in genp:
 304                 pagetxt = page.edit()
 305                 if pagetxt != genp[page.name]:
 306                     logging.debug("Updating modified page " + page.name)
 307                     page.save(genp[page.name], summary='Auto-updated')
 308                 else:
 309                     logging.debug("Page " + page.name + " is unchanged")
 310             else:
 311                 logging.warn("Deleting page " + page.name)
 312                 page.delete('No longer published')
 313         for pagename, text in genp.items():
 314             logging.debug("Checking " + pagename)
 315             if pagename not in existingpages:
 316                 logging.debug("Creating page " + pagename)
 317                 try:
 318                     newpage = site.Pages[pagename]
 319                     newpage.save(text, summary='Auto-created')
 320                 except Exception as e:
 321                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 322
 323     # Purge server cache to ensure counts are up to date
 324     site.Pages['Repository Maintenance'].purge()
 325
 326     # Write a page with the last build log for this version code
 327     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 328     newpage = site.Pages[wiki_page_path]
 329     txt = ''
 330     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 331     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 332     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 333     txt += "\n\n"
 334     txt += common.get_android_tools_version_log()
 335     newpage.save(txt, summary='Run log')
 336     newpage = site.Pages['update']
 337     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 338
 339
 340 def delete_disabled_builds(apps, apkcache, repodirs):
 341     """Delete disabled build outputs.
 342
 343     :param apps: list of all applications, as per metadata.read_metadata
 344     :param apkcache: current apk cache information
 345     :param repodirs: the repo directories to process
 346     """
 347     for appid, app in apps.items():
 348         for build in app['builds']:
 349             if not build.disable:
 350                 continue
 351             apkfilename = common.get_release_filename(app, build)
 352             iconfilename = "%s.%s.png" % (
 353                 appid,
 354                 build.versionCode)
 355             for repodir in repodirs:
 356                 files = [
 357                     os.path.join(repodir, apkfilename),
 358                     os.path.join(repodir, apkfilename + '.asc'),
 359                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 360                 ]
 361                 for density in all_screen_densities:
 362                     repo_dir = get_icon_dir(repodir, density)
 363                     files.append(os.path.join(repo_dir, iconfilename))
 364
 365                 for f in files:
 366                     if os.path.exists(f):
 367                         logging.info("Deleting disabled build output " + f)
 368                         os.remove(f)
 369             if apkfilename in apkcache:
 370                 del apkcache[apkfilename]
 371
 372
 373 def resize_icon(iconpath, density):
 374
 375     if not os.path.isfile(iconpath):
 376         return
 377
 378     fp = None
 379     try:
 380         fp = open(iconpath, 'rb')
 381         im = Image.open(fp)
 382         size = dpi_to_px(density)
 383
 384         if any(length > size for length in im.size):
 385             oldsize = im.size
 386             im.thumbnail((size, size), Image.ANTIALIAS)
 387             logging.debug("%s was too large at %s - new size is %s" % (
 388                 iconpath, oldsize, im.size))
 389             im.save(iconpath, "PNG", optimize=True,
 390                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 391
 392     except Exception as e:
 393         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 394
 395     finally:
 396         if fp:
 397             fp.close()
 398
 399
 400 def resize_all_icons(repodirs):
 401     """Resize all icons that exceed the max size
 402
 403     :param repodirs: the repo directories to process
 404     """
 405     for repodir in repodirs:
 406         for density in screen_densities:
 407             icon_dir = get_icon_dir(repodir, density)
 408             icon_glob = os.path.join(icon_dir, '*.png')
 409             for iconpath in glob.glob(icon_glob):
 410                 resize_icon(iconpath, density)
 411
 412
 413 def getsig(apkpath):
 414     """ Get the signing certificate of an apk. To get the same md5 has that
 415     Android gets, we encode the .RSA certificate in a specific format and pass
 416     it hex-encoded to the md5 digest algorithm.
 417
 418     :param apkpath: path to the apk
 419     :returns: A string containing the md5 of the signature of the apk or None
 420               if an error occurred.
 421     """
 422
 423     with zipfile.ZipFile(apkpath, 'r') as apk:
 424         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 425
 426         if len(certs) < 1:
 427             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 428             return None
 429         if len(certs) > 1:
 430             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 431             return None
 432
 433         cert = apk.read(certs[0])
 434
 435     cert_encoded = common.get_certificate(cert)
 436
 437     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 438
 439
 440 def get_cache_file():
 441     return os.path.join('tmp', 'apkcache')
 442
 443
 444 def get_cache():
 445     """Get the cached dict of the APK index
 446
 447     Gather information about all the apk files in the repo directory,
 448     using cached data if possible. Some of the index operations take a
 449     long time, like calculating the SHA-256 and verifying the APK
 450     signature.
 451
 452     The cache is invalidated if the metadata version is different, or
 453     the 'allow_disabled_algorithms' config/option is different.  In
 454     those cases, there is no easy way to know what has changed from
 455     the cache, so just rerun the whole thing.
 456
 457     :return: apkcache
 458
 459     """
 460     apkcachefile = get_cache_file()
 461     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 462     if not options.clean and os.path.exists(apkcachefile):
 463         with open(apkcachefile, 'rb') as cf:
 464             apkcache = pickle.load(cf, encoding='utf-8')
 465         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 466            or apkcache.get('allow_disabled_algorithms') != ada:
 467             apkcache = {}
 468     else:
 469         apkcache = {}
 470
 471     apkcache["METADATA_VERSION"] = METADATA_VERSION
 472     apkcache['allow_disabled_algorithms'] = ada
 473
 474     return apkcache
 475
 476
 477 def write_cache(apkcache):
 478     apkcachefile = get_cache_file()
 479     cache_path = os.path.dirname(apkcachefile)
 480     if not os.path.exists(cache_path):
 481         os.makedirs(cache_path)
 482     with open(apkcachefile, 'wb') as cf:
 483         pickle.dump(apkcache, cf)
 484
 485
 486 def get_icon_bytes(apkzip, iconsrc):
 487     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 488     try:
 489         return apkzip.read(iconsrc)
 490     except KeyError:
 491         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 492
 493
 494 def sha256sum(filename):
 495     '''Calculate the sha256 of the given file'''
 496     sha = hashlib.sha256()
 497     with open(filename, 'rb') as f:
 498         while True:
 499             t = f.read(16384)
 500             if len(t) == 0:
 501                 break
 502             sha.update(t)
 503     return sha.hexdigest()
 504
 505
 506 def has_known_vulnerability(filename):
 507     """checks for known vulnerabilities in the APK
 508
 509     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 510     version.  Google also enforces this:
 511     https://support.google.com/faqs/answer/6376725?hl=en
 512
 513     Checks whether there are more than one classes.dex or AndroidManifest.xml
 514     files, which is invalid and an essential part of the "Master Key" attack.
 515     http://www.saurik.com/id/17
 516
 517     Janus is similar to Master Key but is perhaps easier to scan for.
 518     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 519     """
 520
 521     found_vuln = False
 522
 523     # statically load this pattern
 524     if not hasattr(has_known_vulnerability, "pattern"):
 525         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 526
 527     with open(filename.encode(), 'rb') as fp:
 528         first4 = fp.read(4)
 529     if first4 != b'\x50\x4b\x03\x04':
 530         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 531                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 532                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 533
 534     files_in_apk = set()
 535     with zipfile.ZipFile(filename) as zf:
 536         for name in zf.namelist():
 537             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 538                 lib = zf.open(name)
 539                 while True:
 540                     chunk = lib.read(4096)
 541                     if chunk == b'':
 542                         break
 543                     m = has_known_vulnerability.pattern.search(chunk)
 544                     if m:
 545                         version = m.group(1).decode('ascii')
 546                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 547                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 548                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 549                             logging.debug(_('"{path}" contains recent {name} ({version})')
 550                                           .format(path=filename, name=name, version=version))
 551                         else:
 552                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 553                                             .format(path=filename, name=name, version=version))
 554                             found_vuln = True
 555                         break
 556             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 557                 if name in files_in_apk:
 558                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 559                                     .format(apkfilename=filename, name=name))
 560                     found_vuln = True
 561                 files_in_apk.add(name)
 562     return found_vuln
 563
 564
 565 def insert_obbs(repodir, apps, apks):
 566     """Scans the .obb files in a given repo directory and adds them to the
 567     relevant APK instances.  OBB files have versionCodes like APK
 568     files, and they are loosely associated.  If there is an OBB file
 569     present, then any APK with the same or higher versionCode will use
 570     that OBB file.  There are two OBB types: main and patch, each APK
 571     can only have only have one of each.
 572
 573     https://developer.android.com/google/play/expansion-files.html
 574
 575     :param repodir: repo directory to scan
 576     :param apps: list of current, valid apps
 577     :param apks: current information on all APKs
 578
 579     """
 580
 581     def obbWarnDelete(f, msg):
 582         logging.warning(msg + ' ' + f)
 583         if options.delete_unknown:
 584             logging.error(_("Deleting unknown file: {path}").format(path=f))
 585             os.remove(f)
 586
 587     obbs = []
 588     java_Integer_MIN_VALUE = -pow(2, 31)
 589     currentPackageNames = apps.keys()
 590     for f in glob.glob(os.path.join(repodir, '*.obb')):
 591         obbfile = os.path.basename(f)
 592         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 593         chunks = obbfile.split('.')
 594         if chunks[0] != 'main' and chunks[0] != 'patch':
 595             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 596             continue
 597         if not re.match(r'^-?[0-9]+$', chunks[1]):
 598             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 599                           .format(name=chunks[0]))
 600             continue
 601         versionCode = int(chunks[1])
 602         packagename = ".".join(chunks[2:-1])
 603
 604         highestVersionCode = java_Integer_MIN_VALUE
 605         if packagename not in currentPackageNames:
 606             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 607             continue
 608         for apk in apks:
 609             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 610                 highestVersionCode = apk['versionCode']
 611         if versionCode > highestVersionCode:
 612             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 613                           .format(integer=str(versionCode)))
 614             continue
 615         obbsha256 = sha256sum(f)
 616         obbs.append((packagename, versionCode, obbfile, obbsha256))
 617
 618     for apk in apks:
 619         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 620             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 621                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 622                     apk['obbMainFile'] = obbfile
 623                     apk['obbMainFileSha256'] = obbsha256
 624                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 625                     apk['obbPatchFile'] = obbfile
 626                     apk['obbPatchFileSha256'] = obbsha256
 627             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 628                 break
 629
 630
 631 def translate_per_build_anti_features(apps, apks):
 632     """Grab the anti-features list from the build metadata
 633
 634     For most Anti-Features, they are really most applicable per-APK,
 635     not for an app.  An app can fix a vulnerability, add/remove
 636     tracking, etc.  This reads the 'antifeatures' list from the Build
 637     entries in the fdroiddata metadata file, then transforms it into
 638     the 'antiFeatures' list of unique items for the index.
 639
 640     The field key is all lower case in the metadata file to match the
 641     rest of the Build fields.  It is 'antiFeatures' camel case in the
 642     implementation, index, and fdroidclient since it is translated
 643     from the build 'antifeatures' field, not directly included.
 644
 645     """
 646
 647     antiFeatures = dict()
 648     for packageName, app in apps.items():
 649         d = dict()
 650         for build in app['builds']:
 651             afl = build.get('antifeatures')
 652             if afl:
 653                 d[int(build.versionCode)] = afl
 654         if len(d) > 0:
 655             antiFeatures[packageName] = d
 656
 657     for apk in apks:
 658         d = antiFeatures.get(apk['packageName'])
 659         if d:
 660             afl = d.get(apk['versionCode'])
 661             if afl:
 662                 apk['antiFeatures'].update(afl)
 663
 664
 665 def _get_localized_dict(app, locale):
 666     '''get the dict to add localized store metadata to'''
 667     if 'localized' not in app:
 668         app['localized'] = collections.OrderedDict()
 669     if locale not in app['localized']:
 670         app['localized'][locale] = collections.OrderedDict()
 671     return app['localized'][locale]
 672
 673
 674 def _set_localized_text_entry(app, locale, key, f):
 675     limit = config['char_limits'][key]
 676     localized = _get_localized_dict(app, locale)
 677     with open(f) as fp:
 678         text = fp.read()[:limit]
 679         if len(text) > 0:
 680             localized[key] = text
 681
 682
 683 def _set_author_entry(app, key, f):
 684     limit = config['char_limits']['author']
 685     with open(f) as fp:
 686         text = fp.read()[:limit]
 687         if len(text) > 0:
 688             app[key] = text
 689
 690
 691 def _strip_and_copy_image(inpath, outpath):
 692     """Remove any metadata from image and copy it to new path
 693
 694     Sadly, image metadata like EXIF can be used to exploit devices.
 695     It is not used at all in the F-Droid ecosystem, so its much safer
 696     just to remove it entirely.
 697
 698     """
 699
 700     extension = common.get_extension(inpath)[1]
 701     if os.path.isdir(outpath):
 702         outpath = os.path.join(outpath, os.path.basename(inpath))
 703     if extension == 'png':
 704         with open(inpath, 'rb') as fp:
 705             in_image = Image.open(fp)
 706             in_image.save(outpath, "PNG", optimize=True,
 707                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 708     elif extension == 'jpg' or extension == 'jpeg':
 709         with open(inpath, 'rb') as fp:
 710             in_image = Image.open(fp)
 711             data = list(in_image.getdata())
 712             out_image = Image.new(in_image.mode, in_image.size)
 713         out_image.putdata(data)
 714         out_image.save(outpath, "JPEG", optimize=True)
 715     else:
 716         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 717                               .format(extension=extension))
 718
 719
 720 def copy_triple_t_store_metadata(apps):
 721     """Include store metadata from the app's source repo
 722
 723     The Triple-T Gradle Play Publisher is a plugin that has a standard
 724     file layout for all of the metadata and graphics that the Google
 725     Play Store accepts.  Since F-Droid has the git repo, it can just
 726     pluck those files directly.  This method reads any text files into
 727     the app dict, then copies any graphics into the fdroid repo
 728     directory structure.
 729
 730     This needs to be run before insert_localized_app_metadata() so that
 731     the graphics files that are copied into the fdroid repo get
 732     properly indexed.
 733
 734     https://github.com/Triple-T/gradle-play-publisher#upload-images
 735     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 736
 737     """
 738
 739     if not os.path.isdir('build'):
 740         return  # nothing to do
 741
 742     for packageName, app in apps.items():
 743         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 744             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 745             for root, dirs, files in os.walk(d):
 746                 segments = root.split('/')
 747                 locale = segments[-2]
 748                 for f in files:
 749                     if f == 'fulldescription':
 750                         _set_localized_text_entry(app, locale, 'description',
 751                                                   os.path.join(root, f))
 752                         continue
 753                     elif f == 'shortdescription':
 754                         _set_localized_text_entry(app, locale, 'summary',
 755                                                   os.path.join(root, f))
 756                         continue
 757                     elif f == 'title':
 758                         _set_localized_text_entry(app, locale, 'name',
 759                                                   os.path.join(root, f))
 760                         continue
 761                     elif f == 'video':
 762                         _set_localized_text_entry(app, locale, 'video',
 763                                                   os.path.join(root, f))
 764                         continue
 765                     elif f == 'whatsnew':
 766                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 767                                                   os.path.join(root, f))
 768                         continue
 769                     elif f == 'contactEmail':
 770                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 771                         continue
 772                     elif f == 'contactPhone':
 773                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 774                         continue
 775                     elif f == 'contactWebsite':
 776                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 777                         continue
 778
 779                     base, extension = common.get_extension(f)
 780                     dirname = os.path.basename(root)
 781                     if extension in ALLOWED_EXTENSIONS \
 782                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 783                         if segments[-2] == 'listing':
 784                             locale = segments[-3]
 785                         else:
 786                             locale = segments[-2]
 787                         destdir = os.path.join('repo', packageName, locale, dirname)
 788                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 789                         sourcefile = os.path.join(root, f)
 790                         destfile = os.path.join(destdir, os.path.basename(f))
 791                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 792                         _strip_and_copy_image(sourcefile, destfile)
 793
 794
 795 def insert_localized_app_metadata(apps):
 796     """scans standard locations for graphics and localized text
 797
 798     Scans for localized description files, store graphics, and
 799     screenshot PNG files in statically defined screenshots directory
 800     and adds them to the app metadata.  The screenshots and graphic
 801     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 802     and must be in the following layout:
 803     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 804
 805     repo/packageName/locale/featureGraphic.png
 806     repo/packageName/locale/phoneScreenshots/1.png
 807     repo/packageName/locale/phoneScreenshots/2.png
 808
 809     The changelog files must be text files named with the versionCode
 810     ending with ".txt" and must be in the following layout:
 811     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 812
 813     repo/packageName/locale/changelogs/12345.txt
 814
 815     This will scan the each app's source repo then the metadata/ dir
 816     for these standard locations of changelog files.  If it finds
 817     them, they will be added to the dict of all packages, with the
 818     versions in the metadata/ folder taking precendence over the what
 819     is in the app's source repo.
 820
 821     Where "packageName" is the app's packageName and "locale" is the locale
 822     of the graphics, e.g. what language they are in, using the IETF RFC5646
 823     format (en-US, fr-CA, es-MX, etc).
 824
 825     This will also scan the app's git for a fastlane folder, and the
 826     metadata/ folder and the apps' source repos for standard locations
 827     of graphic and screenshot files.  If it finds them, it will copy
 828     them into the repo.  The fastlane files follow this pattern:
 829     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 830
 831     """
 832
 833     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 834     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 837
 838     for srcd in sorted(sourcedirs):
 839         if not os.path.isdir(srcd):
 840             continue
 841         for root, dirs, files in os.walk(srcd):
 842             segments = root.split('/')
 843             packageName = segments[1]
 844             if packageName not in apps:
 845                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 846                 continue
 847             locale = segments[-1]
 848             destdir = os.path.join('repo', packageName, locale)
 849
 850             # flavours specified in build receipt
 851             build_flavours = ""
 852             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 853                     and 'gradle' in apps[packageName].builds[-1]:
 854                 build_flavours = apps[packageName].builds[-1].gradle
 855
 856             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 857                 logging.debug("ignoring due to wrong flavour")
 858                 continue
 859
 860             for f in files:
 861                 if f in ('description.txt', 'full_description.txt'):
 862                     _set_localized_text_entry(apps[packageName], locale, 'description',
 863                                               os.path.join(root, f))
 864                     continue
 865                 elif f in ('summary.txt', 'short_description.txt'):
 866                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 867                                               os.path.join(root, f))
 868                     continue
 869                 elif f in ('name.txt', 'title.txt'):
 870                     _set_localized_text_entry(apps[packageName], locale, 'name',
 871                                               os.path.join(root, f))
 872                     continue
 873                 elif f == 'video.txt':
 874                     _set_localized_text_entry(apps[packageName], locale, 'video',
 875                                               os.path.join(root, f))
 876                     continue
 877                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 878                     locale = segments[-2]
 879                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 880                                               os.path.join(root, f))
 881                     continue
 882
 883                 base, extension = common.get_extension(f)
 884                 if locale == 'images':
 885                     locale = segments[-2]
 886                     destdir = os.path.join('repo', packageName, locale)
 887                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 888                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 889                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 890                     _strip_and_copy_image(os.path.join(root, f), destdir)
 891             for d in dirs:
 892                 if d in SCREENSHOT_DIRS:
 893                     if locale == 'images':
 894                         locale = segments[-2]
 895                         destdir = os.path.join('repo', packageName, locale)
 896                     for f in glob.glob(os.path.join(root, d, '*.*')):
 897                         _ignored, extension = common.get_extension(f)
 898                         if extension in ALLOWED_EXTENSIONS:
 899                             screenshotdestdir = os.path.join(destdir, d)
 900                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 901                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 902                             _strip_and_copy_image(f, screenshotdestdir)
 903
 904     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 905     for d in repofiles:
 906         if not os.path.isdir(d):
 907             continue
 908         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 909             if not os.path.isfile(f):
 910                 continue
 911             segments = f.split('/')
 912             packageName = segments[1]
 913             locale = segments[2]
 914             screenshotdir = segments[3]
 915             filename = os.path.basename(f)
 916             base, extension = common.get_extension(filename)
 917
 918             if packageName not in apps:
 919                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 920                                 .format(path=filename, name=packageName))
 921                 continue
 922             graphics = _get_localized_dict(apps[packageName], locale)
 923
 924             if extension not in ALLOWED_EXTENSIONS:
 925                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 926             elif base in GRAPHIC_NAMES:
 927                 # there can only be zero or one of these per locale
 928                 graphics[base] = filename
 929             elif screenshotdir in SCREENSHOT_DIRS:
 930                 # there can any number of these per locale
 931                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 932                 if screenshotdir not in graphics:
 933                     graphics[screenshotdir] = []
 934                 graphics[screenshotdir].append(filename)
 935             else:
 936                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 937
 938
 939 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 940     """Scan a repo for all files with an extension except APK/OBB
 941
 942     :param apkcache: current cached info about all repo files
 943     :param repodir: repo directory to scan
 944     :param knownapks: list of all known files, as per metadata.read_metadata
 945     :param use_date_from_file: use date from file (instead of current date)
 946                                for newly added files
 947     """
 948
 949     cachechanged = False
 950     repo_files = []
 951     repodir = repodir.encode('utf-8')
 952     for name in os.listdir(repodir):
 953         file_extension = common.get_file_extension(name)
 954         if file_extension == 'apk' or file_extension == 'obb':
 955             continue
 956         filename = os.path.join(repodir, name)
 957         name_utf8 = name.decode('utf-8')
 958         if filename.endswith(b'_src.tar.gz'):
 959             logging.debug(_('skipping source tarball: {path}')
 960                           .format(path=filename.decode('utf-8')))
 961             continue
 962         if not common.is_repo_file(filename):
 963             continue
 964         stat = os.stat(filename)
 965         if stat.st_size == 0:
 966             raise FDroidException(_('{path} is zero size!')
 967                                   .format(path=filename))
 968
 969         shasum = sha256sum(filename)
 970         usecache = False
 971         if name in apkcache:
 972             repo_file = apkcache[name]
 973             # added time is cached as tuple but used here as datetime instance
 974             if 'added' in repo_file:
 975                 a = repo_file['added']
 976                 if isinstance(a, datetime):
 977                     repo_file['added'] = a
 978                 else:
 979                     repo_file['added'] = datetime(*a[:6])
 980             if repo_file.get('hash') == shasum:
 981                 logging.debug(_("Reading {apkfilename} from cache")
 982                               .format(apkfilename=name_utf8))
 983                 usecache = True
 984             else:
 985                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 986                               .format(apkfilename=name_utf8))
 987
 988         if not usecache:
 989             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 990             repo_file = collections.OrderedDict()
 991             repo_file['name'] = os.path.splitext(name_utf8)[0]
 992             # TODO rename apkname globally to something more generic
 993             repo_file['apkName'] = name_utf8
 994             repo_file['hash'] = shasum
 995             repo_file['hashType'] = 'sha256'
 996             repo_file['versionCode'] = 0
 997             repo_file['versionName'] = shasum
 998             # the static ID is the SHA256 unless it is set in the metadata
 999             repo_file['packageName'] = shasum
1000
1001             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1002             if m:
1003                 repo_file['packageName'] = m.group(1)
1004                 repo_file['versionCode'] = int(m.group(2))
1005             srcfilename = name + b'_src.tar.gz'
1006             if os.path.exists(os.path.join(repodir, srcfilename)):
1007                 repo_file['srcname'] = srcfilename.decode('utf-8')
1008             repo_file['size'] = stat.st_size
1009
1010             apkcache[name] = repo_file
1011             cachechanged = True
1012
1013         if use_date_from_file:
1014             timestamp = stat.st_ctime
1015             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1016         else:
1017             default_date_param = None
1018
1019         # Record in knownapks, getting the added date at the same time..
1020         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1021                                     default_date=default_date_param)
1022         if added:
1023             repo_file['added'] = added
1024
1025         repo_files.append(repo_file)
1026
1027     return repo_files, cachechanged
1028
1029
1030 def scan_apk(apk_file):
1031     """
1032     Scans an APK file and returns dictionary with metadata of the APK.
1033
1034     Attention: This does *not* verify that the APK signature is correct.
1035
1036     :param apk_file: The (ideally absolute) path to the APK file
1037     :raises BuildException
1038     :return A dict containing APK metadata
1039     """
1040     apk = {
1041         'hash': sha256sum(apk_file),
1042         'hashType': 'sha256',
1043         'uses-permission': [],
1044         'uses-permission-sdk-23': [],
1045         'features': [],
1046         'icons_src': {},
1047         'icons': {},
1048         'antiFeatures': set(),
1049     }
1050
1051     if SdkToolsPopen(['aapt', 'version'], output=False):
1052         scan_apk_aapt(apk, apk_file)
1053     else:
1054         scan_apk_androguard(apk, apk_file)
1055
1056     # Get the signature, or rather the signing key fingerprints
1057     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1058     apk['sig'] = getsig(apk_file)
1059     if not apk['sig']:
1060         raise BuildException("Failed to get apk signature")
1061     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1062                                                                apk_file))
1063     if not apk.get('signer'):
1064         raise BuildException("Failed to get apk signing key fingerprint")
1065
1066     # Get size of the APK
1067     apk['size'] = os.path.getsize(apk_file)
1068
1069     if 'minSdkVersion' not in apk:
1070         logging.warning("No SDK version information found in {0}".format(apk_file))
1071         apk['minSdkVersion'] = 1
1072
1073     # Check for known vulnerabilities
1074     if has_known_vulnerability(apk_file):
1075         apk['antiFeatures'].add('KnownVuln')
1076
1077     return apk
1078
1079
1080 def scan_apk_aapt(apk, apkfile):
1081     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1082     if p.returncode != 0:
1083         if options.delete_unknown:
1084             if os.path.exists(apkfile):
1085                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1086                 os.remove(apkfile)
1087             else:
1088                 logging.error("Could not find {0} to remove it".format(apkfile))
1089         else:
1090             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1091         raise BuildException(_("Invalid APK"))
1092     for line in p.output.splitlines():
1093         if line.startswith("package:"):
1094             try:
1095                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1096                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1097                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1098             except Exception as e:
1099                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1100         elif line.startswith("application:"):
1101             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1102             # Keep path to non-dpi icon in case we need it
1103             match = re.match(APK_ICON_PAT_NODPI, line)
1104             if match:
1105                 apk['icons_src']['-1'] = match.group(1)
1106         elif line.startswith("launchable-activity:"):
1107             # Only use launchable-activity as fallback to application
1108             if not apk['name']:
1109                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1110             if '-1' not in apk['icons_src']:
1111                 match = re.match(APK_ICON_PAT_NODPI, line)
1112                 if match:
1113                     apk['icons_src']['-1'] = match.group(1)
1114         elif line.startswith("application-icon-"):
1115             match = re.match(APK_ICON_PAT, line)
1116             if match:
1117                 density = match.group(1)
1118                 path = match.group(2)
1119                 apk['icons_src'][density] = path
1120         elif line.startswith("sdkVersion:"):
1121             m = re.match(APK_SDK_VERSION_PAT, line)
1122             if m is None:
1123                 logging.error(line.replace('sdkVersion:', '')
1124                               + ' is not a valid minSdkVersion!')
1125             else:
1126                 apk['minSdkVersion'] = m.group(1)
1127                 # if target not set, default to min
1128                 if 'targetSdkVersion' not in apk:
1129                     apk['targetSdkVersion'] = m.group(1)
1130         elif line.startswith("targetSdkVersion:"):
1131             m = re.match(APK_SDK_VERSION_PAT, line)
1132             if m is None:
1133                 logging.error(line.replace('targetSdkVersion:', '')
1134                               + ' is not a valid targetSdkVersion!')
1135             else:
1136                 apk['targetSdkVersion'] = m.group(1)
1137         elif line.startswith("maxSdkVersion:"):
1138             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1139         elif line.startswith("native-code:"):
1140             apk['nativecode'] = []
1141             for arch in line[13:].split(' '):
1142                 apk['nativecode'].append(arch[1:-1])
1143         elif line.startswith('uses-permission:'):
1144             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1145             if perm_match['maxSdkVersion']:
1146                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1147             permission = UsesPermission(
1148                 perm_match['name'],
1149                 perm_match['maxSdkVersion']
1150             )
1151
1152             apk['uses-permission'].append(permission)
1153         elif line.startswith('uses-permission-sdk-23:'):
1154             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1155             if perm_match['maxSdkVersion']:
1156                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1157             permission_sdk_23 = UsesPermissionSdk23(
1158                 perm_match['name'],
1159                 perm_match['maxSdkVersion']
1160             )
1161
1162             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1163
1164         elif line.startswith('uses-feature:'):
1165             feature = re.match(APK_FEATURE_PAT, line).group(1)
1166             # Filter out this, it's only added with the latest SDK tools and
1167             # causes problems for lots of apps.
1168             if feature != "android.hardware.screen.portrait" \
1169                     and feature != "android.hardware.screen.landscape":
1170                 if feature.startswith("android.feature."):
1171                     feature = feature[16:]
1172                 apk['features'].add(feature)
1173
1174
1175 def scan_apk_androguard(apk, apkfile):
1176     try:
1177         from androguard.core.bytecodes.apk import APK
1178         apkobject = APK(apkfile)
1179         if apkobject.is_valid_APK():
1180             arsc = apkobject.get_android_resources()
1181         else:
1182             if options.delete_unknown:
1183                 if os.path.exists(apkfile):
1184                     logging.error(_("Failed to get apk information, deleting {path}")
1185                                   .format(path=apkfile))
1186                     os.remove(apkfile)
1187                 else:
1188                     logging.error(_("Could not find {path} to remove it")
1189                                   .format(path=apkfile))
1190             else:
1191                 logging.error(_("Failed to get apk information, skipping {path}")
1192                               .format(path=apkfile))
1193             raise BuildException(_("Invalid APK"))
1194     except ImportError:
1195         raise FDroidException("androguard library is not installed and aapt not present")
1196     except FileNotFoundError:
1197         logging.error(_("Could not open apk file for analysis"))
1198         raise BuildException(_("Invalid APK"))
1199
1200     apk['packageName'] = apkobject.get_package()
1201     apk['versionCode'] = int(apkobject.get_androidversion_code())
1202     apk['versionName'] = apkobject.get_androidversion_name()
1203     if apk['versionName'][0] == "@":
1204         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1205         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1206         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1207     apk['name'] = apkobject.get_app_name()
1208
1209     if apkobject.get_max_sdk_version() is not None:
1210         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1211     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1212     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1213
1214     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1215     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1216
1217     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1218
1219     for file in apkobject.get_files():
1220         d_re = density_re.match(file)
1221         if d_re:
1222             folder = d_re.group(1).split('-')
1223             if len(folder) > 1:
1224                 resolution = folder[1]
1225             else:
1226                 resolution = 'mdpi'
1227             density = screen_resolutions[resolution]
1228             apk['icons_src'][density] = d_re.group(0)
1229
1230     if apk['icons_src'].get('-1') is None:
1231         apk['icons_src']['-1'] = apk['icons_src']['160']
1232
1233     arch_re = re.compile("^lib/(.*)/.*$")
1234     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1235     if len(arch) >= 1:
1236         apk['nativecode'] = []
1237         apk['nativecode'].extend(sorted(list(arch)))
1238
1239     xml = apkobject.get_android_manifest_xml()
1240
1241     for item in xml.getElementsByTagName('uses-permission'):
1242         name = str(item.getAttribute("android:name"))
1243         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1244         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1245         permission = UsesPermission(
1246             name,
1247             maxSdkVersion
1248         )
1249         apk['uses-permission'].append(permission)
1250
1251     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1252         name = str(item.getAttribute("android:name"))
1253         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1254         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1255         permission_sdk_23 = UsesPermissionSdk23(
1256             name,
1257             maxSdkVersion
1258         )
1259         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1260
1261     for item in xml.getElementsByTagName('uses-feature'):
1262         feature = str(item.getAttribute("android:name"))
1263         if feature != "android.hardware.screen.portrait" \
1264                 and feature != "android.hardware.screen.landscape":
1265             if feature.startswith("android.feature."):
1266                 feature = feature[16:]
1267         apk['features'].append(feature)
1268
1269
1270 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1271                 allow_disabled_algorithms=False, archive_bad_sig=False):
1272     """Processes the apk with the given filename in the given repo directory.
1273
1274     This also extracts the icons.
1275
1276     :param apkcache: current apk cache information
1277     :param apkfilename: the filename of the apk to scan
1278     :param repodir: repo directory to scan
1279     :param knownapks: known apks info
1280     :param use_date_from_apk: use date from APK (instead of current date)
1281                               for newly added APKs
1282     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1283                                       disabled algorithms in the signature (e.g. MD5)
1284     :param archive_bad_sig: move APKs with a bad signature to the archive
1285     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1286      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1287     """
1288
1289     apk = {}
1290     apkfile = os.path.join(repodir, apkfilename)
1291
1292     cachechanged = False
1293     usecache = False
1294     if apkfilename in apkcache:
1295         apk = apkcache[apkfilename]
1296         if apk.get('hash') == sha256sum(apkfile):
1297             logging.debug(_("Reading {apkfilename} from cache")
1298                           .format(apkfilename=apkfilename))
1299             usecache = True
1300         else:
1301             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1302                           .format(apkfilename=apkfilename))
1303
1304     if not usecache:
1305         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1306
1307         try:
1308             apk = scan_apk(apkfile)
1309         except BuildException:
1310             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1311                             .format(apkfilename=apkfilename))
1312             return True, None, False
1313
1314         # Check for debuggable apks...
1315         if common.isApkAndDebuggable(apkfile):
1316             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1317
1318         if options.rename_apks:
1319             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1320             std_short_name = os.path.join(repodir, n)
1321             if apkfile != std_short_name:
1322                 if os.path.exists(std_short_name):
1323                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1324                     if apkfile != std_long_name:
1325                         if os.path.exists(std_long_name):
1326                             dupdir = os.path.join('duplicates', repodir)
1327                             if not os.path.isdir(dupdir):
1328                                 os.makedirs(dupdir, exist_ok=True)
1329                             dupfile = os.path.join('duplicates', std_long_name)
1330                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1331                             os.rename(apkfile, dupfile)
1332                             return True, None, False
1333                         else:
1334                             os.rename(apkfile, std_long_name)
1335                     apkfile = std_long_name
1336                 else:
1337                     os.rename(apkfile, std_short_name)
1338                     apkfile = std_short_name
1339                 apkfilename = apkfile[len(repodir) + 1:]
1340
1341         apk['apkName'] = apkfilename
1342         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1343         if os.path.exists(os.path.join(repodir, srcfilename)):
1344             apk['srcname'] = srcfilename
1345
1346         # verify the jar signature is correct, allow deprecated
1347         # algorithms only if the APK is in the archive.
1348         skipapk = False
1349         if not common.verify_apk_signature(apkfile):
1350             if repodir == 'archive' or allow_disabled_algorithms:
1351                 if common.verify_old_apk_signature(apkfile):
1352                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1353                 else:
1354                     skipapk = True
1355             else:
1356                 skipapk = True
1357
1358         if skipapk:
1359             if archive_bad_sig:
1360                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1361                                 .format(apkfilename=apkfilename))
1362                 move_apk_between_sections(repodir, 'archive', apk)
1363             else:
1364                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1365                                 .format(apkfilename=apkfilename))
1366             return True, None, False
1367
1368         apkzip = zipfile.ZipFile(apkfile, 'r')
1369
1370         manifest = apkzip.getinfo('AndroidManifest.xml')
1371         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1372         if (1980, 0, 0) != manifest.date_time[0:3]:
1373             try:
1374                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1375             except ValueError as e:
1376                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1377                                 .format(apkfilename=apkfile) + str(e))
1378
1379         # extract icons from APK zip file
1380         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1381         try:
1382             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1383         finally:
1384             apkzip.close()  # ensure that APK zip file gets closed
1385
1386         # resize existing icons for densities missing in the APK
1387         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1388
1389         if use_date_from_apk and manifest.date_time[1] != 0:
1390             default_date_param = datetime(*manifest.date_time)
1391         else:
1392             default_date_param = None
1393
1394         # Record in known apks, getting the added date at the same time..
1395         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1396                                     default_date=default_date_param)
1397         if added:
1398             apk['added'] = added
1399
1400         apkcache[apkfilename] = apk
1401         cachechanged = True
1402
1403     return False, apk, cachechanged
1404
1405
1406 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1407     """Processes the apks in the given repo directory.
1408
1409     This also extracts the icons.
1410
1411     :param apkcache: current apk cache information
1412     :param repodir: repo directory to scan
1413     :param knownapks: known apks info
1414     :param use_date_from_apk: use date from APK (instead of current date)
1415                               for newly added APKs
1416     :returns: (apks, cachechanged) where apks is a list of apk information,
1417               and cachechanged is True if the apkcache got changed.
1418     """
1419
1420     cachechanged = False
1421
1422     for icon_dir in get_all_icon_dirs(repodir):
1423         if os.path.exists(icon_dir):
1424             if options.clean:
1425                 shutil.rmtree(icon_dir)
1426                 os.makedirs(icon_dir)
1427         else:
1428             os.makedirs(icon_dir)
1429
1430     apks = []
1431     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1432         apkfilename = apkfile[len(repodir) + 1:]
1433         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1434         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1435                                              use_date_from_apk, ada, True)
1436         if skip:
1437             continue
1438         apks.append(apk)
1439         cachechanged = cachechanged or cachethis
1440
1441     return apks, cachechanged
1442
1443
1444 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1445     """Extracts PNG icons from an APK with the supported pixel densities
1446
1447     Extracts icons from the given APK zip in various densities, saves
1448     them into given repo directory and stores their names in the APK
1449     metadata dictionary.  If the icon is an XML icon, then this tries
1450     to find PNG icon that can replace it.
1451
1452     :param icon_filename: A string representing the icon's file name
1453     :param apk: A populated dictionary containing APK metadata.
1454                 Needs to have 'icons_src' key
1455     :param apkzip: An opened zipfile.ZipFile of the APK file
1456     :param repo_dir: The directory of the APK's repository
1457     :return: A list of icon densities that are missing
1458
1459     """
1460     res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1461     pngs = dict()
1462     for f in apkzip.namelist():
1463         m = res_name_re.match(f)
1464         if m and m.group(4) == 'png':
1465             density = screen_resolutions[m.group(2)]
1466             pngs[m.group(3) + '/' + density] = m.group(0)
1467     empty_densities = []
1468     for density in screen_densities:
1469         if density not in apk['icons_src']:
1470             empty_densities.append(density)
1471             continue
1472         icon_src = apk['icons_src'][density]
1473         icon_dir = get_icon_dir(repo_dir, density)
1474         icon_dest = os.path.join(icon_dir, icon_filename)
1475
1476         # Extract the icon files per density
1477         if icon_src.endswith('.xml'):
1478             m = res_name_re.match(icon_src)
1479             if m:
1480                 name = pngs.get(m.group(3) + '/' + str(density))
1481                 if name:
1482                     icon_src = name
1483             if icon_src.endswith('.xml'):
1484                 empty_densities.append(density)
1485                 continue
1486         try:
1487             with open(icon_dest, 'wb') as f:
1488                 f.write(get_icon_bytes(apkzip, icon_src))
1489             apk['icons'][density] = icon_filename
1490         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1491             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1492             del apk['icons_src'][density]
1493             empty_densities.append(density)
1494
1495     if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
1496         icon_src = apk['icons_src']['-1']
1497         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1498         with open(icon_path, 'wb') as f:
1499             f.write(get_icon_bytes(apkzip, icon_src))
1500         im = None
1501         try:
1502             im = Image.open(icon_path)
1503             dpi = px_to_dpi(im.size[0])
1504             for density in screen_densities:
1505                 if density in apk['icons']:
1506                     break
1507                 if density == screen_densities[-1] or dpi >= int(density):
1508                     apk['icons'][density] = icon_filename
1509                     shutil.move(icon_path,
1510                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1511                     empty_densities.remove(density)
1512                     break
1513         except Exception as e:
1514             logging.warning(_("Failed reading {path}: {error}")
1515                             .format(path=icon_path, error=e))
1516         finally:
1517             if im and hasattr(im, 'close'):
1518                 im.close()
1519
1520     if apk['icons']:
1521         apk['icon'] = icon_filename
1522
1523     return empty_densities
1524
1525
1526 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1527     """
1528     Resize existing icons for densities missing in the APK to ensure all densities are available
1529
1530     :param empty_densities: A list of icon densities that are missing
1531     :param icon_filename: A string representing the icon's file name
1532     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1533     :param repo_dir: The directory of the APK's repository
1534     """
1535     # First try resizing down to not lose quality
1536     last_density = None
1537     for density in screen_densities:
1538         if density not in empty_densities:
1539             last_density = density
1540             continue
1541         if last_density is None:
1542             continue
1543         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1544
1545         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1546         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1547         fp = None
1548         try:
1549             fp = open(last_icon_path, 'rb')
1550             im = Image.open(fp)
1551
1552             size = dpi_to_px(density)
1553
1554             im.thumbnail((size, size), Image.ANTIALIAS)
1555             im.save(icon_path, "PNG", optimize=True,
1556                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1557             empty_densities.remove(density)
1558         except Exception as e:
1559             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1560         finally:
1561             if fp:
1562                 fp.close()
1563
1564     # Then just copy from the highest resolution available
1565     last_density = None
1566     for density in reversed(screen_densities):
1567         if density not in empty_densities:
1568             last_density = density
1569             continue
1570
1571         if last_density is None:
1572             continue
1573
1574         shutil.copyfile(
1575             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1576             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1577         )
1578         empty_densities.remove(density)
1579
1580     for density in screen_densities:
1581         icon_dir = get_icon_dir(repo_dir, density)
1582         icon_dest = os.path.join(icon_dir, icon_filename)
1583         resize_icon(icon_dest, density)
1584
1585     # Copy from icons-mdpi to icons since mdpi is the baseline density
1586     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1587     if os.path.isfile(baseline):
1588         apk['icons']['0'] = icon_filename
1589         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1590
1591
1592 def apply_info_from_latest_apk(apps, apks):
1593     """
1594     Some information from the apks needs to be applied up to the application level.
1595     When doing this, we use the info from the most recent version's apk.
1596     We deal with figuring out when the app was added and last updated at the same time.
1597     """
1598     for appid, app in apps.items():
1599         bestver = UNSET_VERSION_CODE
1600         for apk in apks:
1601             if apk['packageName'] == appid:
1602                 if apk['versionCode'] > bestver:
1603                     bestver = apk['versionCode']
1604                     bestapk = apk
1605
1606                 if 'added' in apk:
1607                     if not app.added or apk['added'] < app.added:
1608                         app.added = apk['added']
1609                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1610                         app.lastUpdated = apk['added']
1611
1612         if not app.added:
1613             logging.debug("Don't know when " + appid + " was added")
1614         if not app.lastUpdated:
1615             logging.debug("Don't know when " + appid + " was last updated")
1616
1617         if bestver == UNSET_VERSION_CODE:
1618
1619             if app.Name is None:
1620                 app.Name = app.AutoName or appid
1621             app.icon = None
1622             logging.debug("Application " + appid + " has no packages")
1623         else:
1624             if app.Name is None:
1625                 app.Name = bestapk['name']
1626             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1627             if app.CurrentVersionCode is None:
1628                 app.CurrentVersionCode = str(bestver)
1629
1630
1631 def make_categories_txt(repodir, categories):
1632     '''Write a category list in the repo to allow quick access'''
1633     catdata = ''
1634     for cat in sorted(categories):
1635         catdata += cat + '\n'
1636     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1637         f.write(catdata)
1638
1639
1640 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1641
1642     def filter_apk_list_sorted(apk_list):
1643         res = []
1644         for apk in apk_list:
1645             if apk['packageName'] == appid:
1646                 res.append(apk)
1647
1648         # Sort the apk list by version code. First is highest/newest.
1649         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1650
1651     for appid, app in apps.items():
1652
1653         if app.ArchivePolicy:
1654             keepversions = int(app.ArchivePolicy[:-9])
1655         else:
1656             keepversions = defaultkeepversions
1657
1658         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1659                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1660
1661         current_app_apks = filter_apk_list_sorted(apks)
1662         if len(current_app_apks) > keepversions:
1663             # Move back the ones we don't want.
1664             for apk in current_app_apks[keepversions:]:
1665                 move_apk_between_sections(repodir, archivedir, apk)
1666                 archapks.append(apk)
1667                 apks.remove(apk)
1668
1669         current_app_archapks = filter_apk_list_sorted(archapks)
1670         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1671             kept = 0
1672             # Move forward the ones we want again, except DisableAlgorithm
1673             for apk in current_app_archapks:
1674                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1675                     move_apk_between_sections(archivedir, repodir, apk)
1676                     archapks.remove(apk)
1677                     apks.append(apk)
1678                     kept += 1
1679                 if kept == keepversions:
1680                     break
1681
1682
1683 def move_apk_between_sections(from_dir, to_dir, apk):
1684     """move an APK from repo to archive or vice versa"""
1685
1686     def _move_file(from_dir, to_dir, filename, ignore_missing):
1687         from_path = os.path.join(from_dir, filename)
1688         if ignore_missing and not os.path.exists(from_path):
1689             return
1690         to_path = os.path.join(to_dir, filename)
1691         if not os.path.exists(to_dir):
1692             os.mkdir(to_dir)
1693         shutil.move(from_path, to_path)
1694
1695     if from_dir == to_dir:
1696         return
1697
1698     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1699     _move_file(from_dir, to_dir, apk['apkName'], False)
1700     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1701     for density in all_screen_densities:
1702         from_icon_dir = get_icon_dir(from_dir, density)
1703         to_icon_dir = get_icon_dir(to_dir, density)
1704         if density not in apk.get('icons', []):
1705             continue
1706         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1707     if 'srcname' in apk:
1708         _move_file(from_dir, to_dir, apk['srcname'], False)
1709
1710
1711 def add_apks_to_per_app_repos(repodir, apks):
1712     apks_per_app = dict()
1713     for apk in apks:
1714         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1715         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1716         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1717         apks_per_app[apk['packageName']] = apk
1718
1719         if not os.path.exists(apk['per_app_icons']):
1720             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1721             os.makedirs(apk['per_app_icons'])
1722
1723         apkpath = os.path.join(repodir, apk['apkName'])
1724         shutil.copy(apkpath, apk['per_app_repo'])
1725         apksigpath = apkpath + '.sig'
1726         if os.path.exists(apksigpath):
1727             shutil.copy(apksigpath, apk['per_app_repo'])
1728         apkascpath = apkpath + '.asc'
1729         if os.path.exists(apkascpath):
1730             shutil.copy(apkascpath, apk['per_app_repo'])
1731
1732
1733 def create_metadata_from_template(apk):
1734     '''create a new metadata file using internal or external template
1735
1736     Generate warnings for apk's with no metadata (or create skeleton
1737     metadata files, if requested on the command line).  Though the
1738     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1739     since those impose things on the metadata file made from the
1740     template: field sort order, empty field value, formatting, etc.
1741     '''
1742
1743     import yaml
1744     if os.path.exists('template.yml'):
1745         with open('template.yml') as f:
1746             metatxt = f.read()
1747         if 'name' in apk and apk['name'] != '':
1748             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1749                              r'\1 ' + apk['name'],
1750                              metatxt,
1751                              flags=re.IGNORECASE | re.MULTILINE)
1752         else:
1753             logging.warning(_('{appid} does not have a name! Using package name instead.')
1754                             .format(appid=apk['packageName']))
1755             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1756                              r'\1 ' + apk['packageName'],
1757                              metatxt,
1758                              flags=re.IGNORECASE | re.MULTILINE)
1759         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1760             f.write(metatxt)
1761     else:
1762         app = dict()
1763         app['Categories'] = [os.path.basename(os.getcwd())]
1764         # include some blanks as part of the template
1765         app['AuthorName'] = ''
1766         app['Summary'] = ''
1767         app['WebSite'] = ''
1768         app['IssueTracker'] = ''
1769         app['SourceCode'] = ''
1770         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1771         if 'name' in apk and apk['name'] != '':
1772             app['Name'] = apk['name']
1773         else:
1774             logging.warning(_('{appid} does not have a name! Using package name instead.')
1775                             .format(appid=apk['packageName']))
1776             app['Name'] = apk['packageName']
1777         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1778             yaml.dump(app, f, default_flow_style=False)
1779     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1780
1781
1782 config = None
1783 options = None
1784 start_timestamp = time.gmtime()
1785
1786
1787 def main():
1788
1789     global config, options
1790
1791     # Parse command line...
1792     parser = ArgumentParser()
1793     common.setup_global_opts(parser)
1794     parser.add_argument("--create-key", action="store_true", default=False,
1795                         help=_("Add a repo signing key to an unsigned repo"))
1796     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1797                         help=_("Add skeleton metadata files for APKs that are missing them"))
1798     parser.add_argument("--delete-unknown", action="store_true", default=False,
1799                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1800     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1801                         help=_("Report on build data status"))
1802     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1803                         help=_("Interactively ask about things that need updating."))
1804     parser.add_argument("-I", "--icons", action="store_true", default=False,
1805                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1806     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1807                         help=_("Specify editor to use in interactive mode. Default " +
1808                                "is {path}").format(path='/etc/alternatives/editor'))
1809     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1810                         help=_("Update the wiki"))
1811     parser.add_argument("--pretty", action="store_true", default=False,
1812                         help=_("Produce human-readable XML/JSON for index files"))
1813     parser.add_argument("--clean", action="store_true", default=False,
1814                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1815     parser.add_argument("--nosign", action="store_true", default=False,
1816                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1817     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1818                         help=_("Use date from APK instead of current time for newly added APKs"))
1819     parser.add_argument("--rename-apks", action="store_true", default=False,
1820                         help=_("Rename APK files that do not match package.name_123.apk"))
1821     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1822                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1823     metadata.add_metadata_arguments(parser)
1824     options = parser.parse_args()
1825     metadata.warnings_action = options.W
1826
1827     config = common.read_config(options)
1828
1829     if not ('jarsigner' in config and 'keytool' in config):
1830         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1831
1832     repodirs = ['repo']
1833     if config['archive_older'] != 0:
1834         repodirs.append('archive')
1835         if not os.path.exists('archive'):
1836             os.mkdir('archive')
1837
1838     if options.icons:
1839         resize_all_icons(repodirs)
1840         sys.exit(0)
1841
1842     if options.rename_apks:
1843         options.clean = True
1844
1845     # check that icons exist now, rather than fail at the end of `fdroid update`
1846     for k in ['repo_icon', 'archive_icon']:
1847         if k in config:
1848             if not os.path.exists(config[k]):
1849                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1850                                  .format(name=k, path=config[k]))
1851                 sys.exit(1)
1852
1853     # if the user asks to create a keystore, do it now, reusing whatever it can
1854     if options.create_key:
1855         if os.path.exists(config['keystore']):
1856             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1857             logging.critical("\t'" + config['keystore'] + "'")
1858             sys.exit(1)
1859
1860         if 'repo_keyalias' not in config:
1861             config['repo_keyalias'] = socket.getfqdn()
1862             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1863         if 'keydname' not in config:
1864             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1865             common.write_to_config(config, 'keydname', config['keydname'])
1866         if 'keystore' not in config:
1867             config['keystore'] = common.default_config['keystore']
1868             common.write_to_config(config, 'keystore', config['keystore'])
1869
1870         password = common.genpassword()
1871         if 'keystorepass' not in config:
1872             config['keystorepass'] = password
1873             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1874         if 'keypass' not in config:
1875             config['keypass'] = password
1876             common.write_to_config(config, 'keypass', config['keypass'])
1877         common.genkeystore(config)
1878
1879     # Get all apps...
1880     apps = metadata.read_metadata()
1881
1882     # Generate a list of categories...
1883     categories = set()
1884     for app in apps.values():
1885         categories.update(app.Categories)
1886
1887     # Read known apks data (will be updated and written back when we've finished)
1888     knownapks = common.KnownApks()
1889
1890     # Get APK cache
1891     apkcache = get_cache()
1892
1893     # Delete builds for disabled apps
1894     delete_disabled_builds(apps, apkcache, repodirs)
1895
1896     # Scan all apks in the main repo
1897     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1898
1899     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1900                                            options.use_date_from_apk)
1901     cachechanged = cachechanged or fcachechanged
1902     apks += files
1903     for apk in apks:
1904         if apk['packageName'] not in apps:
1905             if options.create_metadata:
1906                 create_metadata_from_template(apk)
1907                 apps = metadata.read_metadata()
1908             else:
1909                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1910                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1911                 if options.delete_unknown:
1912                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1913                                  .format(apkfilename=apk['apkName']))
1914                     rmf = os.path.join(repodirs[0], apk['apkName'])
1915                     if not os.path.exists(rmf):
1916                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1917                     else:
1918                         os.remove(rmf)
1919                 else:
1920                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1921
1922     copy_triple_t_store_metadata(apps)
1923     insert_obbs(repodirs[0], apps, apks)
1924     insert_localized_app_metadata(apps)
1925     translate_per_build_anti_features(apps, apks)
1926
1927     # Scan the archive repo for apks as well
1928     if len(repodirs) > 1:
1929         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1930         if cc:
1931             cachechanged = True
1932     else:
1933         archapks = []
1934
1935     # Apply information from latest apks to the application and update dates
1936     apply_info_from_latest_apk(apps, apks + archapks)
1937
1938     # Sort the app list by name, then the web site doesn't have to by default.
1939     # (we had to wait until we'd scanned the apks to do this, because mostly the
1940     # name comes from there!)
1941     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1942
1943     # APKs are placed into multiple repos based on the app package, providing
1944     # per-app subscription feeds for nightly builds and things like it
1945     if config['per_app_repos']:
1946         add_apks_to_per_app_repos(repodirs[0], apks)
1947         for appid, app in apps.items():
1948             repodir = os.path.join(appid, 'fdroid', 'repo')
1949             appdict = dict()
1950             appdict[appid] = app
1951             if os.path.isdir(repodir):
1952                 index.make(appdict, [appid], apks, repodir, False)
1953             else:
1954                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1955         return
1956
1957     if len(repodirs) > 1:
1958         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1959
1960     # Make the index for the main repo...
1961     index.make(apps, sortedids, apks, repodirs[0], False)
1962     make_categories_txt(repodirs[0], categories)
1963
1964     # If there's an archive repo,  make the index for it. We already scanned it
1965     # earlier on.
1966     if len(repodirs) > 1:
1967         index.make(apps, sortedids, archapks, repodirs[1], True)
1968
1969     git_remote = config.get('binary_transparency_remote')
1970     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1971         from . import btlog
1972         btlog.make_binary_transparency_log(repodirs)
1973
1974     if config['update_stats']:
1975         # Update known apks info...
1976         knownapks.writeifchanged()
1977
1978         # Generate latest apps data for widget
1979         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1980             data = ''
1981             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1982                 for line in f:
1983                     appid = line.rstrip()
1984                     data += appid + "\t"
1985                     app = apps[appid]
1986                     data += app.Name + "\t"
1987                     if app.icon is not None:
1988                         data += app.icon + "\t"
1989                     data += app.License + "\n"
1990             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1991                 f.write(data)
1992
1993     if cachechanged:
1994         write_cache(apkcache)
1995
1996     # Update the wiki...
1997     if options.wiki:
1998         update_wiki(apps, sortedids, apks + archapks)
1999
2000     logging.info(_("Finished"))
2001
2002
2003 if __name__ == "__main__":
2004     main()