fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0':
 100         return os.path.join(repodir, "icons")
 101     return os.path.join(repodir, "icons-%s" % density)
 102
 103
 104 def get_icon_dirs(repodir):
 105     for density in screen_densities:
 106         yield get_icon_dir(repodir, density)
 107
 108
 109 def get_all_icon_dirs(repodir):
 110     for density in all_screen_densities:
 111         yield get_icon_dir(repodir, density)
 112
 113
 114 def update_wiki(apps, sortedids, apks):
 115     """Update the wiki
 116
 117     :param apps: fully populated list of all applications
 118     :param apks: all apks, except...
 119     """
 120     logging.info("Updating wiki")
 121     wikicat = 'Apps'
 122     wikiredircat = 'App Redirects'
 123     import mwclient
 124     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 125                          path=config['wiki_path'])
 126     site.login(config['wiki_user'], config['wiki_password'])
 127     generated_pages = {}
 128     generated_redirects = {}
 129
 130     for appid in sortedids:
 131         app = metadata.App(apps[appid])
 132
 133         wikidata = ''
 134         if app.Disabled:
 135             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 136         if app.AntiFeatures:
 137             for af in sorted(app.AntiFeatures):
 138                 wikidata += '{{AntiFeature|' + af + '}}\n'
 139         if app.RequiresRoot:
 140             requiresroot = 'Yes'
 141         else:
 142             requiresroot = 'No'
 143         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 144             appid,
 145             app.Name,
 146             app.added.strftime('%Y-%m-%d') if app.added else '',
 147             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 148             app.SourceCode,
 149             app.IssueTracker,
 150             app.WebSite,
 151             app.Changelog,
 152             app.Donate,
 153             app.FlattrID,
 154             app.LiberapayID,
 155             app.Bitcoin,
 156             app.Litecoin,
 157             app.License,
 158             requiresroot,
 159             app.AuthorName,
 160             app.AuthorEmail)
 161
 162         if app.Provides:
 163             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 164
 165         wikidata += app.Summary
 166         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 167
 168         wikidata += "=Description=\n"
 169         wikidata += metadata.description_wiki(app.Description) + "\n"
 170
 171         wikidata += "=Maintainer Notes=\n"
 172         if app.MaintainerNotes:
 173             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 174         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 175
 176         # Get a list of all packages for this application...
 177         apklist = []
 178         gotcurrentver = False
 179         cantupdate = False
 180         buildfails = False
 181         for apk in apks:
 182             if apk['packageName'] == appid:
 183                 if str(apk['versionCode']) == app.CurrentVersionCode:
 184                     gotcurrentver = True
 185                 apklist.append(apk)
 186         # Include ones we can't build, as a special case...
 187         for build in app.builds:
 188             if build.disable:
 189                 if build.versionCode == app.CurrentVersionCode:
 190                     cantupdate = True
 191                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 192                 apklist.append({'versionCode': int(build.versionCode),
 193                                 'versionName': build.versionName,
 194                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 195                                 })
 196             else:
 197                 builtit = False
 198                 for apk in apklist:
 199                     if apk['versionCode'] == int(build.versionCode):
 200                         builtit = True
 201                         break
 202                 if not builtit:
 203                     buildfails = True
 204                     apklist.append({'versionCode': int(build.versionCode),
 205                                     'versionName': build.versionName,
 206                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 207                                     })
 208         if app.CurrentVersionCode == '0':
 209             cantupdate = True
 210         # Sort with most recent first...
 211         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 212
 213         wikidata += "=Versions=\n"
 214         if len(apklist) == 0:
 215             wikidata += "We currently have no versions of this app available."
 216         elif not gotcurrentver:
 217             wikidata += "We don't have the current version of this app."
 218         else:
 219             wikidata += "We have the current version of this app."
 220         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 221         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 222         if len(app.NoSourceSince) > 0:
 223             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 224         if len(app.CurrentVersion) > 0:
 225             wikidata += "The current (recommended) version is " + app.CurrentVersion
 226             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 227         validapks = 0
 228         for apk in apklist:
 229             wikidata += "==" + apk['versionName'] + "==\n"
 230
 231             if 'buildproblem' in apk:
 232                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 233             else:
 234                 validapks += 1
 235                 wikidata += "This version is built and signed by "
 236                 if 'srcname' in apk:
 237                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 238                 else:
 239                     wikidata += "the original developer.\n\n"
 240             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 241
 242         wikidata += '\n[[Category:' + wikicat + ']]\n'
 243         if len(app.NoSourceSince) > 0:
 244             wikidata += '\n[[Category:Apps missing source code]]\n'
 245         if validapks == 0 and not app.Disabled:
 246             wikidata += '\n[[Category:Apps with no packages]]\n'
 247         if cantupdate and not app.Disabled:
 248             wikidata += "\n[[Category:Apps we cannot update]]\n"
 249         if buildfails and not app.Disabled:
 250             wikidata += "\n[[Category:Apps with failing builds]]\n"
 251         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 252             wikidata += '\n[[Category:Apps to Update]]\n'
 253         if app.Disabled:
 254             wikidata += '\n[[Category:Apps that are disabled]]\n'
 255         if app.UpdateCheckMode == 'None' and not app.Disabled:
 256             wikidata += '\n[[Category:Apps with no update check]]\n'
 257         for appcat in app.Categories:
 258             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 259
 260         # We can't have underscores in the page name, even if they're in
 261         # the package ID, because MediaWiki messes with them...
 262         pagename = appid.replace('_', ' ')
 263
 264         # Drop a trailing newline, because mediawiki is going to drop it anyway
 265         # and it we don't we'll think the page has changed when it hasn't...
 266         if wikidata.endswith('\n'):
 267             wikidata = wikidata[:-1]
 268
 269         generated_pages[pagename] = wikidata
 270
 271         # Make a redirect from the name to the ID too, unless there's
 272         # already an existing page with the name and it isn't a redirect.
 273         noclobber = False
 274         apppagename = app.Name.replace('_', ' ')
 275         apppagename = apppagename.replace('{', '')
 276         apppagename = apppagename.replace('}', ' ')
 277         apppagename = apppagename.replace(':', ' ')
 278         apppagename = apppagename.replace('[', ' ')
 279         apppagename = apppagename.replace(']', ' ')
 280         # Drop double spaces caused mostly by replacing ':' above
 281         apppagename = apppagename.replace('  ', ' ')
 282         for expagename in site.allpages(prefix=apppagename,
 283                                         filterredir='nonredirects',
 284                                         generator=False):
 285             if expagename == apppagename:
 286                 noclobber = True
 287         # Another reason not to make the redirect page is if the app name
 288         # is the same as it's ID, because that will overwrite the real page
 289         # with an redirect to itself! (Although it seems like an odd
 290         # scenario this happens a lot, e.g. where there is metadata but no
 291         # builds or binaries to extract a name from.
 292         if apppagename == pagename:
 293             noclobber = True
 294         if not noclobber:
 295             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 296
 297     for tcat, genp in [(wikicat, generated_pages),
 298                        (wikiredircat, generated_redirects)]:
 299         catpages = site.Pages['Category:' + tcat]
 300         existingpages = []
 301         for page in catpages:
 302             existingpages.append(page.name)
 303             if page.name in genp:
 304                 pagetxt = page.edit()
 305                 if pagetxt != genp[page.name]:
 306                     logging.debug("Updating modified page " + page.name)
 307                     page.save(genp[page.name], summary='Auto-updated')
 308                 else:
 309                     logging.debug("Page " + page.name + " is unchanged")
 310             else:
 311                 logging.warn("Deleting page " + page.name)
 312                 page.delete('No longer published')
 313         for pagename, text in genp.items():
 314             logging.debug("Checking " + pagename)
 315             if pagename not in existingpages:
 316                 logging.debug("Creating page " + pagename)
 317                 try:
 318                     newpage = site.Pages[pagename]
 319                     newpage.save(text, summary='Auto-created')
 320                 except Exception as e:
 321                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 322
 323     # Purge server cache to ensure counts are up to date
 324     site.Pages['Repository Maintenance'].purge()
 325
 326     # Write a page with the last build log for this version code
 327     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 328     newpage = site.Pages[wiki_page_path]
 329     txt = ''
 330     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 331     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 332     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 333     txt += "\n\n"
 334     newpage.save(txt, summary='Run log')
 335     newpage = site.Pages['update']
 336     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 337
 338
 339 def delete_disabled_builds(apps, apkcache, repodirs):
 340     """Delete disabled build outputs.
 341
 342     :param apps: list of all applications, as per metadata.read_metadata
 343     :param apkcache: current apk cache information
 344     :param repodirs: the repo directories to process
 345     """
 346     for appid, app in apps.items():
 347         for build in app['builds']:
 348             if not build.disable:
 349                 continue
 350             apkfilename = common.get_release_filename(app, build)
 351             iconfilename = "%s.%s.png" % (
 352                 appid,
 353                 build.versionCode)
 354             for repodir in repodirs:
 355                 files = [
 356                     os.path.join(repodir, apkfilename),
 357                     os.path.join(repodir, apkfilename + '.asc'),
 358                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 359                 ]
 360                 for density in all_screen_densities:
 361                     repo_dir = get_icon_dir(repodir, density)
 362                     files.append(os.path.join(repo_dir, iconfilename))
 363
 364                 for f in files:
 365                     if os.path.exists(f):
 366                         logging.info("Deleting disabled build output " + f)
 367                         os.remove(f)
 368             if apkfilename in apkcache:
 369                 del apkcache[apkfilename]
 370
 371
 372 def resize_icon(iconpath, density):
 373
 374     if not os.path.isfile(iconpath):
 375         return
 376
 377     fp = None
 378     try:
 379         fp = open(iconpath, 'rb')
 380         im = Image.open(fp)
 381         size = dpi_to_px(density)
 382
 383         if any(length > size for length in im.size):
 384             oldsize = im.size
 385             im.thumbnail((size, size), Image.ANTIALIAS)
 386             logging.debug("%s was too large at %s - new size is %s" % (
 387                 iconpath, oldsize, im.size))
 388             im.save(iconpath, "PNG", optimize=True,
 389                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 390
 391     except Exception as e:
 392         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 393
 394     finally:
 395         if fp:
 396             fp.close()
 397
 398
 399 def resize_all_icons(repodirs):
 400     """Resize all icons that exceed the max size
 401
 402     :param repodirs: the repo directories to process
 403     """
 404     for repodir in repodirs:
 405         for density in screen_densities:
 406             icon_dir = get_icon_dir(repodir, density)
 407             icon_glob = os.path.join(icon_dir, '*.png')
 408             for iconpath in glob.glob(icon_glob):
 409                 resize_icon(iconpath, density)
 410
 411
 412 def getsig(apkpath):
 413     """ Get the signing certificate of an apk. To get the same md5 has that
 414     Android gets, we encode the .RSA certificate in a specific format and pass
 415     it hex-encoded to the md5 digest algorithm.
 416
 417     :param apkpath: path to the apk
 418     :returns: A string containing the md5 of the signature of the apk or None
 419               if an error occurred.
 420     """
 421
 422     with zipfile.ZipFile(apkpath, 'r') as apk:
 423         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 424
 425         if len(certs) < 1:
 426             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 427             return None
 428         if len(certs) > 1:
 429             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 430             return None
 431
 432         cert = apk.read(certs[0])
 433
 434     cert_encoded = common.get_certificate(cert)
 435
 436     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 437
 438
 439 def get_cache_file():
 440     return os.path.join('tmp', 'apkcache')
 441
 442
 443 def get_cache():
 444     """Get the cached dict of the APK index
 445
 446     Gather information about all the apk files in the repo directory,
 447     using cached data if possible. Some of the index operations take a
 448     long time, like calculating the SHA-256 and verifying the APK
 449     signature.
 450
 451     The cache is invalidated if the metadata version is different, or
 452     the 'allow_disabled_algorithms' config/option is different.  In
 453     those cases, there is no easy way to know what has changed from
 454     the cache, so just rerun the whole thing.
 455
 456     :return: apkcache
 457
 458     """
 459     apkcachefile = get_cache_file()
 460     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 461     if not options.clean and os.path.exists(apkcachefile):
 462         with open(apkcachefile, 'rb') as cf:
 463             apkcache = pickle.load(cf, encoding='utf-8')
 464         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 465            or apkcache.get('allow_disabled_algorithms') != ada:
 466             apkcache = {}
 467     else:
 468         apkcache = {}
 469
 470     apkcache["METADATA_VERSION"] = METADATA_VERSION
 471     apkcache['allow_disabled_algorithms'] = ada
 472
 473     return apkcache
 474
 475
 476 def write_cache(apkcache):
 477     apkcachefile = get_cache_file()
 478     cache_path = os.path.dirname(apkcachefile)
 479     if not os.path.exists(cache_path):
 480         os.makedirs(cache_path)
 481     with open(apkcachefile, 'wb') as cf:
 482         pickle.dump(apkcache, cf)
 483
 484
 485 def get_icon_bytes(apkzip, iconsrc):
 486     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 487     try:
 488         return apkzip.read(iconsrc)
 489     except KeyError:
 490         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 491
 492
 493 def sha256sum(filename):
 494     '''Calculate the sha256 of the given file'''
 495     sha = hashlib.sha256()
 496     with open(filename, 'rb') as f:
 497         while True:
 498             t = f.read(16384)
 499             if len(t) == 0:
 500                 break
 501             sha.update(t)
 502     return sha.hexdigest()
 503
 504
 505 def has_known_vulnerability(filename):
 506     """checks for known vulnerabilities in the APK
 507
 508     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 509     version.  Google also enforces this:
 510     https://support.google.com/faqs/answer/6376725?hl=en
 511
 512     Checks whether there are more than one classes.dex or AndroidManifest.xml
 513     files, which is invalid and an essential part of the "Master Key" attack.
 514     http://www.saurik.com/id/17
 515
 516     Janus is similar to Master Key but is perhaps easier to scan for.
 517     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 518     """
 519
 520     found_vuln = False
 521
 522     # statically load this pattern
 523     if not hasattr(has_known_vulnerability, "pattern"):
 524         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 525
 526     with open(filename.encode(), 'rb') as fp:
 527         first4 = fp.read(4)
 528     if first4 != b'\x50\x4b\x03\x04':
 529         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 530                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 531                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 532
 533     files_in_apk = set()
 534     with zipfile.ZipFile(filename) as zf:
 535         for name in zf.namelist():
 536             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 537                 lib = zf.open(name)
 538                 while True:
 539                     chunk = lib.read(4096)
 540                     if chunk == b'':
 541                         break
 542                     m = has_known_vulnerability.pattern.search(chunk)
 543                     if m:
 544                         version = m.group(1).decode('ascii')
 545                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 546                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 547                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 548                             logging.debug(_('"{path}" contains recent {name} ({version})')
 549                                           .format(path=filename, name=name, version=version))
 550                         else:
 551                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 552                                             .format(path=filename, name=name, version=version))
 553                             found_vuln = True
 554                         break
 555             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 556                 if name in files_in_apk:
 557                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 558                                     .format(apkfilename=filename, name=name))
 559                     found_vuln = True
 560                 files_in_apk.add(name)
 561     return found_vuln
 562
 563
 564 def insert_obbs(repodir, apps, apks):
 565     """Scans the .obb files in a given repo directory and adds them to the
 566     relevant APK instances.  OBB files have versionCodes like APK
 567     files, and they are loosely associated.  If there is an OBB file
 568     present, then any APK with the same or higher versionCode will use
 569     that OBB file.  There are two OBB types: main and patch, each APK
 570     can only have only have one of each.
 571
 572     https://developer.android.com/google/play/expansion-files.html
 573
 574     :param repodir: repo directory to scan
 575     :param apps: list of current, valid apps
 576     :param apks: current information on all APKs
 577
 578     """
 579
 580     def obbWarnDelete(f, msg):
 581         logging.warning(msg + ' ' + f)
 582         if options.delete_unknown:
 583             logging.error(_("Deleting unknown file: {path}").format(path=f))
 584             os.remove(f)
 585
 586     obbs = []
 587     java_Integer_MIN_VALUE = -pow(2, 31)
 588     currentPackageNames = apps.keys()
 589     for f in glob.glob(os.path.join(repodir, '*.obb')):
 590         obbfile = os.path.basename(f)
 591         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 592         chunks = obbfile.split('.')
 593         if chunks[0] != 'main' and chunks[0] != 'patch':
 594             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 595             continue
 596         if not re.match(r'^-?[0-9]+$', chunks[1]):
 597             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 598                           .format(name=chunks[0]))
 599             continue
 600         versionCode = int(chunks[1])
 601         packagename = ".".join(chunks[2:-1])
 602
 603         highestVersionCode = java_Integer_MIN_VALUE
 604         if packagename not in currentPackageNames:
 605             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 606             continue
 607         for apk in apks:
 608             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 609                 highestVersionCode = apk['versionCode']
 610         if versionCode > highestVersionCode:
 611             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 612                           .format(integer=str(versionCode)))
 613             continue
 614         obbsha256 = sha256sum(f)
 615         obbs.append((packagename, versionCode, obbfile, obbsha256))
 616
 617     for apk in apks:
 618         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 619             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 620                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 621                     apk['obbMainFile'] = obbfile
 622                     apk['obbMainFileSha256'] = obbsha256
 623                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 624                     apk['obbPatchFile'] = obbfile
 625                     apk['obbPatchFileSha256'] = obbsha256
 626             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 627                 break
 628
 629
 630 def translate_per_build_anti_features(apps, apks):
 631     """Grab the anti-features list from the build metadata
 632
 633     For most Anti-Features, they are really most applicable per-APK,
 634     not for an app.  An app can fix a vulnerability, add/remove
 635     tracking, etc.  This reads the 'antifeatures' list from the Build
 636     entries in the fdroiddata metadata file, then transforms it into
 637     the 'antiFeatures' list of unique items for the index.
 638
 639     The field key is all lower case in the metadata file to match the
 640     rest of the Build fields.  It is 'antiFeatures' camel case in the
 641     implementation, index, and fdroidclient since it is translated
 642     from the build 'antifeatures' field, not directly included.
 643
 644     """
 645
 646     antiFeatures = dict()
 647     for packageName, app in apps.items():
 648         d = dict()
 649         for build in app['builds']:
 650             afl = build.get('antifeatures')
 651             if afl:
 652                 d[int(build.versionCode)] = afl
 653         if len(d) > 0:
 654             antiFeatures[packageName] = d
 655
 656     for apk in apks:
 657         d = antiFeatures.get(apk['packageName'])
 658         if d:
 659             afl = d.get(apk['versionCode'])
 660             if afl:
 661                 apk['antiFeatures'].update(afl)
 662
 663
 664 def _get_localized_dict(app, locale):
 665     '''get the dict to add localized store metadata to'''
 666     if 'localized' not in app:
 667         app['localized'] = collections.OrderedDict()
 668     if locale not in app['localized']:
 669         app['localized'][locale] = collections.OrderedDict()
 670     return app['localized'][locale]
 671
 672
 673 def _set_localized_text_entry(app, locale, key, f):
 674     limit = config['char_limits'][key]
 675     localized = _get_localized_dict(app, locale)
 676     with open(f) as fp:
 677         text = fp.read()[:limit]
 678         if len(text) > 0:
 679             localized[key] = text
 680
 681
 682 def _set_author_entry(app, key, f):
 683     limit = config['char_limits']['author']
 684     with open(f) as fp:
 685         text = fp.read()[:limit]
 686         if len(text) > 0:
 687             app[key] = text
 688
 689
 690 def _strip_and_copy_image(inpath, outpath):
 691     """Remove any metadata from image and copy it to new path
 692
 693     Sadly, image metadata like EXIF can be used to exploit devices.
 694     It is not used at all in the F-Droid ecosystem, so its much safer
 695     just to remove it entirely.
 696
 697     """
 698
 699     extension = common.get_extension(inpath)[1]
 700     if os.path.isdir(outpath):
 701         outpath = os.path.join(outpath, os.path.basename(inpath))
 702     if extension == 'png':
 703         with open(inpath, 'rb') as fp:
 704             in_image = Image.open(fp)
 705             in_image.save(outpath, "PNG", optimize=True,
 706                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 707     elif extension == 'jpg' or extension == 'jpeg':
 708         with open(inpath, 'rb') as fp:
 709             in_image = Image.open(fp)
 710             data = list(in_image.getdata())
 711             out_image = Image.new(in_image.mode, in_image.size)
 712         out_image.putdata(data)
 713         out_image.save(outpath, "JPEG", optimize=True)
 714     else:
 715         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 716                               .format(extension=extension))
 717
 718
 719 def copy_triple_t_store_metadata(apps):
 720     """Include store metadata from the app's source repo
 721
 722     The Triple-T Gradle Play Publisher is a plugin that has a standard
 723     file layout for all of the metadata and graphics that the Google
 724     Play Store accepts.  Since F-Droid has the git repo, it can just
 725     pluck those files directly.  This method reads any text files into
 726     the app dict, then copies any graphics into the fdroid repo
 727     directory structure.
 728
 729     This needs to be run before insert_localized_app_metadata() so that
 730     the graphics files that are copied into the fdroid repo get
 731     properly indexed.
 732
 733     https://github.com/Triple-T/gradle-play-publisher#upload-images
 734     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 735
 736     """
 737
 738     if not os.path.isdir('build'):
 739         return  # nothing to do
 740
 741     for packageName, app in apps.items():
 742         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 743             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 744             for root, dirs, files in os.walk(d):
 745                 segments = root.split('/')
 746                 locale = segments[-2]
 747                 for f in files:
 748                     if f == 'fulldescription':
 749                         _set_localized_text_entry(app, locale, 'description',
 750                                                   os.path.join(root, f))
 751                         continue
 752                     elif f == 'shortdescription':
 753                         _set_localized_text_entry(app, locale, 'summary',
 754                                                   os.path.join(root, f))
 755                         continue
 756                     elif f == 'title':
 757                         _set_localized_text_entry(app, locale, 'name',
 758                                                   os.path.join(root, f))
 759                         continue
 760                     elif f == 'video':
 761                         _set_localized_text_entry(app, locale, 'video',
 762                                                   os.path.join(root, f))
 763                         continue
 764                     elif f == 'whatsnew':
 765                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 766                                                   os.path.join(root, f))
 767                         continue
 768                     elif f == 'contactEmail':
 769                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 770                         continue
 771                     elif f == 'contactPhone':
 772                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 773                         continue
 774                     elif f == 'contactWebsite':
 775                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 776                         continue
 777
 778                     base, extension = common.get_extension(f)
 779                     dirname = os.path.basename(root)
 780                     if extension in ALLOWED_EXTENSIONS \
 781                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 782                         if segments[-2] == 'listing':
 783                             locale = segments[-3]
 784                         else:
 785                             locale = segments[-2]
 786                         destdir = os.path.join('repo', packageName, locale, dirname)
 787                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 788                         sourcefile = os.path.join(root, f)
 789                         destfile = os.path.join(destdir, os.path.basename(f))
 790                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 791                         _strip_and_copy_image(sourcefile, destfile)
 792
 793
 794 def insert_localized_app_metadata(apps):
 795     """scans standard locations for graphics and localized text
 796
 797     Scans for localized description files, store graphics, and
 798     screenshot PNG files in statically defined screenshots directory
 799     and adds them to the app metadata.  The screenshots and graphic
 800     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 801     and must be in the following layout:
 802     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 803
 804     repo/packageName/locale/featureGraphic.png
 805     repo/packageName/locale/phoneScreenshots/1.png
 806     repo/packageName/locale/phoneScreenshots/2.png
 807
 808     The changelog files must be text files named with the versionCode
 809     ending with ".txt" and must be in the following layout:
 810     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 811
 812     repo/packageName/locale/changelogs/12345.txt
 813
 814     This will scan the each app's source repo then the metadata/ dir
 815     for these standard locations of changelog files.  If it finds
 816     them, they will be added to the dict of all packages, with the
 817     versions in the metadata/ folder taking precendence over the what
 818     is in the app's source repo.
 819
 820     Where "packageName" is the app's packageName and "locale" is the locale
 821     of the graphics, e.g. what language they are in, using the IETF RFC5646
 822     format (en-US, fr-CA, es-MX, etc).
 823
 824     This will also scan the app's git for a fastlane folder, and the
 825     metadata/ folder and the apps' source repos for standard locations
 826     of graphic and screenshot files.  If it finds them, it will copy
 827     them into the repo.  The fastlane files follow this pattern:
 828     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 829
 830     """
 831
 832     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 833     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 834     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 836
 837     for srcd in sorted(sourcedirs):
 838         if not os.path.isdir(srcd):
 839             continue
 840         for root, dirs, files in os.walk(srcd):
 841             segments = root.split('/')
 842             packageName = segments[1]
 843             if packageName not in apps:
 844                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 845                 continue
 846             locale = segments[-1]
 847             destdir = os.path.join('repo', packageName, locale)
 848
 849             # flavours specified in build receipt
 850             build_flavours = ""
 851             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 852                     and 'gradle' in apps[packageName].builds[-1]:
 853                 build_flavours = apps[packageName].builds[-1].gradle
 854
 855             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 856                 logging.debug("ignoring due to wrong flavour")
 857                 continue
 858
 859             for f in files:
 860                 if f in ('description.txt', 'full_description.txt'):
 861                     _set_localized_text_entry(apps[packageName], locale, 'description',
 862                                               os.path.join(root, f))
 863                     continue
 864                 elif f in ('summary.txt', 'short_description.txt'):
 865                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 866                                               os.path.join(root, f))
 867                     continue
 868                 elif f in ('name.txt', 'title.txt'):
 869                     _set_localized_text_entry(apps[packageName], locale, 'name',
 870                                               os.path.join(root, f))
 871                     continue
 872                 elif f == 'video.txt':
 873                     _set_localized_text_entry(apps[packageName], locale, 'video',
 874                                               os.path.join(root, f))
 875                     continue
 876                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 877                     locale = segments[-2]
 878                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 879                                               os.path.join(root, f))
 880                     continue
 881
 882                 base, extension = common.get_extension(f)
 883                 if locale == 'images':
 884                     locale = segments[-2]
 885                     destdir = os.path.join('repo', packageName, locale)
 886                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 887                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 888                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 889                     _strip_and_copy_image(os.path.join(root, f), destdir)
 890             for d in dirs:
 891                 if d in SCREENSHOT_DIRS:
 892                     if locale == 'images':
 893                         locale = segments[-2]
 894                         destdir = os.path.join('repo', packageName, locale)
 895                     for f in glob.glob(os.path.join(root, d, '*.*')):
 896                         _ignored, extension = common.get_extension(f)
 897                         if extension in ALLOWED_EXTENSIONS:
 898                             screenshotdestdir = os.path.join(destdir, d)
 899                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 900                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 901                             _strip_and_copy_image(f, screenshotdestdir)
 902
 903     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 904     for d in repofiles:
 905         if not os.path.isdir(d):
 906             continue
 907         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 908             if not os.path.isfile(f):
 909                 continue
 910             segments = f.split('/')
 911             packageName = segments[1]
 912             locale = segments[2]
 913             screenshotdir = segments[3]
 914             filename = os.path.basename(f)
 915             base, extension = common.get_extension(filename)
 916
 917             if packageName not in apps:
 918                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 919                                 .format(path=filename, name=packageName))
 920                 continue
 921             graphics = _get_localized_dict(apps[packageName], locale)
 922
 923             if extension not in ALLOWED_EXTENSIONS:
 924                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 925             elif base in GRAPHIC_NAMES:
 926                 # there can only be zero or one of these per locale
 927                 graphics[base] = filename
 928             elif screenshotdir in SCREENSHOT_DIRS:
 929                 # there can any number of these per locale
 930                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 931                 if screenshotdir not in graphics:
 932                     graphics[screenshotdir] = []
 933                 graphics[screenshotdir].append(filename)
 934             else:
 935                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 936
 937
 938 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 939     """Scan a repo for all files with an extension except APK/OBB
 940
 941     :param apkcache: current cached info about all repo files
 942     :param repodir: repo directory to scan
 943     :param knownapks: list of all known files, as per metadata.read_metadata
 944     :param use_date_from_file: use date from file (instead of current date)
 945                                for newly added files
 946     """
 947
 948     cachechanged = False
 949     repo_files = []
 950     repodir = repodir.encode('utf-8')
 951     for name in os.listdir(repodir):
 952         file_extension = common.get_file_extension(name)
 953         if file_extension == 'apk' or file_extension == 'obb':
 954             continue
 955         filename = os.path.join(repodir, name)
 956         name_utf8 = name.decode('utf-8')
 957         if filename.endswith(b'_src.tar.gz'):
 958             logging.debug(_('skipping source tarball: {path}')
 959                           .format(path=filename.decode('utf-8')))
 960             continue
 961         if not common.is_repo_file(filename):
 962             continue
 963         stat = os.stat(filename)
 964         if stat.st_size == 0:
 965             raise FDroidException(_('{path} is zero size!')
 966                                   .format(path=filename))
 967
 968         shasum = sha256sum(filename)
 969         usecache = False
 970         if name in apkcache:
 971             repo_file = apkcache[name]
 972             # added time is cached as tuple but used here as datetime instance
 973             if 'added' in repo_file:
 974                 a = repo_file['added']
 975                 if isinstance(a, datetime):
 976                     repo_file['added'] = a
 977                 else:
 978                     repo_file['added'] = datetime(*a[:6])
 979             if repo_file.get('hash') == shasum:
 980                 logging.debug(_("Reading {apkfilename} from cache")
 981                               .format(apkfilename=name_utf8))
 982                 usecache = True
 983             else:
 984                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 985                               .format(apkfilename=name_utf8))
 986
 987         if not usecache:
 988             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 989             repo_file = collections.OrderedDict()
 990             repo_file['name'] = os.path.splitext(name_utf8)[0]
 991             # TODO rename apkname globally to something more generic
 992             repo_file['apkName'] = name_utf8
 993             repo_file['hash'] = shasum
 994             repo_file['hashType'] = 'sha256'
 995             repo_file['versionCode'] = 0
 996             repo_file['versionName'] = shasum
 997             # the static ID is the SHA256 unless it is set in the metadata
 998             repo_file['packageName'] = shasum
 999
1000             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1001             if m:
1002                 repo_file['packageName'] = m.group(1)
1003                 repo_file['versionCode'] = int(m.group(2))
1004             srcfilename = name + b'_src.tar.gz'
1005             if os.path.exists(os.path.join(repodir, srcfilename)):
1006                 repo_file['srcname'] = srcfilename.decode('utf-8')
1007             repo_file['size'] = stat.st_size
1008
1009             apkcache[name] = repo_file
1010             cachechanged = True
1011
1012         if use_date_from_file:
1013             timestamp = stat.st_ctime
1014             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1015         else:
1016             default_date_param = None
1017
1018         # Record in knownapks, getting the added date at the same time..
1019         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1020                                     default_date=default_date_param)
1021         if added:
1022             repo_file['added'] = added
1023
1024         repo_files.append(repo_file)
1025
1026     return repo_files, cachechanged
1027
1028
1029 def scan_apk(apk_file):
1030     """
1031     Scans an APK file and returns dictionary with metadata of the APK.
1032
1033     Attention: This does *not* verify that the APK signature is correct.
1034
1035     :param apk_file: The (ideally absolute) path to the APK file
1036     :raises BuildException
1037     :return A dict containing APK metadata
1038     """
1039     apk = {
1040         'hash': sha256sum(apk_file),
1041         'hashType': 'sha256',
1042         'uses-permission': [],
1043         'uses-permission-sdk-23': [],
1044         'features': [],
1045         'icons_src': {},
1046         'icons': {},
1047         'antiFeatures': set(),
1048     }
1049
1050     if SdkToolsPopen(['aapt', 'version'], output=False):
1051         scan_apk_aapt(apk, apk_file)
1052     else:
1053         scan_apk_androguard(apk, apk_file)
1054
1055     # Get the signature, or rather the signing key fingerprints
1056     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1057     apk['sig'] = getsig(apk_file)
1058     if not apk['sig']:
1059         raise BuildException("Failed to get apk signature")
1060     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1061                                                                apk_file))
1062     if not apk.get('signer'):
1063         raise BuildException("Failed to get apk signing key fingerprint")
1064
1065     # Get size of the APK
1066     apk['size'] = os.path.getsize(apk_file)
1067
1068     if 'minSdkVersion' not in apk:
1069         logging.warning("No SDK version information found in {0}".format(apk_file))
1070         apk['minSdkVersion'] = 1
1071
1072     # Check for known vulnerabilities
1073     if has_known_vulnerability(apk_file):
1074         apk['antiFeatures'].add('KnownVuln')
1075
1076     return apk
1077
1078
1079 def scan_apk_aapt(apk, apkfile):
1080     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1081     if p.returncode != 0:
1082         if options.delete_unknown:
1083             if os.path.exists(apkfile):
1084                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1085                 os.remove(apkfile)
1086             else:
1087                 logging.error("Could not find {0} to remove it".format(apkfile))
1088         else:
1089             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1090         raise BuildException(_("Invalid APK"))
1091     for line in p.output.splitlines():
1092         if line.startswith("package:"):
1093             try:
1094                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1095                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1096                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1097             except Exception as e:
1098                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1099         elif line.startswith("application:"):
1100             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1101             # Keep path to non-dpi icon in case we need it
1102             match = re.match(APK_ICON_PAT_NODPI, line)
1103             if match:
1104                 apk['icons_src']['-1'] = match.group(1)
1105         elif line.startswith("launchable-activity:"):
1106             # Only use launchable-activity as fallback to application
1107             if not apk['name']:
1108                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1109             if '-1' not in apk['icons_src']:
1110                 match = re.match(APK_ICON_PAT_NODPI, line)
1111                 if match:
1112                     apk['icons_src']['-1'] = match.group(1)
1113         elif line.startswith("application-icon-"):
1114             match = re.match(APK_ICON_PAT, line)
1115             if match:
1116                 density = match.group(1)
1117                 path = match.group(2)
1118                 apk['icons_src'][density] = path
1119         elif line.startswith("sdkVersion:"):
1120             m = re.match(APK_SDK_VERSION_PAT, line)
1121             if m is None:
1122                 logging.error(line.replace('sdkVersion:', '')
1123                               + ' is not a valid minSdkVersion!')
1124             else:
1125                 apk['minSdkVersion'] = m.group(1)
1126                 # if target not set, default to min
1127                 if 'targetSdkVersion' not in apk:
1128                     apk['targetSdkVersion'] = m.group(1)
1129         elif line.startswith("targetSdkVersion:"):
1130             m = re.match(APK_SDK_VERSION_PAT, line)
1131             if m is None:
1132                 logging.error(line.replace('targetSdkVersion:', '')
1133                               + ' is not a valid targetSdkVersion!')
1134             else:
1135                 apk['targetSdkVersion'] = m.group(1)
1136         elif line.startswith("maxSdkVersion:"):
1137             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1138         elif line.startswith("native-code:"):
1139             apk['nativecode'] = []
1140             for arch in line[13:].split(' '):
1141                 apk['nativecode'].append(arch[1:-1])
1142         elif line.startswith('uses-permission:'):
1143             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1144             if perm_match['maxSdkVersion']:
1145                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1146             permission = UsesPermission(
1147                 perm_match['name'],
1148                 perm_match['maxSdkVersion']
1149             )
1150
1151             apk['uses-permission'].append(permission)
1152         elif line.startswith('uses-permission-sdk-23:'):
1153             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1154             if perm_match['maxSdkVersion']:
1155                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1156             permission_sdk_23 = UsesPermissionSdk23(
1157                 perm_match['name'],
1158                 perm_match['maxSdkVersion']
1159             )
1160
1161             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1162
1163         elif line.startswith('uses-feature:'):
1164             feature = re.match(APK_FEATURE_PAT, line).group(1)
1165             # Filter out this, it's only added with the latest SDK tools and
1166             # causes problems for lots of apps.
1167             if feature != "android.hardware.screen.portrait" \
1168                     and feature != "android.hardware.screen.landscape":
1169                 if feature.startswith("android.feature."):
1170                     feature = feature[16:]
1171                 apk['features'].add(feature)
1172
1173
1174 def scan_apk_androguard(apk, apkfile):
1175     try:
1176         from androguard.core.bytecodes.apk import APK
1177         apkobject = APK(apkfile)
1178         if apkobject.is_valid_APK():
1179             arsc = apkobject.get_android_resources()
1180         else:
1181             if options.delete_unknown:
1182                 if os.path.exists(apkfile):
1183                     logging.error(_("Failed to get apk information, deleting {path}")
1184                                   .format(path=apkfile))
1185                     os.remove(apkfile)
1186                 else:
1187                     logging.error(_("Could not find {path} to remove it")
1188                                   .format(path=apkfile))
1189             else:
1190                 logging.error(_("Failed to get apk information, skipping {path}")
1191                               .format(path=apkfile))
1192             raise BuildException(_("Invalid APK"))
1193     except ImportError:
1194         raise FDroidException("androguard library is not installed and aapt not present")
1195     except FileNotFoundError:
1196         logging.error(_("Could not open apk file for analysis"))
1197         raise BuildException(_("Invalid APK"))
1198
1199     apk['packageName'] = apkobject.get_package()
1200     apk['versionCode'] = int(apkobject.get_androidversion_code())
1201     apk['versionName'] = apkobject.get_androidversion_name()
1202     if apk['versionName'][0] == "@":
1203         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1204         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1205         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1206     apk['name'] = apkobject.get_app_name()
1207
1208     if apkobject.get_max_sdk_version() is not None:
1209         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1210     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1211     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1212
1213     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1214     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1215
1216     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1217
1218     for file in apkobject.get_files():
1219         d_re = density_re.match(file)
1220         if d_re:
1221             folder = d_re.group(1).split('-')
1222             if len(folder) > 1:
1223                 resolution = folder[1]
1224             else:
1225                 resolution = 'mdpi'
1226             density = screen_resolutions[resolution]
1227             apk['icons_src'][density] = d_re.group(0)
1228
1229     if apk['icons_src'].get('-1') is None:
1230         apk['icons_src']['-1'] = apk['icons_src']['160']
1231
1232     arch_re = re.compile("^lib/(.*)/.*$")
1233     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1234     if len(arch) >= 1:
1235         apk['nativecode'] = []
1236         apk['nativecode'].extend(sorted(list(arch)))
1237
1238     xml = apkobject.get_android_manifest_xml()
1239
1240     for item in xml.getElementsByTagName('uses-permission'):
1241         name = str(item.getAttribute("android:name"))
1242         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1243         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1244         permission = UsesPermission(
1245             name,
1246             maxSdkVersion
1247         )
1248         apk['uses-permission'].append(permission)
1249
1250     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1251         name = str(item.getAttribute("android:name"))
1252         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1253         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1254         permission_sdk_23 = UsesPermissionSdk23(
1255             name,
1256             maxSdkVersion
1257         )
1258         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1259
1260     for item in xml.getElementsByTagName('uses-feature'):
1261         feature = str(item.getAttribute("android:name"))
1262         if feature != "android.hardware.screen.portrait" \
1263                 and feature != "android.hardware.screen.landscape":
1264             if feature.startswith("android.feature."):
1265                 feature = feature[16:]
1266         apk['features'].append(feature)
1267
1268
1269 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1270                 allow_disabled_algorithms=False, archive_bad_sig=False):
1271     """Processes the apk with the given filename in the given repo directory.
1272
1273     This also extracts the icons.
1274
1275     :param apkcache: current apk cache information
1276     :param apkfilename: the filename of the apk to scan
1277     :param repodir: repo directory to scan
1278     :param knownapks: known apks info
1279     :param use_date_from_apk: use date from APK (instead of current date)
1280                               for newly added APKs
1281     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1282                                       disabled algorithms in the signature (e.g. MD5)
1283     :param archive_bad_sig: move APKs with a bad signature to the archive
1284     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1285      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1286     """
1287
1288     apk = {}
1289     apkfile = os.path.join(repodir, apkfilename)
1290
1291     cachechanged = False
1292     usecache = False
1293     if apkfilename in apkcache:
1294         apk = apkcache[apkfilename]
1295         if apk.get('hash') == sha256sum(apkfile):
1296             logging.debug(_("Reading {apkfilename} from cache")
1297                           .format(apkfilename=apkfilename))
1298             usecache = True
1299         else:
1300             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1301                           .format(apkfilename=apkfilename))
1302
1303     if not usecache:
1304         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1305
1306         try:
1307             apk = scan_apk(apkfile)
1308         except BuildException:
1309             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1310                             .format(apkfilename=apkfilename))
1311             return True, None, False
1312
1313         # Check for debuggable apks...
1314         if common.isApkAndDebuggable(apkfile):
1315             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1316
1317         if options.rename_apks:
1318             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1319             std_short_name = os.path.join(repodir, n)
1320             if apkfile != std_short_name:
1321                 if os.path.exists(std_short_name):
1322                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1323                     if apkfile != std_long_name:
1324                         if os.path.exists(std_long_name):
1325                             dupdir = os.path.join('duplicates', repodir)
1326                             if not os.path.isdir(dupdir):
1327                                 os.makedirs(dupdir, exist_ok=True)
1328                             dupfile = os.path.join('duplicates', std_long_name)
1329                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1330                             os.rename(apkfile, dupfile)
1331                             return True, None, False
1332                         else:
1333                             os.rename(apkfile, std_long_name)
1334                     apkfile = std_long_name
1335                 else:
1336                     os.rename(apkfile, std_short_name)
1337                     apkfile = std_short_name
1338                 apkfilename = apkfile[len(repodir) + 1:]
1339
1340         apk['apkName'] = apkfilename
1341         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1342         if os.path.exists(os.path.join(repodir, srcfilename)):
1343             apk['srcname'] = srcfilename
1344
1345         # verify the jar signature is correct, allow deprecated
1346         # algorithms only if the APK is in the archive.
1347         skipapk = False
1348         if not common.verify_apk_signature(apkfile):
1349             if repodir == 'archive' or allow_disabled_algorithms:
1350                 if common.verify_old_apk_signature(apkfile):
1351                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1352                 else:
1353                     skipapk = True
1354             else:
1355                 skipapk = True
1356
1357         if skipapk:
1358             if archive_bad_sig:
1359                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1360                                 .format(apkfilename=apkfilename))
1361                 move_apk_between_sections(repodir, 'archive', apk)
1362             else:
1363                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1364                                 .format(apkfilename=apkfilename))
1365             return True, None, False
1366
1367         apkzip = zipfile.ZipFile(apkfile, 'r')
1368
1369         manifest = apkzip.getinfo('AndroidManifest.xml')
1370         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1371         if (1980, 0, 0) != manifest.date_time[0:3]:
1372             try:
1373                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1374             except ValueError as e:
1375                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1376                                 .format(apkfilename=apkfile) + str(e))
1377
1378         # extract icons from APK zip file
1379         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1380         try:
1381             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1382         finally:
1383             apkzip.close()  # ensure that APK zip file gets closed
1384
1385         # resize existing icons for densities missing in the APK
1386         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1387
1388         if use_date_from_apk and manifest.date_time[1] != 0:
1389             default_date_param = datetime(*manifest.date_time)
1390         else:
1391             default_date_param = None
1392
1393         # Record in known apks, getting the added date at the same time..
1394         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1395                                     default_date=default_date_param)
1396         if added:
1397             apk['added'] = added
1398
1399         apkcache[apkfilename] = apk
1400         cachechanged = True
1401
1402     return False, apk, cachechanged
1403
1404
1405 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1406     """Processes the apks in the given repo directory.
1407
1408     This also extracts the icons.
1409
1410     :param apkcache: current apk cache information
1411     :param repodir: repo directory to scan
1412     :param knownapks: known apks info
1413     :param use_date_from_apk: use date from APK (instead of current date)
1414                               for newly added APKs
1415     :returns: (apks, cachechanged) where apks is a list of apk information,
1416               and cachechanged is True if the apkcache got changed.
1417     """
1418
1419     cachechanged = False
1420
1421     for icon_dir in get_all_icon_dirs(repodir):
1422         if os.path.exists(icon_dir):
1423             if options.clean:
1424                 shutil.rmtree(icon_dir)
1425                 os.makedirs(icon_dir)
1426         else:
1427             os.makedirs(icon_dir)
1428
1429     apks = []
1430     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1431         apkfilename = apkfile[len(repodir) + 1:]
1432         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1433         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1434                                              use_date_from_apk, ada, True)
1435         if skip:
1436             continue
1437         apks.append(apk)
1438         cachechanged = cachechanged or cachethis
1439
1440     return apks, cachechanged
1441
1442
1443 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1444     """
1445     Extracts icons from the given APK zip in various densities,
1446     saves them into given repo directory
1447     and stores their names in the APK metadata dictionary.
1448
1449     :param icon_filename: A string representing the icon's file name
1450     :param apk: A populated dictionary containing APK metadata.
1451                 Needs to have 'icons_src' key
1452     :param apkzip: An opened zipfile.ZipFile of the APK file
1453     :param repo_dir: The directory of the APK's repository
1454     :return: A list of icon densities that are missing
1455     """
1456     empty_densities = []
1457     for density in screen_densities:
1458         if density not in apk['icons_src']:
1459             empty_densities.append(density)
1460             continue
1461         icon_src = apk['icons_src'][density]
1462         icon_dir = get_icon_dir(repo_dir, density)
1463         icon_dest = os.path.join(icon_dir, icon_filename)
1464
1465         # Extract the icon files per density
1466         if icon_src.endswith('.xml'):
1467             png = os.path.basename(icon_src)[:-4] + '.png'
1468             for f in apkzip.namelist():
1469                 if f.endswith(png):
1470                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1471                     if m and screen_resolutions[m.group(2)] == density:
1472                         icon_src = f
1473             if icon_src.endswith('.xml'):
1474                 empty_densities.append(density)
1475                 continue
1476         try:
1477             with open(icon_dest, 'wb') as f:
1478                 f.write(get_icon_bytes(apkzip, icon_src))
1479             apk['icons'][density] = icon_filename
1480         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1481             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1482             del apk['icons_src'][density]
1483             empty_densities.append(density)
1484
1485     if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
1486         icon_src = apk['icons_src']['-1']
1487         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1488         with open(icon_path, 'wb') as f:
1489             f.write(get_icon_bytes(apkzip, icon_src))
1490         im = None
1491         try:
1492             im = Image.open(icon_path)
1493             dpi = px_to_dpi(im.size[0])
1494             for density in screen_densities:
1495                 if density in apk['icons']:
1496                     break
1497                 if density == screen_densities[-1] or dpi >= int(density):
1498                     apk['icons'][density] = icon_filename
1499                     shutil.move(icon_path,
1500                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1501                     empty_densities.remove(density)
1502                     break
1503         except Exception as e:
1504             logging.warning(_("Failed reading {path}: {error}")
1505                             .format(path=icon_path, error=e))
1506         finally:
1507             if im and hasattr(im, 'close'):
1508                 im.close()
1509
1510     if apk['icons']:
1511         apk['icon'] = icon_filename
1512
1513     return empty_densities
1514
1515
1516 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1517     """
1518     Resize existing icons for densities missing in the APK to ensure all densities are available
1519
1520     :param empty_densities: A list of icon densities that are missing
1521     :param icon_filename: A string representing the icon's file name
1522     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1523     :param repo_dir: The directory of the APK's repository
1524     """
1525     # First try resizing down to not lose quality
1526     last_density = None
1527     for density in screen_densities:
1528         if density not in empty_densities:
1529             last_density = density
1530             continue
1531         if last_density is None:
1532             continue
1533         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1534
1535         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1536         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1537         fp = None
1538         try:
1539             fp = open(last_icon_path, 'rb')
1540             im = Image.open(fp)
1541
1542             size = dpi_to_px(density)
1543
1544             im.thumbnail((size, size), Image.ANTIALIAS)
1545             im.save(icon_path, "PNG", optimize=True,
1546                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1547             empty_densities.remove(density)
1548         except Exception as e:
1549             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1550         finally:
1551             if fp:
1552                 fp.close()
1553
1554     # Then just copy from the highest resolution available
1555     last_density = None
1556     for density in reversed(screen_densities):
1557         if density not in empty_densities:
1558             last_density = density
1559             continue
1560
1561         if last_density is None:
1562             continue
1563
1564         shutil.copyfile(
1565             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1566             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1567         )
1568         empty_densities.remove(density)
1569
1570     for density in screen_densities:
1571         icon_dir = get_icon_dir(repo_dir, density)
1572         icon_dest = os.path.join(icon_dir, icon_filename)
1573         resize_icon(icon_dest, density)
1574
1575     # Copy from icons-mdpi to icons since mdpi is the baseline density
1576     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1577     if os.path.isfile(baseline):
1578         apk['icons']['0'] = icon_filename
1579         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1580
1581
1582 def apply_info_from_latest_apk(apps, apks):
1583     """
1584     Some information from the apks needs to be applied up to the application level.
1585     When doing this, we use the info from the most recent version's apk.
1586     We deal with figuring out when the app was added and last updated at the same time.
1587     """
1588     for appid, app in apps.items():
1589         bestver = UNSET_VERSION_CODE
1590         for apk in apks:
1591             if apk['packageName'] == appid:
1592                 if apk['versionCode'] > bestver:
1593                     bestver = apk['versionCode']
1594                     bestapk = apk
1595
1596                 if 'added' in apk:
1597                     if not app.added or apk['added'] < app.added:
1598                         app.added = apk['added']
1599                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1600                         app.lastUpdated = apk['added']
1601
1602         if not app.added:
1603             logging.debug("Don't know when " + appid + " was added")
1604         if not app.lastUpdated:
1605             logging.debug("Don't know when " + appid + " was last updated")
1606
1607         if bestver == UNSET_VERSION_CODE:
1608
1609             if app.Name is None:
1610                 app.Name = app.AutoName or appid
1611             app.icon = None
1612             logging.debug("Application " + appid + " has no packages")
1613         else:
1614             if app.Name is None:
1615                 app.Name = bestapk['name']
1616             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1617             if app.CurrentVersionCode is None:
1618                 app.CurrentVersionCode = str(bestver)
1619
1620
1621 def make_categories_txt(repodir, categories):
1622     '''Write a category list in the repo to allow quick access'''
1623     catdata = ''
1624     for cat in sorted(categories):
1625         catdata += cat + '\n'
1626     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1627         f.write(catdata)
1628
1629
1630 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1631
1632     def filter_apk_list_sorted(apk_list):
1633         res = []
1634         for apk in apk_list:
1635             if apk['packageName'] == appid:
1636                 res.append(apk)
1637
1638         # Sort the apk list by version code. First is highest/newest.
1639         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1640
1641     for appid, app in apps.items():
1642
1643         if app.ArchivePolicy:
1644             keepversions = int(app.ArchivePolicy[:-9])
1645         else:
1646             keepversions = defaultkeepversions
1647
1648         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1649                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1650
1651         current_app_apks = filter_apk_list_sorted(apks)
1652         if len(current_app_apks) > keepversions:
1653             # Move back the ones we don't want.
1654             for apk in current_app_apks[keepversions:]:
1655                 move_apk_between_sections(repodir, archivedir, apk)
1656                 archapks.append(apk)
1657                 apks.remove(apk)
1658
1659         current_app_archapks = filter_apk_list_sorted(archapks)
1660         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1661             kept = 0
1662             # Move forward the ones we want again, except DisableAlgorithm
1663             for apk in current_app_archapks:
1664                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1665                     move_apk_between_sections(archivedir, repodir, apk)
1666                     archapks.remove(apk)
1667                     apks.append(apk)
1668                     kept += 1
1669                 if kept == keepversions:
1670                     break
1671
1672
1673 def move_apk_between_sections(from_dir, to_dir, apk):
1674     """move an APK from repo to archive or vice versa"""
1675
1676     def _move_file(from_dir, to_dir, filename, ignore_missing):
1677         from_path = os.path.join(from_dir, filename)
1678         if ignore_missing and not os.path.exists(from_path):
1679             return
1680         to_path = os.path.join(to_dir, filename)
1681         if not os.path.exists(to_dir):
1682             os.mkdir(to_dir)
1683         shutil.move(from_path, to_path)
1684
1685     if from_dir == to_dir:
1686         return
1687
1688     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1689     _move_file(from_dir, to_dir, apk['apkName'], False)
1690     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1691     for density in all_screen_densities:
1692         from_icon_dir = get_icon_dir(from_dir, density)
1693         to_icon_dir = get_icon_dir(to_dir, density)
1694         if density not in apk.get('icons', []):
1695             continue
1696         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1697     if 'srcname' in apk:
1698         _move_file(from_dir, to_dir, apk['srcname'], False)
1699
1700
1701 def add_apks_to_per_app_repos(repodir, apks):
1702     apks_per_app = dict()
1703     for apk in apks:
1704         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1705         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1706         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1707         apks_per_app[apk['packageName']] = apk
1708
1709         if not os.path.exists(apk['per_app_icons']):
1710             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1711             os.makedirs(apk['per_app_icons'])
1712
1713         apkpath = os.path.join(repodir, apk['apkName'])
1714         shutil.copy(apkpath, apk['per_app_repo'])
1715         apksigpath = apkpath + '.sig'
1716         if os.path.exists(apksigpath):
1717             shutil.copy(apksigpath, apk['per_app_repo'])
1718         apkascpath = apkpath + '.asc'
1719         if os.path.exists(apkascpath):
1720             shutil.copy(apkascpath, apk['per_app_repo'])
1721
1722
1723 def create_metadata_from_template(apk):
1724     '''create a new metadata file using internal or external template
1725
1726     Generate warnings for apk's with no metadata (or create skeleton
1727     metadata files, if requested on the command line).  Though the
1728     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1729     since those impose things on the metadata file made from the
1730     template: field sort order, empty field value, formatting, etc.
1731     '''
1732
1733     import yaml
1734     if os.path.exists('template.yml'):
1735         with open('template.yml') as f:
1736             metatxt = f.read()
1737         if 'name' in apk and apk['name'] != '':
1738             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1739                              r'\1 ' + apk['name'],
1740                              metatxt,
1741                              flags=re.IGNORECASE | re.MULTILINE)
1742         else:
1743             logging.warning(_('{appid} does not have a name! Using package name instead.')
1744                             .format(appid=apk['packageName']))
1745             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1746                              r'\1 ' + apk['packageName'],
1747                              metatxt,
1748                              flags=re.IGNORECASE | re.MULTILINE)
1749         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1750             f.write(metatxt)
1751     else:
1752         app = dict()
1753         app['Categories'] = [os.path.basename(os.getcwd())]
1754         # include some blanks as part of the template
1755         app['AuthorName'] = ''
1756         app['Summary'] = ''
1757         app['WebSite'] = ''
1758         app['IssueTracker'] = ''
1759         app['SourceCode'] = ''
1760         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1761         if 'name' in apk and apk['name'] != '':
1762             app['Name'] = apk['name']
1763         else:
1764             logging.warning(_('{appid} does not have a name! Using package name instead.')
1765                             .format(appid=apk['packageName']))
1766             app['Name'] = apk['packageName']
1767         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1768             yaml.dump(app, f, default_flow_style=False)
1769     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1770
1771
1772 config = None
1773 options = None
1774 start_timestamp = time.gmtime()
1775
1776
1777 def main():
1778
1779     global config, options
1780
1781     # Parse command line...
1782     parser = ArgumentParser()
1783     common.setup_global_opts(parser)
1784     parser.add_argument("--create-key", action="store_true", default=False,
1785                         help=_("Add a repo signing key to an unsigned repo"))
1786     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1787                         help=_("Add skeleton metadata files for APKs that are missing them"))
1788     parser.add_argument("--delete-unknown", action="store_true", default=False,
1789                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1790     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1791                         help=_("Report on build data status"))
1792     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1793                         help=_("Interactively ask about things that need updating."))
1794     parser.add_argument("-I", "--icons", action="store_true", default=False,
1795                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1796     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1797                         help=_("Specify editor to use in interactive mode. Default " +
1798                                "is {path}").format(path='/etc/alternatives/editor'))
1799     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1800                         help=_("Update the wiki"))
1801     parser.add_argument("--pretty", action="store_true", default=False,
1802                         help=_("Produce human-readable XML/JSON for index files"))
1803     parser.add_argument("--clean", action="store_true", default=False,
1804                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1805     parser.add_argument("--nosign", action="store_true", default=False,
1806                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1807     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1808                         help=_("Use date from APK instead of current time for newly added APKs"))
1809     parser.add_argument("--rename-apks", action="store_true", default=False,
1810                         help=_("Rename APK files that do not match package.name_123.apk"))
1811     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1812                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1813     metadata.add_metadata_arguments(parser)
1814     options = parser.parse_args()
1815     metadata.warnings_action = options.W
1816
1817     config = common.read_config(options)
1818
1819     if not ('jarsigner' in config and 'keytool' in config):
1820         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1821
1822     repodirs = ['repo']
1823     if config['archive_older'] != 0:
1824         repodirs.append('archive')
1825         if not os.path.exists('archive'):
1826             os.mkdir('archive')
1827
1828     if options.icons:
1829         resize_all_icons(repodirs)
1830         sys.exit(0)
1831
1832     if options.rename_apks:
1833         options.clean = True
1834
1835     # check that icons exist now, rather than fail at the end of `fdroid update`
1836     for k in ['repo_icon', 'archive_icon']:
1837         if k in config:
1838             if not os.path.exists(config[k]):
1839                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1840                                  .format(name=k, path=config[k]))
1841                 sys.exit(1)
1842
1843     # if the user asks to create a keystore, do it now, reusing whatever it can
1844     if options.create_key:
1845         if os.path.exists(config['keystore']):
1846             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1847             logging.critical("\t'" + config['keystore'] + "'")
1848             sys.exit(1)
1849
1850         if 'repo_keyalias' not in config:
1851             config['repo_keyalias'] = socket.getfqdn()
1852             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1853         if 'keydname' not in config:
1854             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1855             common.write_to_config(config, 'keydname', config['keydname'])
1856         if 'keystore' not in config:
1857             config['keystore'] = common.default_config['keystore']
1858             common.write_to_config(config, 'keystore', config['keystore'])
1859
1860         password = common.genpassword()
1861         if 'keystorepass' not in config:
1862             config['keystorepass'] = password
1863             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1864         if 'keypass' not in config:
1865             config['keypass'] = password
1866             common.write_to_config(config, 'keypass', config['keypass'])
1867         common.genkeystore(config)
1868
1869     # Get all apps...
1870     apps = metadata.read_metadata()
1871
1872     # Generate a list of categories...
1873     categories = set()
1874     for app in apps.values():
1875         categories.update(app.Categories)
1876
1877     # Read known apks data (will be updated and written back when we've finished)
1878     knownapks = common.KnownApks()
1879
1880     # Get APK cache
1881     apkcache = get_cache()
1882
1883     # Delete builds for disabled apps
1884     delete_disabled_builds(apps, apkcache, repodirs)
1885
1886     # Scan all apks in the main repo
1887     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1888
1889     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1890                                            options.use_date_from_apk)
1891     cachechanged = cachechanged or fcachechanged
1892     apks += files
1893     for apk in apks:
1894         if apk['packageName'] not in apps:
1895             if options.create_metadata:
1896                 create_metadata_from_template(apk)
1897                 apps = metadata.read_metadata()
1898             else:
1899                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1900                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1901                 if options.delete_unknown:
1902                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1903                                  .format(apkfilename=apk['apkName']))
1904                     rmf = os.path.join(repodirs[0], apk['apkName'])
1905                     if not os.path.exists(rmf):
1906                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1907                     else:
1908                         os.remove(rmf)
1909                 else:
1910                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1911
1912     copy_triple_t_store_metadata(apps)
1913     insert_obbs(repodirs[0], apps, apks)
1914     insert_localized_app_metadata(apps)
1915     translate_per_build_anti_features(apps, apks)
1916
1917     # Scan the archive repo for apks as well
1918     if len(repodirs) > 1:
1919         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1920         if cc:
1921             cachechanged = True
1922     else:
1923         archapks = []
1924
1925     # Apply information from latest apks to the application and update dates
1926     apply_info_from_latest_apk(apps, apks + archapks)
1927
1928     # Sort the app list by name, then the web site doesn't have to by default.
1929     # (we had to wait until we'd scanned the apks to do this, because mostly the
1930     # name comes from there!)
1931     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1932
1933     # APKs are placed into multiple repos based on the app package, providing
1934     # per-app subscription feeds for nightly builds and things like it
1935     if config['per_app_repos']:
1936         add_apks_to_per_app_repos(repodirs[0], apks)
1937         for appid, app in apps.items():
1938             repodir = os.path.join(appid, 'fdroid', 'repo')
1939             appdict = dict()
1940             appdict[appid] = app
1941             if os.path.isdir(repodir):
1942                 index.make(appdict, [appid], apks, repodir, False)
1943             else:
1944                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1945         return
1946
1947     if len(repodirs) > 1:
1948         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1949
1950     # Make the index for the main repo...
1951     index.make(apps, sortedids, apks, repodirs[0], False)
1952     make_categories_txt(repodirs[0], categories)
1953
1954     # If there's an archive repo,  make the index for it. We already scanned it
1955     # earlier on.
1956     if len(repodirs) > 1:
1957         index.make(apps, sortedids, archapks, repodirs[1], True)
1958
1959     git_remote = config.get('binary_transparency_remote')
1960     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1961         from . import btlog
1962         btlog.make_binary_transparency_log(repodirs)
1963
1964     if config['update_stats']:
1965         # Update known apks info...
1966         knownapks.writeifchanged()
1967
1968         # Generate latest apps data for widget
1969         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1970             data = ''
1971             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1972                 for line in f:
1973                     appid = line.rstrip()
1974                     data += appid + "\t"
1975                     app = apps[appid]
1976                     data += app.Name + "\t"
1977                     if app.icon is not None:
1978                         data += app.icon + "\t"
1979                     data += app.License + "\n"
1980             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1981                 f.write(data)
1982
1983     if cachechanged:
1984         write_cache(apkcache)
1985
1986     # Update the wiki...
1987     if options.wiki:
1988         update_wiki(apps, sortedids, apks + archapks)
1989
1990     logging.info(_("Finished"))
1991
1992
1993 if __name__ == "__main__":
1994     main()