fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import _
  41 from . import common
  42 from . import index
  43 from . import metadata
  44 from .common import SdkToolsPopen
  45 from .exception import BuildException, FDroidException
  46
  47 METADATA_VERSION = 19
  48
  49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  50 UNSET_VERSION_CODE = -0x100000000
  51
  52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  59 APK_PERMISSION_PAT = \
  60     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  62
  63 screen_densities = ['640', '480', '320', '240', '160', '120']
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "undefined": '-1',
  72     "anydpi": '65534',
  73     "nodpi": '65535'
  74 }
  75
  76 all_screen_densities = ['0'] + screen_densities
  77
  78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  80
  81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  84                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  85
  86
  87 def dpi_to_px(density):
  88     return (int(density) * 48) / 160
  89
  90
  91 def px_to_dpi(px):
  92     return (int(px) * 160) / 48
  93
  94
  95 def get_icon_dir(repodir, density):
  96     if density == '0':
  97         return os.path.join(repodir, "icons")
  98     return os.path.join(repodir, "icons-%s" % density)
  99
 100
 101 def get_icon_dirs(repodir):
 102     for density in screen_densities:
 103         yield get_icon_dir(repodir, density)
 104
 105
 106 def get_all_icon_dirs(repodir):
 107     for density in all_screen_densities:
 108         yield get_icon_dir(repodir, density)
 109
 110
 111 def update_wiki(apps, sortedids, apks):
 112     """Update the wiki
 113
 114     :param apps: fully populated list of all applications
 115     :param apks: all apks, except...
 116     """
 117     logging.info("Updating wiki")
 118     wikicat = 'Apps'
 119     wikiredircat = 'App Redirects'
 120     import mwclient
 121     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 122                          path=config['wiki_path'])
 123     site.login(config['wiki_user'], config['wiki_password'])
 124     generated_pages = {}
 125     generated_redirects = {}
 126
 127     for appid in sortedids:
 128         app = metadata.App(apps[appid])
 129
 130         wikidata = ''
 131         if app.Disabled:
 132             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 133         if app.AntiFeatures:
 134             for af in app.AntiFeatures:
 135                 wikidata += '{{AntiFeature|' + af + '}}\n'
 136         if app.RequiresRoot:
 137             requiresroot = 'Yes'
 138         else:
 139             requiresroot = 'No'
 140         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 141             appid,
 142             app.Name,
 143             app.added.strftime('%Y-%m-%d') if app.added else '',
 144             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 145             app.SourceCode,
 146             app.IssueTracker,
 147             app.WebSite,
 148             app.Changelog,
 149             app.Donate,
 150             app.FlattrID,
 151             app.Bitcoin,
 152             app.Litecoin,
 153             app.License,
 154             requiresroot,
 155             app.AuthorName,
 156             app.AuthorEmail)
 157
 158         if app.Provides:
 159             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 160
 161         wikidata += app.Summary
 162         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 163
 164         wikidata += "=Description=\n"
 165         wikidata += metadata.description_wiki(app.Description) + "\n"
 166
 167         wikidata += "=Maintainer Notes=\n"
 168         if app.MaintainerNotes:
 169             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 170         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 171
 172         # Get a list of all packages for this application...
 173         apklist = []
 174         gotcurrentver = False
 175         cantupdate = False
 176         buildfails = False
 177         for apk in apks:
 178             if apk['packageName'] == appid:
 179                 if str(apk['versionCode']) == app.CurrentVersionCode:
 180                     gotcurrentver = True
 181                 apklist.append(apk)
 182         # Include ones we can't build, as a special case...
 183         for build in app.builds:
 184             if build.disable:
 185                 if build.versionCode == app.CurrentVersionCode:
 186                     cantupdate = True
 187                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 188                 apklist.append({'versionCode': int(build.versionCode),
 189                                 'versionName': build.versionName,
 190                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 191                                 })
 192             else:
 193                 builtit = False
 194                 for apk in apklist:
 195                     if apk['versionCode'] == int(build.versionCode):
 196                         builtit = True
 197                         break
 198                 if not builtit:
 199                     buildfails = True
 200                     apklist.append({'versionCode': int(build.versionCode),
 201                                     'versionName': build.versionName,
 202                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 203                                     })
 204         if app.CurrentVersionCode == '0':
 205             cantupdate = True
 206         # Sort with most recent first...
 207         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 208
 209         wikidata += "=Versions=\n"
 210         if len(apklist) == 0:
 211             wikidata += "We currently have no versions of this app available."
 212         elif not gotcurrentver:
 213             wikidata += "We don't have the current version of this app."
 214         else:
 215             wikidata += "We have the current version of this app."
 216         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 217         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 218         if len(app.NoSourceSince) > 0:
 219             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 220         if len(app.CurrentVersion) > 0:
 221             wikidata += "The current (recommended) version is " + app.CurrentVersion
 222             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 223         validapks = 0
 224         for apk in apklist:
 225             wikidata += "==" + apk['versionName'] + "==\n"
 226
 227             if 'buildproblem' in apk:
 228                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 229             else:
 230                 validapks += 1
 231                 wikidata += "This version is built and signed by "
 232                 if 'srcname' in apk:
 233                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 234                 else:
 235                     wikidata += "the original developer.\n\n"
 236             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 237
 238         wikidata += '\n[[Category:' + wikicat + ']]\n'
 239         if len(app.NoSourceSince) > 0:
 240             wikidata += '\n[[Category:Apps missing source code]]\n'
 241         if validapks == 0 and not app.Disabled:
 242             wikidata += '\n[[Category:Apps with no packages]]\n'
 243         if cantupdate and not app.Disabled:
 244             wikidata += "\n[[Category:Apps we cannot update]]\n"
 245         if buildfails and not app.Disabled:
 246             wikidata += "\n[[Category:Apps with failing builds]]\n"
 247         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 248             wikidata += '\n[[Category:Apps to Update]]\n'
 249         if app.Disabled:
 250             wikidata += '\n[[Category:Apps that are disabled]]\n'
 251         if app.UpdateCheckMode == 'None' and not app.Disabled:
 252             wikidata += '\n[[Category:Apps with no update check]]\n'
 253         for appcat in app.Categories:
 254             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 255
 256         # We can't have underscores in the page name, even if they're in
 257         # the package ID, because MediaWiki messes with them...
 258         pagename = appid.replace('_', ' ')
 259
 260         # Drop a trailing newline, because mediawiki is going to drop it anyway
 261         # and it we don't we'll think the page has changed when it hasn't...
 262         if wikidata.endswith('\n'):
 263             wikidata = wikidata[:-1]
 264
 265         generated_pages[pagename] = wikidata
 266
 267         # Make a redirect from the name to the ID too, unless there's
 268         # already an existing page with the name and it isn't a redirect.
 269         noclobber = False
 270         apppagename = app.Name.replace('_', ' ')
 271         apppagename = apppagename.replace('{', '')
 272         apppagename = apppagename.replace('}', ' ')
 273         apppagename = apppagename.replace(':', ' ')
 274         apppagename = apppagename.replace('[', ' ')
 275         apppagename = apppagename.replace(']', ' ')
 276         # Drop double spaces caused mostly by replacing ':' above
 277         apppagename = apppagename.replace('  ', ' ')
 278         for expagename in site.allpages(prefix=apppagename,
 279                                         filterredir='nonredirects',
 280                                         generator=False):
 281             if expagename == apppagename:
 282                 noclobber = True
 283         # Another reason not to make the redirect page is if the app name
 284         # is the same as it's ID, because that will overwrite the real page
 285         # with an redirect to itself! (Although it seems like an odd
 286         # scenario this happens a lot, e.g. where there is metadata but no
 287         # builds or binaries to extract a name from.
 288         if apppagename == pagename:
 289             noclobber = True
 290         if not noclobber:
 291             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 292
 293     for tcat, genp in [(wikicat, generated_pages),
 294                        (wikiredircat, generated_redirects)]:
 295         catpages = site.Pages['Category:' + tcat]
 296         existingpages = []
 297         for page in catpages:
 298             existingpages.append(page.name)
 299             if page.name in genp:
 300                 pagetxt = page.edit()
 301                 if pagetxt != genp[page.name]:
 302                     logging.debug("Updating modified page " + page.name)
 303                     page.save(genp[page.name], summary='Auto-updated')
 304                 else:
 305                     logging.debug("Page " + page.name + " is unchanged")
 306             else:
 307                 logging.warn("Deleting page " + page.name)
 308                 page.delete('No longer published')
 309         for pagename, text in genp.items():
 310             logging.debug("Checking " + pagename)
 311             if pagename not in existingpages:
 312                 logging.debug("Creating page " + pagename)
 313                 try:
 314                     newpage = site.Pages[pagename]
 315                     newpage.save(text, summary='Auto-created')
 316                 except Exception as e:
 317                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 318
 319     # Purge server cache to ensure counts are up to date
 320     site.pages['Repository Maintenance'].purge()
 321
 322
 323 def delete_disabled_builds(apps, apkcache, repodirs):
 324     """Delete disabled build outputs.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodirs: the repo directories to process
 329     """
 330     for appid, app in apps.items():
 331         for build in app['builds']:
 332             if not build.disable:
 333                 continue
 334             apkfilename = common.get_release_filename(app, build)
 335             iconfilename = "%s.%s.png" % (
 336                 appid,
 337                 build.versionCode)
 338             for repodir in repodirs:
 339                 files = [
 340                     os.path.join(repodir, apkfilename),
 341                     os.path.join(repodir, apkfilename + '.asc'),
 342                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 343                 ]
 344                 for density in all_screen_densities:
 345                     repo_dir = get_icon_dir(repodir, density)
 346                     files.append(os.path.join(repo_dir, iconfilename))
 347
 348                 for f in files:
 349                     if os.path.exists(f):
 350                         logging.info("Deleting disabled build output " + f)
 351                         os.remove(f)
 352             if apkfilename in apkcache:
 353                 del apkcache[apkfilename]
 354
 355
 356 def resize_icon(iconpath, density):
 357
 358     if not os.path.isfile(iconpath):
 359         return
 360
 361     fp = None
 362     try:
 363         fp = open(iconpath, 'rb')
 364         im = Image.open(fp)
 365         size = dpi_to_px(density)
 366
 367         if any(length > size for length in im.size):
 368             oldsize = im.size
 369             im.thumbnail((size, size), Image.ANTIALIAS)
 370             logging.debug("%s was too large at %s - new size is %s" % (
 371                 iconpath, oldsize, im.size))
 372             im.save(iconpath, "PNG")
 373
 374     except Exception as e:
 375         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 376
 377     finally:
 378         if fp:
 379             fp.close()
 380
 381
 382 def resize_all_icons(repodirs):
 383     """Resize all icons that exceed the max size
 384
 385     :param repodirs: the repo directories to process
 386     """
 387     for repodir in repodirs:
 388         for density in screen_densities:
 389             icon_dir = get_icon_dir(repodir, density)
 390             icon_glob = os.path.join(icon_dir, '*.png')
 391             for iconpath in glob.glob(icon_glob):
 392                 resize_icon(iconpath, density)
 393
 394
 395 def getsig(apkpath):
 396     """ Get the signing certificate of an apk. To get the same md5 has that
 397     Android gets, we encode the .RSA certificate in a specific format and pass
 398     it hex-encoded to the md5 digest algorithm.
 399
 400     :param apkpath: path to the apk
 401     :returns: A string containing the md5 of the signature of the apk or None
 402               if an error occurred.
 403     """
 404
 405     with zipfile.ZipFile(apkpath, 'r') as apk:
 406         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 407
 408         if len(certs) < 1:
 409             logging.error("Found no signing certificates on %s" % apkpath)
 410             return None
 411         if len(certs) > 1:
 412             logging.error("Found multiple signing certificates on %s" % apkpath)
 413             return None
 414
 415         cert = apk.read(certs[0])
 416
 417     cert_encoded = common.get_certificate(cert)
 418
 419     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 420
 421
 422 def get_cache_file():
 423     return os.path.join('tmp', 'apkcache')
 424
 425
 426 def get_cache():
 427     """Get the cached dict of the APK index
 428
 429     Gather information about all the apk files in the repo directory,
 430     using cached data if possible. Some of the index operations take a
 431     long time, like calculating the SHA-256 and verifying the APK
 432     signature.
 433
 434     The cache is invalidated if the metadata version is different, or
 435     the 'allow_disabled_algorithms' config/option is different.  In
 436     those cases, there is no easy way to know what has changed from
 437     the cache, so just rerun the whole thing.
 438
 439     :return: apkcache
 440
 441     """
 442     apkcachefile = get_cache_file()
 443     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 444     if not options.clean and os.path.exists(apkcachefile):
 445         with open(apkcachefile, 'rb') as cf:
 446             apkcache = pickle.load(cf, encoding='utf-8')
 447         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 448            or apkcache.get('allow_disabled_algorithms') != ada:
 449             apkcache = {}
 450     else:
 451         apkcache = {}
 452
 453     apkcache["METADATA_VERSION"] = METADATA_VERSION
 454     apkcache['allow_disabled_algorithms'] = ada
 455
 456     return apkcache
 457
 458
 459 def write_cache(apkcache):
 460     apkcachefile = get_cache_file()
 461     cache_path = os.path.dirname(apkcachefile)
 462     if not os.path.exists(cache_path):
 463         os.makedirs(cache_path)
 464     with open(apkcachefile, 'wb') as cf:
 465         pickle.dump(apkcache, cf)
 466
 467
 468 def get_icon_bytes(apkzip, iconsrc):
 469     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 470     try:
 471         return apkzip.read(iconsrc)
 472     except KeyError:
 473         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 474
 475
 476 def sha256sum(filename):
 477     '''Calculate the sha256 of the given file'''
 478     sha = hashlib.sha256()
 479     with open(filename, 'rb') as f:
 480         while True:
 481             t = f.read(16384)
 482             if len(t) == 0:
 483                 break
 484             sha.update(t)
 485     return sha.hexdigest()
 486
 487
 488 def has_known_vulnerability(filename):
 489     """checks for known vulnerabilities in the APK
 490
 491     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 492     version.  Google also enforces this:
 493     https://support.google.com/faqs/answer/6376725?hl=en
 494
 495     Checks whether there are more than one classes.dex or AndroidManifest.xml
 496     files, which is invalid and an essential part of the "Master Key" attack.
 497
 498     http://www.saurik.com/id/17
 499     """
 500
 501     # statically load this pattern
 502     if not hasattr(has_known_vulnerability, "pattern"):
 503         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 504
 505     files_in_apk = set()
 506     with zipfile.ZipFile(filename) as zf:
 507         for name in zf.namelist():
 508             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 509                 lib = zf.open(name)
 510                 while True:
 511                     chunk = lib.read(4096)
 512                     if chunk == b'':
 513                         break
 514                     m = has_known_vulnerability.pattern.search(chunk)
 515                     if m:
 516                         version = m.group(1).decode('ascii')
 517                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 518                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 519                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 520                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 521                         else:
 522                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 523                             return True
 524                         break
 525             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 526                 if name in files_in_apk:
 527                     return True
 528                 files_in_apk.add(name)
 529
 530     return False
 531
 532
 533 def insert_obbs(repodir, apps, apks):
 534     """Scans the .obb files in a given repo directory and adds them to the
 535     relevant APK instances.  OBB files have versionCodes like APK
 536     files, and they are loosely associated.  If there is an OBB file
 537     present, then any APK with the same or higher versionCode will use
 538     that OBB file.  There are two OBB types: main and patch, each APK
 539     can only have only have one of each.
 540
 541     https://developer.android.com/google/play/expansion-files.html
 542
 543     :param repodir: repo directory to scan
 544     :param apps: list of current, valid apps
 545     :param apks: current information on all APKs
 546
 547     """
 548
 549     def obbWarnDelete(f, msg):
 550         logging.warning(msg + f)
 551         if options.delete_unknown:
 552             logging.error("Deleting unknown file: " + f)
 553             os.remove(f)
 554
 555     obbs = []
 556     java_Integer_MIN_VALUE = -pow(2, 31)
 557     currentPackageNames = apps.keys()
 558     for f in glob.glob(os.path.join(repodir, '*.obb')):
 559         obbfile = os.path.basename(f)
 560         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 561         chunks = obbfile.split('.')
 562         if chunks[0] != 'main' and chunks[0] != 'patch':
 563             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 564             continue
 565         if not re.match(r'^-?[0-9]+$', chunks[1]):
 566             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 567             continue
 568         versionCode = int(chunks[1])
 569         packagename = ".".join(chunks[2:-1])
 570
 571         highestVersionCode = java_Integer_MIN_VALUE
 572         if packagename not in currentPackageNames:
 573             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 574             continue
 575         for apk in apks:
 576             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 577                 highestVersionCode = apk['versionCode']
 578         if versionCode > highestVersionCode:
 579             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 580                           + ') than any APK: ')
 581             continue
 582         obbsha256 = sha256sum(f)
 583         obbs.append((packagename, versionCode, obbfile, obbsha256))
 584
 585     for apk in apks:
 586         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 587             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 588                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 589                     apk['obbMainFile'] = obbfile
 590                     apk['obbMainFileSha256'] = obbsha256
 591                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 592                     apk['obbPatchFile'] = obbfile
 593                     apk['obbPatchFileSha256'] = obbsha256
 594             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 595                 break
 596
 597
 598 def translate_per_build_anti_features(apps, apks):
 599     """Grab the anti-features list from the build metadata
 600
 601     For most Anti-Features, they are really most applicable per-APK,
 602     not for an app.  An app can fix a vulnerability, add/remove
 603     tracking, etc.  This reads the 'antifeatures' list from the Build
 604     entries in the fdroiddata metadata file, then transforms it into
 605     the 'antiFeatures' list of unique items for the index.
 606
 607     The field key is all lower case in the metadata file to match the
 608     rest of the Build fields.  It is 'antiFeatures' camel case in the
 609     implementation, index, and fdroidclient since it is translated
 610     from the build 'antifeatures' field, not directly included.
 611
 612     """
 613
 614     antiFeatures = dict()
 615     for packageName, app in apps.items():
 616         d = dict()
 617         for build in app['builds']:
 618             afl = build.get('antifeatures')
 619             if afl:
 620                 d[int(build.versionCode)] = afl
 621         if len(d) > 0:
 622             antiFeatures[packageName] = d
 623
 624     for apk in apks:
 625         d = antiFeatures.get(apk['packageName'])
 626         if d:
 627             afl = d.get(apk['versionCode'])
 628             if afl:
 629                 apk['antiFeatures'].update(afl)
 630
 631
 632 def _get_localized_dict(app, locale):
 633     '''get the dict to add localized store metadata to'''
 634     if 'localized' not in app:
 635         app['localized'] = collections.OrderedDict()
 636     if locale not in app['localized']:
 637         app['localized'][locale] = collections.OrderedDict()
 638     return app['localized'][locale]
 639
 640
 641 def _set_localized_text_entry(app, locale, key, f):
 642     limit = config['char_limits'][key]
 643     localized = _get_localized_dict(app, locale)
 644     with open(f) as fp:
 645         text = fp.read()[:limit]
 646         if len(text) > 0:
 647             localized[key] = text
 648
 649
 650 def _set_author_entry(app, key, f):
 651     limit = config['char_limits']['author']
 652     with open(f) as fp:
 653         text = fp.read()[:limit]
 654         if len(text) > 0:
 655             app[key] = text
 656
 657
 658 def copy_triple_t_store_metadata(apps):
 659     """Include store metadata from the app's source repo
 660
 661     The Triple-T Gradle Play Publisher is a plugin that has a standard
 662     file layout for all of the metadata and graphics that the Google
 663     Play Store accepts.  Since F-Droid has the git repo, it can just
 664     pluck those files directly.  This method reads any text files into
 665     the app dict, then copies any graphics into the fdroid repo
 666     directory structure.
 667
 668     This needs to be run before insert_localized_app_metadata() so that
 669     the graphics files that are copied into the fdroid repo get
 670     properly indexed.
 671
 672     https://github.com/Triple-T/gradle-play-publisher#upload-images
 673     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 674
 675     """
 676
 677     if not os.path.isdir('build'):
 678         return  # nothing to do
 679
 680     for packageName, app in apps.items():
 681         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 682             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 683             for root, dirs, files in os.walk(d):
 684                 segments = root.split('/')
 685                 locale = segments[-2]
 686                 for f in files:
 687                     if f == 'fulldescription':
 688                         _set_localized_text_entry(app, locale, 'description',
 689                                                   os.path.join(root, f))
 690                         continue
 691                     elif f == 'shortdescription':
 692                         _set_localized_text_entry(app, locale, 'summary',
 693                                                   os.path.join(root, f))
 694                         continue
 695                     elif f == 'title':
 696                         _set_localized_text_entry(app, locale, 'name',
 697                                                   os.path.join(root, f))
 698                         continue
 699                     elif f == 'video':
 700                         _set_localized_text_entry(app, locale, 'video',
 701                                                   os.path.join(root, f))
 702                         continue
 703                     elif f == 'whatsnew':
 704                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 705                                                   os.path.join(root, f))
 706                         continue
 707                     elif f == 'contactEmail':
 708                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 709                         continue
 710                     elif f == 'contactPhone':
 711                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 712                         continue
 713                     elif f == 'contactWebsite':
 714                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 715                         continue
 716
 717                     base, extension = common.get_extension(f)
 718                     dirname = os.path.basename(root)
 719                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 720                         if segments[-2] == 'listing':
 721                             locale = segments[-3]
 722                         else:
 723                             locale = segments[-2]
 724                         destdir = os.path.join('repo', packageName, locale)
 725                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 726                         sourcefile = os.path.join(root, f)
 727                         destfile = os.path.join(destdir, dirname + '.' + extension)
 728                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 729                         shutil.copy(sourcefile, destfile)
 730
 731
 732 def insert_localized_app_metadata(apps):
 733     """scans standard locations for graphics and localized text
 734
 735     Scans for localized description files, store graphics, and
 736     screenshot PNG files in statically defined screenshots directory
 737     and adds them to the app metadata.  The screenshots and graphic
 738     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 739     and must be in the following layout:
 740     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 741
 742     repo/packageName/locale/featureGraphic.png
 743     repo/packageName/locale/phoneScreenshots/1.png
 744     repo/packageName/locale/phoneScreenshots/2.png
 745
 746     The changelog files must be text files named with the versionCode
 747     ending with ".txt" and must be in the following layout:
 748     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 749
 750     repo/packageName/locale/changelogs/12345.txt
 751
 752     This will scan the each app's source repo then the metadata/ dir
 753     for these standard locations of changelog files.  If it finds
 754     them, they will be added to the dict of all packages, with the
 755     versions in the metadata/ folder taking precendence over the what
 756     is in the app's source repo.
 757
 758     Where "packageName" is the app's packageName and "locale" is the locale
 759     of the graphics, e.g. what language they are in, using the IETF RFC5646
 760     format (en-US, fr-CA, es-MX, etc).
 761
 762     This will also scan the app's git for a fastlane folder, and the
 763     metadata/ folder and the apps' source repos for standard locations
 764     of graphic and screenshot files.  If it finds them, it will copy
 765     them into the repo.  The fastlane files follow this pattern:
 766     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 767
 768     """
 769
 770     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 771     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 772     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 773
 774     for srcd in sorted(sourcedirs):
 775         if not os.path.isdir(srcd):
 776             continue
 777         for root, dirs, files in os.walk(srcd):
 778             segments = root.split('/')
 779             packageName = segments[1]
 780             if packageName not in apps:
 781                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 782                 continue
 783             locale = segments[-1]
 784             destdir = os.path.join('repo', packageName, locale)
 785             for f in files:
 786                 if f in ('description.txt', 'full_description.txt'):
 787                     _set_localized_text_entry(apps[packageName], locale, 'description',
 788                                               os.path.join(root, f))
 789                     continue
 790                 elif f in ('summary.txt', 'short_description.txt'):
 791                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 792                                               os.path.join(root, f))
 793                     continue
 794                 elif f in ('name.txt', 'title.txt'):
 795                     _set_localized_text_entry(apps[packageName], locale, 'name',
 796                                               os.path.join(root, f))
 797                     continue
 798                 elif f == 'video.txt':
 799                     _set_localized_text_entry(apps[packageName], locale, 'video',
 800                                               os.path.join(root, f))
 801                     continue
 802                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 803                     locale = segments[-2]
 804                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 805                                               os.path.join(root, f))
 806                     continue
 807
 808                 base, extension = common.get_extension(f)
 809                 if locale == 'images':
 810                     locale = segments[-2]
 811                     destdir = os.path.join('repo', packageName, locale)
 812                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 813                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 814                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 815                     shutil.copy(os.path.join(root, f), destdir)
 816             for d in dirs:
 817                 if d in SCREENSHOT_DIRS:
 818                     for f in glob.glob(os.path.join(root, d, '*.*')):
 819                         _, extension = common.get_extension(f)
 820                         if extension in ALLOWED_EXTENSIONS:
 821                             screenshotdestdir = os.path.join(destdir, d)
 822                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 823                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 824                             shutil.copy(f, screenshotdestdir)
 825
 826     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 827     for d in repofiles:
 828         if not os.path.isdir(d):
 829             continue
 830         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 831             if not os.path.isfile(f):
 832                 continue
 833             segments = f.split('/')
 834             packageName = segments[1]
 835             locale = segments[2]
 836             screenshotdir = segments[3]
 837             filename = os.path.basename(f)
 838             base, extension = common.get_extension(filename)
 839
 840             if packageName not in apps:
 841                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 842                                 % (filename, packageName))
 843                 continue
 844             graphics = _get_localized_dict(apps[packageName], locale)
 845
 846             if extension not in ALLOWED_EXTENSIONS:
 847                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 848             elif base in GRAPHIC_NAMES:
 849                 # there can only be zero or one of these per locale
 850                 graphics[base] = filename
 851             elif screenshotdir in SCREENSHOT_DIRS:
 852                 # there can any number of these per locale
 853                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 854                 if screenshotdir not in graphics:
 855                     graphics[screenshotdir] = []
 856                 graphics[screenshotdir].append(filename)
 857             else:
 858                 logging.warning('Unsupported graphics file found: ' + f)
 859
 860
 861 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 862     """Scan a repo for all files with an extension except APK/OBB
 863
 864     :param apkcache: current cached info about all repo files
 865     :param repodir: repo directory to scan
 866     :param knownapks: list of all known files, as per metadata.read_metadata
 867     :param use_date_from_file: use date from file (instead of current date)
 868                                for newly added files
 869     """
 870
 871     cachechanged = False
 872     repo_files = []
 873     repodir = repodir.encode('utf-8')
 874     for name in os.listdir(repodir):
 875         file_extension = common.get_file_extension(name)
 876         if file_extension == 'apk' or file_extension == 'obb':
 877             continue
 878         filename = os.path.join(repodir, name)
 879         name_utf8 = name.decode('utf-8')
 880         if filename.endswith(b'_src.tar.gz'):
 881             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 882             continue
 883         if not common.is_repo_file(filename):
 884             continue
 885         stat = os.stat(filename)
 886         if stat.st_size == 0:
 887             raise FDroidException(filename + ' is zero size!')
 888
 889         shasum = sha256sum(filename)
 890         usecache = False
 891         if name in apkcache:
 892             repo_file = apkcache[name]
 893             # added time is cached as tuple but used here as datetime instance
 894             if 'added' in repo_file:
 895                 a = repo_file['added']
 896                 if isinstance(a, datetime):
 897                     repo_file['added'] = a
 898                 else:
 899                     repo_file['added'] = datetime(*a[:6])
 900             if repo_file.get('hash') == shasum:
 901                 logging.debug("Reading " + name_utf8 + " from cache")
 902                 usecache = True
 903             else:
 904                 logging.debug("Ignoring stale cache data for " + name)
 905
 906         if not usecache:
 907             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 908             repo_file = collections.OrderedDict()
 909             repo_file['name'] = os.path.splitext(name_utf8)[0]
 910             # TODO rename apkname globally to something more generic
 911             repo_file['apkName'] = name_utf8
 912             repo_file['hash'] = shasum
 913             repo_file['hashType'] = 'sha256'
 914             repo_file['versionCode'] = 0
 915             repo_file['versionName'] = shasum
 916             # the static ID is the SHA256 unless it is set in the metadata
 917             repo_file['packageName'] = shasum
 918
 919             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 920             if m:
 921                 repo_file['packageName'] = m.group(1)
 922                 repo_file['versionCode'] = int(m.group(2))
 923             srcfilename = name + b'_src.tar.gz'
 924             if os.path.exists(os.path.join(repodir, srcfilename)):
 925                 repo_file['srcname'] = srcfilename.decode('utf-8')
 926             repo_file['size'] = stat.st_size
 927
 928             apkcache[name] = repo_file
 929             cachechanged = True
 930
 931         if use_date_from_file:
 932             timestamp = stat.st_ctime
 933             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 934         else:
 935             default_date_param = None
 936
 937         # Record in knownapks, getting the added date at the same time..
 938         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 939                                     default_date=default_date_param)
 940         if added:
 941             repo_file['added'] = added
 942
 943         repo_files.append(repo_file)
 944
 945     return repo_files, cachechanged
 946
 947
 948 def scan_apk(apk_file):
 949     """
 950     Scans an APK file and returns dictionary with metadata of the APK.
 951
 952     Attention: This does *not* verify that the APK signature is correct.
 953
 954     :param apk_file: The (ideally absolute) path to the APK file
 955     :raises BuildException
 956     :return A dict containing APK metadata
 957     """
 958     apk = {
 959         'hash': sha256sum(apk_file),
 960         'hashType': 'sha256',
 961         'uses-permission': [],
 962         'uses-permission-sdk-23': [],
 963         'features': [],
 964         'icons_src': {},
 965         'icons': {},
 966         'antiFeatures': set(),
 967     }
 968
 969     if SdkToolsPopen(['aapt', 'version'], output=False):
 970         scan_apk_aapt(apk, apk_file)
 971     else:
 972         scan_apk_androguard(apk, apk_file)
 973
 974     # Get the signature
 975     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 976     apk['sig'] = getsig(apk_file)
 977     if not apk['sig']:
 978         raise BuildException("Failed to get apk signature")
 979     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
 980                                                                apk_file))
 981
 982     # Get size of the APK
 983     apk['size'] = os.path.getsize(apk_file)
 984
 985     if 'minSdkVersion' not in apk:
 986         logging.warning("No SDK version information found in {0}".format(apk_file))
 987         apk['minSdkVersion'] = 1
 988
 989     # Check for known vulnerabilities
 990     if has_known_vulnerability(apk_file):
 991         apk['antiFeatures'].add('KnownVuln')
 992
 993     return apk
 994
 995
 996 def scan_apk_aapt(apk, apkfile):
 997     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 998     if p.returncode != 0:
 999         if options.delete_unknown:
1000             if os.path.exists(apkfile):
1001                 logging.error("Failed to get apk information, deleting " + apkfile)
1002                 os.remove(apkfile)
1003             else:
1004                 logging.error("Could not find {0} to remove it".format(apkfile))
1005         else:
1006             logging.error("Failed to get apk information, skipping " + apkfile)
1007         raise BuildException("Invalid APK")
1008     for line in p.output.splitlines():
1009         if line.startswith("package:"):
1010             try:
1011                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1012                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1013                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1014             except Exception as e:
1015                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1016         elif line.startswith("application:"):
1017             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1018             # Keep path to non-dpi icon in case we need it
1019             match = re.match(APK_ICON_PAT_NODPI, line)
1020             if match:
1021                 apk['icons_src']['-1'] = match.group(1)
1022         elif line.startswith("launchable-activity:"):
1023             # Only use launchable-activity as fallback to application
1024             if not apk['name']:
1025                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1026             if '-1' not in apk['icons_src']:
1027                 match = re.match(APK_ICON_PAT_NODPI, line)
1028                 if match:
1029                     apk['icons_src']['-1'] = match.group(1)
1030         elif line.startswith("application-icon-"):
1031             match = re.match(APK_ICON_PAT, line)
1032             if match:
1033                 density = match.group(1)
1034                 path = match.group(2)
1035                 apk['icons_src'][density] = path
1036         elif line.startswith("sdkVersion:"):
1037             m = re.match(APK_SDK_VERSION_PAT, line)
1038             if m is None:
1039                 logging.error(line.replace('sdkVersion:', '')
1040                               + ' is not a valid minSdkVersion!')
1041             else:
1042                 apk['minSdkVersion'] = m.group(1)
1043                 # if target not set, default to min
1044                 if 'targetSdkVersion' not in apk:
1045                     apk['targetSdkVersion'] = m.group(1)
1046         elif line.startswith("targetSdkVersion:"):
1047             m = re.match(APK_SDK_VERSION_PAT, line)
1048             if m is None:
1049                 logging.error(line.replace('targetSdkVersion:', '')
1050                               + ' is not a valid targetSdkVersion!')
1051             else:
1052                 apk['targetSdkVersion'] = m.group(1)
1053         elif line.startswith("maxSdkVersion:"):
1054             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1055         elif line.startswith("native-code:"):
1056             apk['nativecode'] = []
1057             for arch in line[13:].split(' '):
1058                 apk['nativecode'].append(arch[1:-1])
1059         elif line.startswith('uses-permission:'):
1060             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1061             if perm_match['maxSdkVersion']:
1062                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1063             permission = UsesPermission(
1064                 perm_match['name'],
1065                 perm_match['maxSdkVersion']
1066             )
1067
1068             apk['uses-permission'].append(permission)
1069         elif line.startswith('uses-permission-sdk-23:'):
1070             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1071             if perm_match['maxSdkVersion']:
1072                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1073             permission_sdk_23 = UsesPermissionSdk23(
1074                 perm_match['name'],
1075                 perm_match['maxSdkVersion']
1076             )
1077
1078             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1079
1080         elif line.startswith('uses-feature:'):
1081             feature = re.match(APK_FEATURE_PAT, line).group(1)
1082             # Filter out this, it's only added with the latest SDK tools and
1083             # causes problems for lots of apps.
1084             if feature != "android.hardware.screen.portrait" \
1085                     and feature != "android.hardware.screen.landscape":
1086                 if feature.startswith("android.feature."):
1087                     feature = feature[16:]
1088                 apk['features'].add(feature)
1089
1090
1091 def scan_apk_androguard(apk, apkfile):
1092     try:
1093         from androguard.core.bytecodes.apk import APK
1094         apkobject = APK(apkfile)
1095         if apkobject.is_valid_APK():
1096             arsc = apkobject.get_android_resources()
1097         else:
1098             if options.delete_unknown:
1099                 if os.path.exists(apkfile):
1100                     logging.error("Failed to get apk information, deleting " + apkfile)
1101                     os.remove(apkfile)
1102                 else:
1103                     logging.error("Could not find {0} to remove it".format(apkfile))
1104             else:
1105                 logging.error("Failed to get apk information, skipping " + apkfile)
1106             raise BuildException("Invaild APK")
1107     except ImportError:
1108         raise FDroidException("androguard library is not installed and aapt not present")
1109     except FileNotFoundError:
1110         logging.error("Could not open apk file for analysis")
1111         raise BuildException("Invalid APK")
1112
1113     apk['packageName'] = apkobject.get_package()
1114     apk['versionCode'] = int(apkobject.get_androidversion_code())
1115     apk['versionName'] = apkobject.get_androidversion_name()
1116     if apk['versionName'][0] == "@":
1117         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1118         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1119         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1120     apk['name'] = apkobject.get_app_name()
1121
1122     if apkobject.get_max_sdk_version() is not None:
1123         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1124     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1125     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1126
1127     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1128     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1129
1130     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1131
1132     for file in apkobject.get_files():
1133         d_re = density_re.match(file)
1134         if d_re:
1135             folder = d_re.group(1).split('-')
1136             if len(folder) > 1:
1137                 resolution = folder[1]
1138             else:
1139                 resolution = 'mdpi'
1140             density = screen_resolutions[resolution]
1141             apk['icons_src'][density] = d_re.group(0)
1142
1143     if apk['icons_src'].get('-1') is None:
1144         apk['icons_src']['-1'] = apk['icons_src']['160']
1145
1146     arch_re = re.compile("^lib/(.*)/.*$")
1147     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1148     if len(arch) >= 1:
1149         apk['nativecode'] = []
1150         apk['nativecode'].extend(sorted(list(arch)))
1151
1152     xml = apkobject.get_android_manifest_xml()
1153
1154     for item in xml.getElementsByTagName('uses-permission'):
1155         name = str(item.getAttribute("android:name"))
1156         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1157         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1158         permission = UsesPermission(
1159             name,
1160             maxSdkVersion
1161         )
1162         apk['uses-permission'].append(permission)
1163
1164     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1165         name = str(item.getAttribute("android:name"))
1166         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1167         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1168         permission_sdk_23 = UsesPermissionSdk23(
1169             name,
1170             maxSdkVersion
1171         )
1172         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1173
1174     for item in xml.getElementsByTagName('uses-feature'):
1175         feature = str(item.getAttribute("android:name"))
1176         if feature != "android.hardware.screen.portrait" \
1177                 and feature != "android.hardware.screen.landscape":
1178             if feature.startswith("android.feature."):
1179                 feature = feature[16:]
1180         apk['features'].append(feature)
1181
1182
1183 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1184                 allow_disabled_algorithms=False, archive_bad_sig=False):
1185     """Processes the apk with the given filename in the given repo directory.
1186
1187     This also extracts the icons.
1188
1189     :param apkcache: current apk cache information
1190     :param apkfilename: the filename of the apk to scan
1191     :param repodir: repo directory to scan
1192     :param knownapks: known apks info
1193     :param use_date_from_apk: use date from APK (instead of current date)
1194                               for newly added APKs
1195     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1196                                       disabled algorithms in the signature (e.g. MD5)
1197     :param archive_bad_sig: move APKs with a bad signature to the archive
1198     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1199      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1200     """
1201
1202     apk = {}
1203     apkfile = os.path.join(repodir, apkfilename)
1204
1205     cachechanged = False
1206     usecache = False
1207     if apkfilename in apkcache:
1208         apk = apkcache[apkfilename]
1209         if apk.get('hash') == sha256sum(apkfile):
1210             logging.debug("Reading " + apkfilename + " from cache")
1211             usecache = True
1212         else:
1213             logging.debug("Ignoring stale cache data for " + apkfilename)
1214
1215     if not usecache:
1216         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1217
1218         try:
1219             apk = scan_apk(apkfile)
1220         except BuildException:
1221             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1222                             .format(apkfilename=apkfilename))
1223             return True, None, False
1224
1225         # Check for debuggable apks...
1226         if common.isApkAndDebuggable(apkfile):
1227             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1228
1229         if options.rename_apks:
1230             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1231             std_short_name = os.path.join(repodir, n)
1232             if apkfile != std_short_name:
1233                 if os.path.exists(std_short_name):
1234                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1235                     if apkfile != std_long_name:
1236                         if os.path.exists(std_long_name):
1237                             dupdir = os.path.join('duplicates', repodir)
1238                             if not os.path.isdir(dupdir):
1239                                 os.makedirs(dupdir, exist_ok=True)
1240                             dupfile = os.path.join('duplicates', std_long_name)
1241                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1242                             os.rename(apkfile, dupfile)
1243                             return True, None, False
1244                         else:
1245                             os.rename(apkfile, std_long_name)
1246                     apkfile = std_long_name
1247                 else:
1248                     os.rename(apkfile, std_short_name)
1249                     apkfile = std_short_name
1250                 apkfilename = apkfile[len(repodir) + 1:]
1251
1252         apk['apkName'] = apkfilename
1253         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1254         if os.path.exists(os.path.join(repodir, srcfilename)):
1255             apk['srcname'] = srcfilename
1256
1257         # verify the jar signature is correct, allow deprecated
1258         # algorithms only if the APK is in the archive.
1259         skipapk = False
1260         if not common.verify_apk_signature(apkfile):
1261             if repodir == 'archive' or allow_disabled_algorithms:
1262                 if common.verify_old_apk_signature(apkfile):
1263                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1264                 else:
1265                     skipapk = True
1266             else:
1267                 skipapk = True
1268
1269         if skipapk:
1270             if archive_bad_sig:
1271                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1272                 move_apk_between_sections(repodir, 'archive', apk)
1273             else:
1274                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1275             return True, None, False
1276
1277         apkzip = zipfile.ZipFile(apkfile, 'r')
1278
1279         # if an APK has files newer than the system time, suggest updating
1280         # the system clock.  This is useful for offline systems, used for
1281         # signing, which do not have another source of clock sync info. It
1282         # has to be more than 24 hours newer because ZIP/APK files do not
1283         # store timezone info
1284         manifest = apkzip.getinfo('AndroidManifest.xml')
1285         if manifest.date_time[1] == 0:  # month can't be zero
1286             logging.debug('AndroidManifest.xml has no date')
1287         else:
1288             dt_obj = datetime(*manifest.date_time)
1289             checkdt = dt_obj - timedelta(1)
1290             if datetime.today() < checkdt:
1291                 logging.warning('System clock is older than manifest in: '
1292                                 + apkfilename
1293                                 + '\nSet clock to that time using:\n'
1294                                 + 'sudo date -s "' + str(dt_obj) + '"')
1295
1296         # extract icons from APK zip file
1297         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1298         try:
1299             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1300         finally:
1301             apkzip.close()  # ensure that APK zip file gets closed
1302
1303         # resize existing icons for densities missing in the APK
1304         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1305
1306         if use_date_from_apk and manifest.date_time[1] != 0:
1307             default_date_param = datetime(*manifest.date_time)
1308         else:
1309             default_date_param = None
1310
1311         # Record in known apks, getting the added date at the same time..
1312         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1313                                     default_date=default_date_param)
1314         if added:
1315             apk['added'] = added
1316
1317         apkcache[apkfilename] = apk
1318         cachechanged = True
1319
1320     return False, apk, cachechanged
1321
1322
1323 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1324     """Processes the apks in the given repo directory.
1325
1326     This also extracts the icons.
1327
1328     :param apkcache: current apk cache information
1329     :param repodir: repo directory to scan
1330     :param knownapks: known apks info
1331     :param use_date_from_apk: use date from APK (instead of current date)
1332                               for newly added APKs
1333     :returns: (apks, cachechanged) where apks is a list of apk information,
1334               and cachechanged is True if the apkcache got changed.
1335     """
1336
1337     cachechanged = False
1338
1339     for icon_dir in get_all_icon_dirs(repodir):
1340         if os.path.exists(icon_dir):
1341             if options.clean:
1342                 shutil.rmtree(icon_dir)
1343                 os.makedirs(icon_dir)
1344         else:
1345             os.makedirs(icon_dir)
1346
1347     apks = []
1348     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1349         apkfilename = apkfile[len(repodir) + 1:]
1350         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1351         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1352                                              use_date_from_apk, ada, True)
1353         if skip:
1354             continue
1355         apks.append(apk)
1356         cachechanged = cachechanged or cachethis
1357
1358     return apks, cachechanged
1359
1360
1361 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1362     """
1363     Extracts icons from the given APK zip in various densities,
1364     saves them into given repo directory
1365     and stores their names in the APK metadata dictionary.
1366
1367     :param icon_filename: A string representing the icon's file name
1368     :param apk: A populated dictionary containing APK metadata.
1369                 Needs to have 'icons_src' key
1370     :param apkzip: An opened zipfile.ZipFile of the APK file
1371     :param repo_dir: The directory of the APK's repository
1372     :return: A list of icon densities that are missing
1373     """
1374     empty_densities = []
1375     for density in screen_densities:
1376         if density not in apk['icons_src']:
1377             empty_densities.append(density)
1378             continue
1379         icon_src = apk['icons_src'][density]
1380         icon_dir = get_icon_dir(repo_dir, density)
1381         icon_dest = os.path.join(icon_dir, icon_filename)
1382
1383         # Extract the icon files per density
1384         if icon_src.endswith('.xml'):
1385             png = os.path.basename(icon_src)[:-4] + '.png'
1386             for f in apkzip.namelist():
1387                 if f.endswith(png):
1388                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1389                     if m and screen_resolutions[m.group(2)] == density:
1390                         icon_src = f
1391             if icon_src.endswith('.xml'):
1392                 empty_densities.append(density)
1393                 continue
1394         try:
1395             with open(icon_dest, 'wb') as f:
1396                 f.write(get_icon_bytes(apkzip, icon_src))
1397             apk['icons'][density] = icon_filename
1398         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1399             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1400             del apk['icons_src'][density]
1401             empty_densities.append(density)
1402
1403     if '-1' in apk['icons_src']:
1404         icon_src = apk['icons_src']['-1']
1405         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1406         with open(icon_path, 'wb') as f:
1407             f.write(get_icon_bytes(apkzip, icon_src))
1408         try:
1409             im = Image.open(icon_path)
1410             dpi = px_to_dpi(im.size[0])
1411             for density in screen_densities:
1412                 if density in apk['icons']:
1413                     break
1414                 if density == screen_densities[-1] or dpi >= int(density):
1415                     apk['icons'][density] = icon_filename
1416                     shutil.move(icon_path,
1417                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1418                     empty_densities.remove(density)
1419                     break
1420         except Exception as e:
1421             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1422
1423     if apk['icons']:
1424         apk['icon'] = icon_filename
1425
1426     return empty_densities
1427
1428
1429 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1430     """
1431     Resize existing icons for densities missing in the APK to ensure all densities are available
1432
1433     :param empty_densities: A list of icon densities that are missing
1434     :param icon_filename: A string representing the icon's file name
1435     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1436     :param repo_dir: The directory of the APK's repository
1437     """
1438     # First try resizing down to not lose quality
1439     last_density = None
1440     for density in screen_densities:
1441         if density not in empty_densities:
1442             last_density = density
1443             continue
1444         if last_density is None:
1445             continue
1446         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1447
1448         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1449         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1450         fp = None
1451         try:
1452             fp = open(last_icon_path, 'rb')
1453             im = Image.open(fp)
1454
1455             size = dpi_to_px(density)
1456
1457             im.thumbnail((size, size), Image.ANTIALIAS)
1458             im.save(icon_path, "PNG")
1459             empty_densities.remove(density)
1460         except Exception as e:
1461             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1462         finally:
1463             if fp:
1464                 fp.close()
1465
1466     # Then just copy from the highest resolution available
1467     last_density = None
1468     for density in reversed(screen_densities):
1469         if density not in empty_densities:
1470             last_density = density
1471             continue
1472
1473         if last_density is None:
1474             continue
1475
1476         shutil.copyfile(
1477             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1478             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1479         )
1480         empty_densities.remove(density)
1481
1482     for density in screen_densities:
1483         icon_dir = get_icon_dir(repo_dir, density)
1484         icon_dest = os.path.join(icon_dir, icon_filename)
1485         resize_icon(icon_dest, density)
1486
1487     # Copy from icons-mdpi to icons since mdpi is the baseline density
1488     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1489     if os.path.isfile(baseline):
1490         apk['icons']['0'] = icon_filename
1491         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1492
1493
1494 def apply_info_from_latest_apk(apps, apks):
1495     """
1496     Some information from the apks needs to be applied up to the application level.
1497     When doing this, we use the info from the most recent version's apk.
1498     We deal with figuring out when the app was added and last updated at the same time.
1499     """
1500     for appid, app in apps.items():
1501         bestver = UNSET_VERSION_CODE
1502         for apk in apks:
1503             if apk['packageName'] == appid:
1504                 if apk['versionCode'] > bestver:
1505                     bestver = apk['versionCode']
1506                     bestapk = apk
1507
1508                 if 'added' in apk:
1509                     if not app.added or apk['added'] < app.added:
1510                         app.added = apk['added']
1511                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1512                         app.lastUpdated = apk['added']
1513
1514         if not app.added:
1515             logging.debug("Don't know when " + appid + " was added")
1516         if not app.lastUpdated:
1517             logging.debug("Don't know when " + appid + " was last updated")
1518
1519         if bestver == UNSET_VERSION_CODE:
1520
1521             if app.Name is None:
1522                 app.Name = app.AutoName or appid
1523             app.icon = None
1524             logging.debug("Application " + appid + " has no packages")
1525         else:
1526             if app.Name is None:
1527                 app.Name = bestapk['name']
1528             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1529             if app.CurrentVersionCode is None:
1530                 app.CurrentVersionCode = str(bestver)
1531
1532
1533 def make_categories_txt(repodir, categories):
1534     '''Write a category list in the repo to allow quick access'''
1535     catdata = ''
1536     for cat in sorted(categories):
1537         catdata += cat + '\n'
1538     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1539         f.write(catdata)
1540
1541
1542 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1543
1544     def filter_apk_list_sorted(apk_list):
1545         res = []
1546         for apk in apk_list:
1547             if apk['packageName'] == appid:
1548                 res.append(apk)
1549
1550         # Sort the apk list by version code. First is highest/newest.
1551         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1552
1553     for appid, app in apps.items():
1554
1555         if app.ArchivePolicy:
1556             keepversions = int(app.ArchivePolicy[:-9])
1557         else:
1558             keepversions = defaultkeepversions
1559
1560         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1561                       .format(appid, len(apks), keepversions, len(archapks)))
1562
1563         current_app_apks = filter_apk_list_sorted(apks)
1564         if len(current_app_apks) > keepversions:
1565             # Move back the ones we don't want.
1566             for apk in current_app_apks[keepversions:]:
1567                 move_apk_between_sections(repodir, archivedir, apk)
1568                 archapks.append(apk)
1569                 apks.remove(apk)
1570
1571         current_app_archapks = filter_apk_list_sorted(archapks)
1572         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1573             kept = 0
1574             # Move forward the ones we want again, except DisableAlgorithm
1575             for apk in current_app_archapks:
1576                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1577                     move_apk_between_sections(archivedir, repodir, apk)
1578                     archapks.remove(apk)
1579                     apks.append(apk)
1580                     kept += 1
1581                 if kept == keepversions:
1582                     break
1583
1584
1585 def move_apk_between_sections(from_dir, to_dir, apk):
1586     """move an APK from repo to archive or vice versa"""
1587
1588     def _move_file(from_dir, to_dir, filename, ignore_missing):
1589         from_path = os.path.join(from_dir, filename)
1590         if ignore_missing and not os.path.exists(from_path):
1591             return
1592         to_path = os.path.join(to_dir, filename)
1593         if not os.path.exists(to_dir):
1594             os.mkdir(to_dir)
1595         shutil.move(from_path, to_path)
1596
1597     if from_dir == to_dir:
1598         return
1599
1600     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1601     _move_file(from_dir, to_dir, apk['apkName'], False)
1602     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1603     for density in all_screen_densities:
1604         from_icon_dir = get_icon_dir(from_dir, density)
1605         to_icon_dir = get_icon_dir(to_dir, density)
1606         if density not in apk['icons']:
1607             continue
1608         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1609     if 'srcname' in apk:
1610         _move_file(from_dir, to_dir, apk['srcname'], False)
1611
1612
1613 def add_apks_to_per_app_repos(repodir, apks):
1614     apks_per_app = dict()
1615     for apk in apks:
1616         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1617         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1618         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1619         apks_per_app[apk['packageName']] = apk
1620
1621         if not os.path.exists(apk['per_app_icons']):
1622             logging.info('Adding new repo for only ' + apk['packageName'])
1623             os.makedirs(apk['per_app_icons'])
1624
1625         apkpath = os.path.join(repodir, apk['apkName'])
1626         shutil.copy(apkpath, apk['per_app_repo'])
1627         apksigpath = apkpath + '.sig'
1628         if os.path.exists(apksigpath):
1629             shutil.copy(apksigpath, apk['per_app_repo'])
1630         apkascpath = apkpath + '.asc'
1631         if os.path.exists(apkascpath):
1632             shutil.copy(apkascpath, apk['per_app_repo'])
1633
1634
1635 def create_metadata_from_template(apk):
1636     '''create a new metadata file using internal or external template
1637
1638     Generate warnings for apk's with no metadata (or create skeleton
1639     metadata files, if requested on the command line).  Though the
1640     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1641     since those impose things on the metadata file made from the
1642     template: field sort order, empty field value, formatting, etc.
1643     '''
1644
1645     import yaml
1646     if os.path.exists('template.yml'):
1647         with open('template.yml') as f:
1648             metatxt = f.read()
1649         if 'name' in apk and apk['name'] != '':
1650             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1651                              r'\1 ' + apk['name'],
1652                              metatxt,
1653                              flags=re.IGNORECASE | re.MULTILINE)
1654         else:
1655             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1656             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1657                              r'\1 ' + apk['packageName'],
1658                              metatxt,
1659                              flags=re.IGNORECASE | re.MULTILINE)
1660         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1661             f.write(metatxt)
1662     else:
1663         app = dict()
1664         app['Categories'] = [os.path.basename(os.getcwd())]
1665         # include some blanks as part of the template
1666         app['AuthorName'] = ''
1667         app['Summary'] = ''
1668         app['WebSite'] = ''
1669         app['IssueTracker'] = ''
1670         app['SourceCode'] = ''
1671         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1672         if 'name' in apk and apk['name'] != '':
1673             app['Name'] = apk['name']
1674         else:
1675             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1676             app['Name'] = apk['packageName']
1677         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1678             yaml.dump(app, f, default_flow_style=False)
1679     logging.info("Generated skeleton metadata for " + apk['packageName'])
1680
1681
1682 config = None
1683 options = None
1684
1685
1686 def main():
1687
1688     global config, options
1689
1690     # Parse command line...
1691     parser = ArgumentParser()
1692     common.setup_global_opts(parser)
1693     parser.add_argument("--create-key", action="store_true", default=False,
1694                         help=_("Create a repo signing key in a keystore"))
1695     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1696                         help=_("Create skeleton metadata files that are missing"))
1697     parser.add_argument("--delete-unknown", action="store_true", default=False,
1698                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1699     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1700                         help=_("Report on build data status"))
1701     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1702                         help=_("Interactively ask about things that need updating."))
1703     parser.add_argument("-I", "--icons", action="store_true", default=False,
1704                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1705     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1706                         help=_("Specify editor to use in interactive mode. Default ") +
1707                         "is /etc/alternatives/editor")
1708     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1709                         help=_("Update the wiki"))
1710     parser.add_argument("--pretty", action="store_true", default=False,
1711                         help=_("Produce human-readable index.xml"))
1712     parser.add_argument("--clean", action="store_true", default=False,
1713                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1714     parser.add_argument("--nosign", action="store_true", default=False,
1715                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1716     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1717                         help=_("Use date from APK instead of current time for newly added APKs"))
1718     parser.add_argument("--rename-apks", action="store_true", default=False,
1719                         help=_("Rename APK files that do not match package.name_123.apk"))
1720     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1721                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1722     metadata.add_metadata_arguments(parser)
1723     options = parser.parse_args()
1724     metadata.warnings_action = options.W
1725
1726     config = common.read_config(options)
1727
1728     if not ('jarsigner' in config and 'keytool' in config):
1729         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1730
1731     repodirs = ['repo']
1732     if config['archive_older'] != 0:
1733         repodirs.append('archive')
1734         if not os.path.exists('archive'):
1735             os.mkdir('archive')
1736
1737     if options.icons:
1738         resize_all_icons(repodirs)
1739         sys.exit(0)
1740
1741     if options.rename_apks:
1742         options.clean = True
1743
1744     # check that icons exist now, rather than fail at the end of `fdroid update`
1745     for k in ['repo_icon', 'archive_icon']:
1746         if k in config:
1747             if not os.path.exists(config[k]):
1748                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1749                 sys.exit(1)
1750
1751     # if the user asks to create a keystore, do it now, reusing whatever it can
1752     if options.create_key:
1753         if os.path.exists(config['keystore']):
1754             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1755             logging.critical("\t'" + config['keystore'] + "'")
1756             sys.exit(1)
1757
1758         if 'repo_keyalias' not in config:
1759             config['repo_keyalias'] = socket.getfqdn()
1760             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1761         if 'keydname' not in config:
1762             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1763             common.write_to_config(config, 'keydname', config['keydname'])
1764         if 'keystore' not in config:
1765             config['keystore'] = common.default_config['keystore']
1766             common.write_to_config(config, 'keystore', config['keystore'])
1767
1768         password = common.genpassword()
1769         if 'keystorepass' not in config:
1770             config['keystorepass'] = password
1771             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1772         if 'keypass' not in config:
1773             config['keypass'] = password
1774             common.write_to_config(config, 'keypass', config['keypass'])
1775         common.genkeystore(config)
1776
1777     # Get all apps...
1778     apps = metadata.read_metadata()
1779
1780     # Generate a list of categories...
1781     categories = set()
1782     for app in apps.values():
1783         categories.update(app.Categories)
1784
1785     # Read known apks data (will be updated and written back when we've finished)
1786     knownapks = common.KnownApks()
1787
1788     # Get APK cache
1789     apkcache = get_cache()
1790
1791     # Delete builds for disabled apps
1792     delete_disabled_builds(apps, apkcache, repodirs)
1793
1794     # Scan all apks in the main repo
1795     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1796
1797     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1798                                            options.use_date_from_apk)
1799     cachechanged = cachechanged or fcachechanged
1800     apks += files
1801     for apk in apks:
1802         if apk['packageName'] not in apps:
1803             if options.create_metadata:
1804                 create_metadata_from_template(apk)
1805                 apps = metadata.read_metadata()
1806             else:
1807                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1808                 if options.delete_unknown:
1809                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1810                     rmf = os.path.join(repodirs[0], apk['apkName'])
1811                     if not os.path.exists(rmf):
1812                         logging.error("Could not find {0} to remove it".format(rmf))
1813                     else:
1814                         os.remove(rmf)
1815                 else:
1816                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1817
1818     copy_triple_t_store_metadata(apps)
1819     insert_obbs(repodirs[0], apps, apks)
1820     insert_localized_app_metadata(apps)
1821     translate_per_build_anti_features(apps, apks)
1822
1823     # Scan the archive repo for apks as well
1824     if len(repodirs) > 1:
1825         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1826         if cc:
1827             cachechanged = True
1828     else:
1829         archapks = []
1830
1831     # Apply information from latest apks to the application and update dates
1832     apply_info_from_latest_apk(apps, apks + archapks)
1833
1834     # Sort the app list by name, then the web site doesn't have to by default.
1835     # (we had to wait until we'd scanned the apks to do this, because mostly the
1836     # name comes from there!)
1837     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1838
1839     # APKs are placed into multiple repos based on the app package, providing
1840     # per-app subscription feeds for nightly builds and things like it
1841     if config['per_app_repos']:
1842         add_apks_to_per_app_repos(repodirs[0], apks)
1843         for appid, app in apps.items():
1844             repodir = os.path.join(appid, 'fdroid', 'repo')
1845             appdict = dict()
1846             appdict[appid] = app
1847             if os.path.isdir(repodir):
1848                 index.make(appdict, [appid], apks, repodir, False)
1849             else:
1850                 logging.info('Skipping index generation for ' + appid)
1851         return
1852
1853     if len(repodirs) > 1:
1854         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1855
1856     # Make the index for the main repo...
1857     index.make(apps, sortedids, apks, repodirs[0], False)
1858     make_categories_txt(repodirs[0], categories)
1859
1860     # If there's an archive repo,  make the index for it. We already scanned it
1861     # earlier on.
1862     if len(repodirs) > 1:
1863         index.make(apps, sortedids, archapks, repodirs[1], True)
1864
1865     git_remote = config.get('binary_transparency_remote')
1866     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1867         from . import btlog
1868         btlog.make_binary_transparency_log(repodirs)
1869
1870     if config['update_stats']:
1871         # Update known apks info...
1872         knownapks.writeifchanged()
1873
1874         # Generate latest apps data for widget
1875         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1876             data = ''
1877             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1878                 for line in f:
1879                     appid = line.rstrip()
1880                     data += appid + "\t"
1881                     app = apps[appid]
1882                     data += app.Name + "\t"
1883                     if app.icon is not None:
1884                         data += app.icon + "\t"
1885                     data += app.License + "\n"
1886             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1887                 f.write(data)
1888
1889     if cachechanged:
1890         write_cache(apkcache)
1891
1892     # Update the wiki...
1893     if options.wiki:
1894         update_wiki(apps, sortedids, apks + archapks)
1895
1896     logging.info(_("Finished"))
1897
1898
1899 if __name__ == "__main__":
1900     main()