fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import _
  41 from . import common
  42 from . import index
  43 from . import metadata
  44 from .common import SdkToolsPopen
  45 from .exception import BuildException, FDroidException
  46
  47 METADATA_VERSION = 18
  48
  49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  50 UNSET_VERSION_CODE = -0x100000000
  51
  52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  59 APK_PERMISSION_PAT = \
  60     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  62
  63 screen_densities = ['640', '480', '320', '240', '160', '120']
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "undefined": '-1',
  72     "anydpi": '65534',
  73     "nodpi": '65535'
  74 }
  75
  76 all_screen_densities = ['0'] + screen_densities
  77
  78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  80
  81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  84                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  85
  86
  87 def dpi_to_px(density):
  88     return (int(density) * 48) / 160
  89
  90
  91 def px_to_dpi(px):
  92     return (int(px) * 160) / 48
  93
  94
  95 def get_icon_dir(repodir, density):
  96     if density == '0':
  97         return os.path.join(repodir, "icons")
  98     return os.path.join(repodir, "icons-%s" % density)
  99
 100
 101 def get_icon_dirs(repodir):
 102     for density in screen_densities:
 103         yield get_icon_dir(repodir, density)
 104
 105
 106 def get_all_icon_dirs(repodir):
 107     for density in all_screen_densities:
 108         yield get_icon_dir(repodir, density)
 109
 110
 111 def update_wiki(apps, sortedids, apks):
 112     """Update the wiki
 113
 114     :param apps: fully populated list of all applications
 115     :param apks: all apks, except...
 116     """
 117     logging.info("Updating wiki")
 118     wikicat = 'Apps'
 119     wikiredircat = 'App Redirects'
 120     import mwclient
 121     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 122                          path=config['wiki_path'])
 123     site.login(config['wiki_user'], config['wiki_password'])
 124     generated_pages = {}
 125     generated_redirects = {}
 126
 127     for appid in sortedids:
 128         app = metadata.App(apps[appid])
 129
 130         wikidata = ''
 131         if app.Disabled:
 132             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 133         if app.AntiFeatures:
 134             for af in app.AntiFeatures:
 135                 wikidata += '{{AntiFeature|' + af + '}}\n'
 136         if app.RequiresRoot:
 137             requiresroot = 'Yes'
 138         else:
 139             requiresroot = 'No'
 140         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 141             appid,
 142             app.Name,
 143             app.added.strftime('%Y-%m-%d') if app.added else '',
 144             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 145             app.SourceCode,
 146             app.IssueTracker,
 147             app.WebSite,
 148             app.Changelog,
 149             app.Donate,
 150             app.FlattrID,
 151             app.Bitcoin,
 152             app.Litecoin,
 153             app.License,
 154             requiresroot,
 155             app.AuthorName,
 156             app.AuthorEmail)
 157
 158         if app.Provides:
 159             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 160
 161         wikidata += app.Summary
 162         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 163
 164         wikidata += "=Description=\n"
 165         wikidata += metadata.description_wiki(app.Description) + "\n"
 166
 167         wikidata += "=Maintainer Notes=\n"
 168         if app.MaintainerNotes:
 169             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 170         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 171
 172         # Get a list of all packages for this application...
 173         apklist = []
 174         gotcurrentver = False
 175         cantupdate = False
 176         buildfails = False
 177         for apk in apks:
 178             if apk['packageName'] == appid:
 179                 if str(apk['versionCode']) == app.CurrentVersionCode:
 180                     gotcurrentver = True
 181                 apklist.append(apk)
 182         # Include ones we can't build, as a special case...
 183         for build in app.builds:
 184             if build.disable:
 185                 if build.versionCode == app.CurrentVersionCode:
 186                     cantupdate = True
 187                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 188                 apklist.append({'versionCode': int(build.versionCode),
 189                                 'versionName': build.versionName,
 190                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 191                                 })
 192             else:
 193                 builtit = False
 194                 for apk in apklist:
 195                     if apk['versionCode'] == int(build.versionCode):
 196                         builtit = True
 197                         break
 198                 if not builtit:
 199                     buildfails = True
 200                     apklist.append({'versionCode': int(build.versionCode),
 201                                     'versionName': build.versionName,
 202                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 203                                     })
 204         if app.CurrentVersionCode == '0':
 205             cantupdate = True
 206         # Sort with most recent first...
 207         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 208
 209         wikidata += "=Versions=\n"
 210         if len(apklist) == 0:
 211             wikidata += "We currently have no versions of this app available."
 212         elif not gotcurrentver:
 213             wikidata += "We don't have the current version of this app."
 214         else:
 215             wikidata += "We have the current version of this app."
 216         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 217         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 218         if len(app.NoSourceSince) > 0:
 219             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 220         if len(app.CurrentVersion) > 0:
 221             wikidata += "The current (recommended) version is " + app.CurrentVersion
 222             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 223         validapks = 0
 224         for apk in apklist:
 225             wikidata += "==" + apk['versionName'] + "==\n"
 226
 227             if 'buildproblem' in apk:
 228                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 229             else:
 230                 validapks += 1
 231                 wikidata += "This version is built and signed by "
 232                 if 'srcname' in apk:
 233                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 234                 else:
 235                     wikidata += "the original developer.\n\n"
 236             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 237
 238         wikidata += '\n[[Category:' + wikicat + ']]\n'
 239         if len(app.NoSourceSince) > 0:
 240             wikidata += '\n[[Category:Apps missing source code]]\n'
 241         if validapks == 0 and not app.Disabled:
 242             wikidata += '\n[[Category:Apps with no packages]]\n'
 243         if cantupdate and not app.Disabled:
 244             wikidata += "\n[[Category:Apps we cannot update]]\n"
 245         if buildfails and not app.Disabled:
 246             wikidata += "\n[[Category:Apps with failing builds]]\n"
 247         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 248             wikidata += '\n[[Category:Apps to Update]]\n'
 249         if app.Disabled:
 250             wikidata += '\n[[Category:Apps that are disabled]]\n'
 251         if app.UpdateCheckMode == 'None' and not app.Disabled:
 252             wikidata += '\n[[Category:Apps with no update check]]\n'
 253         for appcat in app.Categories:
 254             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 255
 256         # We can't have underscores in the page name, even if they're in
 257         # the package ID, because MediaWiki messes with them...
 258         pagename = appid.replace('_', ' ')
 259
 260         # Drop a trailing newline, because mediawiki is going to drop it anyway
 261         # and it we don't we'll think the page has changed when it hasn't...
 262         if wikidata.endswith('\n'):
 263             wikidata = wikidata[:-1]
 264
 265         generated_pages[pagename] = wikidata
 266
 267         # Make a redirect from the name to the ID too, unless there's
 268         # already an existing page with the name and it isn't a redirect.
 269         noclobber = False
 270         apppagename = app.Name.replace('_', ' ')
 271         apppagename = apppagename.replace('{', '')
 272         apppagename = apppagename.replace('}', ' ')
 273         apppagename = apppagename.replace(':', ' ')
 274         apppagename = apppagename.replace('[', ' ')
 275         apppagename = apppagename.replace(']', ' ')
 276         # Drop double spaces caused mostly by replacing ':' above
 277         apppagename = apppagename.replace('  ', ' ')
 278         for expagename in site.allpages(prefix=apppagename,
 279                                         filterredir='nonredirects',
 280                                         generator=False):
 281             if expagename == apppagename:
 282                 noclobber = True
 283         # Another reason not to make the redirect page is if the app name
 284         # is the same as it's ID, because that will overwrite the real page
 285         # with an redirect to itself! (Although it seems like an odd
 286         # scenario this happens a lot, e.g. where there is metadata but no
 287         # builds or binaries to extract a name from.
 288         if apppagename == pagename:
 289             noclobber = True
 290         if not noclobber:
 291             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 292
 293     for tcat, genp in [(wikicat, generated_pages),
 294                        (wikiredircat, generated_redirects)]:
 295         catpages = site.Pages['Category:' + tcat]
 296         existingpages = []
 297         for page in catpages:
 298             existingpages.append(page.name)
 299             if page.name in genp:
 300                 pagetxt = page.edit()
 301                 if pagetxt != genp[page.name]:
 302                     logging.debug("Updating modified page " + page.name)
 303                     page.save(genp[page.name], summary='Auto-updated')
 304                 else:
 305                     logging.debug("Page " + page.name + " is unchanged")
 306             else:
 307                 logging.warn("Deleting page " + page.name)
 308                 page.delete('No longer published')
 309         for pagename, text in genp.items():
 310             logging.debug("Checking " + pagename)
 311             if pagename not in existingpages:
 312                 logging.debug("Creating page " + pagename)
 313                 try:
 314                     newpage = site.Pages[pagename]
 315                     newpage.save(text, summary='Auto-created')
 316                 except Exception as e:
 317                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 318
 319     # Purge server cache to ensure counts are up to date
 320     site.pages['Repository Maintenance'].purge()
 321
 322
 323 def delete_disabled_builds(apps, apkcache, repodirs):
 324     """Delete disabled build outputs.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodirs: the repo directories to process
 329     """
 330     for appid, app in apps.items():
 331         for build in app['builds']:
 332             if not build.disable:
 333                 continue
 334             apkfilename = common.get_release_filename(app, build)
 335             iconfilename = "%s.%s.png" % (
 336                 appid,
 337                 build.versionCode)
 338             for repodir in repodirs:
 339                 files = [
 340                     os.path.join(repodir, apkfilename),
 341                     os.path.join(repodir, apkfilename + '.asc'),
 342                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 343                 ]
 344                 for density in all_screen_densities:
 345                     repo_dir = get_icon_dir(repodir, density)
 346                     files.append(os.path.join(repo_dir, iconfilename))
 347
 348                 for f in files:
 349                     if os.path.exists(f):
 350                         logging.info("Deleting disabled build output " + f)
 351                         os.remove(f)
 352             if apkfilename in apkcache:
 353                 del apkcache[apkfilename]
 354
 355
 356 def resize_icon(iconpath, density):
 357
 358     if not os.path.isfile(iconpath):
 359         return
 360
 361     fp = None
 362     try:
 363         fp = open(iconpath, 'rb')
 364         im = Image.open(fp)
 365         size = dpi_to_px(density)
 366
 367         if any(length > size for length in im.size):
 368             oldsize = im.size
 369             im.thumbnail((size, size), Image.ANTIALIAS)
 370             logging.debug("%s was too large at %s - new size is %s" % (
 371                 iconpath, oldsize, im.size))
 372             im.save(iconpath, "PNG")
 373
 374     except Exception as e:
 375         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 376
 377     finally:
 378         if fp:
 379             fp.close()
 380
 381
 382 def resize_all_icons(repodirs):
 383     """Resize all icons that exceed the max size
 384
 385     :param repodirs: the repo directories to process
 386     """
 387     for repodir in repodirs:
 388         for density in screen_densities:
 389             icon_dir = get_icon_dir(repodir, density)
 390             icon_glob = os.path.join(icon_dir, '*.png')
 391             for iconpath in glob.glob(icon_glob):
 392                 resize_icon(iconpath, density)
 393
 394
 395 def getsig(apkpath):
 396     """ Get the signing certificate of an apk. To get the same md5 has that
 397     Android gets, we encode the .RSA certificate in a specific format and pass
 398     it hex-encoded to the md5 digest algorithm.
 399
 400     :param apkpath: path to the apk
 401     :returns: A string containing the md5 of the signature of the apk or None
 402               if an error occurred.
 403     """
 404
 405     with zipfile.ZipFile(apkpath, 'r') as apk:
 406         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 407
 408         if len(certs) < 1:
 409             logging.error("Found no signing certificates on %s" % apkpath)
 410             return None
 411         if len(certs) > 1:
 412             logging.error("Found multiple signing certificates on %s" % apkpath)
 413             return None
 414
 415         cert = apk.read(certs[0])
 416
 417     cert_encoded = common.get_certificate(cert)
 418
 419     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 420
 421
 422 def get_cache_file():
 423     return os.path.join('tmp', 'apkcache')
 424
 425
 426 def get_cache():
 427     """Get the cached dict of the APK index
 428
 429     Gather information about all the apk files in the repo directory,
 430     using cached data if possible. Some of the index operations take a
 431     long time, like calculating the SHA-256 and verifying the APK
 432     signature.
 433
 434     The cache is invalidated if the metadata version is different, or
 435     the 'allow_disabled_algorithms' config/option is different.  In
 436     those cases, there is no easy way to know what has changed from
 437     the cache, so just rerun the whole thing.
 438
 439     :return: apkcache
 440
 441     """
 442     apkcachefile = get_cache_file()
 443     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 444     if not options.clean and os.path.exists(apkcachefile):
 445         with open(apkcachefile, 'rb') as cf:
 446             apkcache = pickle.load(cf, encoding='utf-8')
 447         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 448            or apkcache.get('allow_disabled_algorithms') != ada:
 449             apkcache = {}
 450     else:
 451         apkcache = {}
 452
 453     apkcache["METADATA_VERSION"] = METADATA_VERSION
 454     apkcache['allow_disabled_algorithms'] = ada
 455
 456     return apkcache
 457
 458
 459 def write_cache(apkcache):
 460     apkcachefile = get_cache_file()
 461     cache_path = os.path.dirname(apkcachefile)
 462     if not os.path.exists(cache_path):
 463         os.makedirs(cache_path)
 464     with open(apkcachefile, 'wb') as cf:
 465         pickle.dump(apkcache, cf)
 466
 467
 468 def get_icon_bytes(apkzip, iconsrc):
 469     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 470     try:
 471         return apkzip.read(iconsrc)
 472     except KeyError:
 473         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 474
 475
 476 def sha256sum(filename):
 477     '''Calculate the sha256 of the given file'''
 478     sha = hashlib.sha256()
 479     with open(filename, 'rb') as f:
 480         while True:
 481             t = f.read(16384)
 482             if len(t) == 0:
 483                 break
 484             sha.update(t)
 485     return sha.hexdigest()
 486
 487
 488 def has_known_vulnerability(filename):
 489     """checks for known vulnerabilities in the APK
 490
 491     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 492     version.  Google also enforces this:
 493     https://support.google.com/faqs/answer/6376725?hl=en
 494
 495     Checks whether there are more than one classes.dex or AndroidManifest.xml
 496     files, which is invalid and an essential part of the "Master Key" attack.
 497
 498     http://www.saurik.com/id/17
 499     """
 500
 501     # statically load this pattern
 502     if not hasattr(has_known_vulnerability, "pattern"):
 503         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 504
 505     files_in_apk = set()
 506     with zipfile.ZipFile(filename) as zf:
 507         for name in zf.namelist():
 508             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 509                 lib = zf.open(name)
 510                 while True:
 511                     chunk = lib.read(4096)
 512                     if chunk == b'':
 513                         break
 514                     m = has_known_vulnerability.pattern.search(chunk)
 515                     if m:
 516                         version = m.group(1).decode('ascii')
 517                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 518                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 519                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 520                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 521                         else:
 522                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 523                             return True
 524                         break
 525             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 526                 if name in files_in_apk:
 527                     return True
 528                 files_in_apk.add(name)
 529
 530     return False
 531
 532
 533 def insert_obbs(repodir, apps, apks):
 534     """Scans the .obb files in a given repo directory and adds them to the
 535     relevant APK instances.  OBB files have versionCodes like APK
 536     files, and they are loosely associated.  If there is an OBB file
 537     present, then any APK with the same or higher versionCode will use
 538     that OBB file.  There are two OBB types: main and patch, each APK
 539     can only have only have one of each.
 540
 541     https://developer.android.com/google/play/expansion-files.html
 542
 543     :param repodir: repo directory to scan
 544     :param apps: list of current, valid apps
 545     :param apks: current information on all APKs
 546
 547     """
 548
 549     def obbWarnDelete(f, msg):
 550         logging.warning(msg + f)
 551         if options.delete_unknown:
 552             logging.error("Deleting unknown file: " + f)
 553             os.remove(f)
 554
 555     obbs = []
 556     java_Integer_MIN_VALUE = -pow(2, 31)
 557     currentPackageNames = apps.keys()
 558     for f in glob.glob(os.path.join(repodir, '*.obb')):
 559         obbfile = os.path.basename(f)
 560         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 561         chunks = obbfile.split('.')
 562         if chunks[0] != 'main' and chunks[0] != 'patch':
 563             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 564             continue
 565         if not re.match(r'^-?[0-9]+$', chunks[1]):
 566             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 567             continue
 568         versionCode = int(chunks[1])
 569         packagename = ".".join(chunks[2:-1])
 570
 571         highestVersionCode = java_Integer_MIN_VALUE
 572         if packagename not in currentPackageNames:
 573             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 574             continue
 575         for apk in apks:
 576             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 577                 highestVersionCode = apk['versionCode']
 578         if versionCode > highestVersionCode:
 579             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 580                           + ') than any APK: ')
 581             continue
 582         obbsha256 = sha256sum(f)
 583         obbs.append((packagename, versionCode, obbfile, obbsha256))
 584
 585     for apk in apks:
 586         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 587             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 588                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 589                     apk['obbMainFile'] = obbfile
 590                     apk['obbMainFileSha256'] = obbsha256
 591                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 592                     apk['obbPatchFile'] = obbfile
 593                     apk['obbPatchFileSha256'] = obbsha256
 594             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 595                 break
 596
 597
 598 def translate_per_build_anti_features(apps, apks):
 599     """Grab the anti-features list from the build metadata
 600
 601     For most Anti-Features, they are really most applicable per-APK,
 602     not for an app.  An app can fix a vulnerability, add/remove
 603     tracking, etc.  This reads the 'antifeatures' list from the Build
 604     entries in the fdroiddata metadata file, then transforms it into
 605     the 'antiFeatures' list of unique items for the index.
 606
 607     The field key is all lower case in the metadata file to match the
 608     rest of the Build fields.  It is 'antiFeatures' camel case in the
 609     implementation, index, and fdroidclient since it is translated
 610     from the build 'antifeatures' field, not directly included.
 611
 612     """
 613
 614     antiFeatures = dict()
 615     for packageName, app in apps.items():
 616         d = dict()
 617         for build in app['builds']:
 618             afl = build.get('antifeatures')
 619             if afl:
 620                 d[int(build.versionCode)] = afl
 621         if len(d) > 0:
 622             antiFeatures[packageName] = d
 623
 624     for apk in apks:
 625         d = antiFeatures.get(apk['packageName'])
 626         if d:
 627             afl = d.get(apk['versionCode'])
 628             if afl:
 629                 apk['antiFeatures'].update(afl)
 630
 631
 632 def _get_localized_dict(app, locale):
 633     '''get the dict to add localized store metadata to'''
 634     if 'localized' not in app:
 635         app['localized'] = collections.OrderedDict()
 636     if locale not in app['localized']:
 637         app['localized'][locale] = collections.OrderedDict()
 638     return app['localized'][locale]
 639
 640
 641 def _set_localized_text_entry(app, locale, key, f):
 642     limit = config['char_limits'][key]
 643     localized = _get_localized_dict(app, locale)
 644     with open(f) as fp:
 645         text = fp.read()[:limit]
 646         if len(text) > 0:
 647             localized[key] = text
 648
 649
 650 def _set_author_entry(app, key, f):
 651     limit = config['char_limits']['author']
 652     with open(f) as fp:
 653         text = fp.read()[:limit]
 654         if len(text) > 0:
 655             app[key] = text
 656
 657
 658 def copy_triple_t_store_metadata(apps):
 659     """Include store metadata from the app's source repo
 660
 661     The Triple-T Gradle Play Publisher is a plugin that has a standard
 662     file layout for all of the metadata and graphics that the Google
 663     Play Store accepts.  Since F-Droid has the git repo, it can just
 664     pluck those files directly.  This method reads any text files into
 665     the app dict, then copies any graphics into the fdroid repo
 666     directory structure.
 667
 668     This needs to be run before insert_localized_app_metadata() so that
 669     the graphics files that are copied into the fdroid repo get
 670     properly indexed.
 671
 672     https://github.com/Triple-T/gradle-play-publisher#upload-images
 673     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 674
 675     """
 676
 677     if not os.path.isdir('build'):
 678         return  # nothing to do
 679
 680     for packageName, app in apps.items():
 681         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 682             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 683             for root, dirs, files in os.walk(d):
 684                 segments = root.split('/')
 685                 locale = segments[-2]
 686                 for f in files:
 687                     if f == 'fulldescription':
 688                         _set_localized_text_entry(app, locale, 'description',
 689                                                   os.path.join(root, f))
 690                         continue
 691                     elif f == 'shortdescription':
 692                         _set_localized_text_entry(app, locale, 'summary',
 693                                                   os.path.join(root, f))
 694                         continue
 695                     elif f == 'title':
 696                         _set_localized_text_entry(app, locale, 'name',
 697                                                   os.path.join(root, f))
 698                         continue
 699                     elif f == 'video':
 700                         _set_localized_text_entry(app, locale, 'video',
 701                                                   os.path.join(root, f))
 702                         continue
 703                     elif f == 'whatsnew':
 704                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 705                                                   os.path.join(root, f))
 706                         continue
 707                     elif f == 'contactEmail':
 708                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 709                         continue
 710                     elif f == 'contactPhone':
 711                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 712                         continue
 713                     elif f == 'contactWebsite':
 714                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 715                         continue
 716
 717                     base, extension = common.get_extension(f)
 718                     dirname = os.path.basename(root)
 719                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 720                         if segments[-2] == 'listing':
 721                             locale = segments[-3]
 722                         else:
 723                             locale = segments[-2]
 724                         destdir = os.path.join('repo', packageName, locale)
 725                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 726                         sourcefile = os.path.join(root, f)
 727                         destfile = os.path.join(destdir, dirname + '.' + extension)
 728                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 729                         shutil.copy(sourcefile, destfile)
 730
 731
 732 def insert_localized_app_metadata(apps):
 733     """scans standard locations for graphics and localized text
 734
 735     Scans for localized description files, store graphics, and
 736     screenshot PNG files in statically defined screenshots directory
 737     and adds them to the app metadata.  The screenshots and graphic
 738     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 739     and must be in the following layout:
 740     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 741
 742     repo/packageName/locale/featureGraphic.png
 743     repo/packageName/locale/phoneScreenshots/1.png
 744     repo/packageName/locale/phoneScreenshots/2.png
 745
 746     The changelog files must be text files named with the versionCode
 747     ending with ".txt" and must be in the following layout:
 748     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 749
 750     repo/packageName/locale/changelogs/12345.txt
 751
 752     This will scan the each app's source repo then the metadata/ dir
 753     for these standard locations of changelog files.  If it finds
 754     them, they will be added to the dict of all packages, with the
 755     versions in the metadata/ folder taking precendence over the what
 756     is in the app's source repo.
 757
 758     Where "packageName" is the app's packageName and "locale" is the locale
 759     of the graphics, e.g. what language they are in, using the IETF RFC5646
 760     format (en-US, fr-CA, es-MX, etc).
 761
 762     This will also scan the app's git for a fastlane folder, and the
 763     metadata/ folder and the apps' source repos for standard locations
 764     of graphic and screenshot files.  If it finds them, it will copy
 765     them into the repo.  The fastlane files follow this pattern:
 766     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 767
 768     """
 769
 770     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 771     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 772     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 773
 774     for srcd in sorted(sourcedirs):
 775         if not os.path.isdir(srcd):
 776             continue
 777         for root, dirs, files in os.walk(srcd):
 778             segments = root.split('/')
 779             packageName = segments[1]
 780             if packageName not in apps:
 781                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 782                 continue
 783             locale = segments[-1]
 784             destdir = os.path.join('repo', packageName, locale)
 785             for f in files:
 786                 if f in ('description.txt', 'full_description.txt'):
 787                     _set_localized_text_entry(apps[packageName], locale, 'description',
 788                                               os.path.join(root, f))
 789                     continue
 790                 elif f in ('summary.txt', 'short_description.txt'):
 791                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 792                                               os.path.join(root, f))
 793                     continue
 794                 elif f in ('name.txt', 'title.txt'):
 795                     _set_localized_text_entry(apps[packageName], locale, 'name',
 796                                               os.path.join(root, f))
 797                     continue
 798                 elif f == 'video.txt':
 799                     _set_localized_text_entry(apps[packageName], locale, 'video',
 800                                               os.path.join(root, f))
 801                     continue
 802                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 803                     locale = segments[-2]
 804                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 805                                               os.path.join(root, f))
 806                     continue
 807
 808                 base, extension = common.get_extension(f)
 809                 if locale == 'images':
 810                     locale = segments[-2]
 811                     destdir = os.path.join('repo', packageName, locale)
 812                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 813                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 814                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 815                     shutil.copy(os.path.join(root, f), destdir)
 816             for d in dirs:
 817                 if d in SCREENSHOT_DIRS:
 818                     for f in glob.glob(os.path.join(root, d, '*.*')):
 819                         _, extension = common.get_extension(f)
 820                         if extension in ALLOWED_EXTENSIONS:
 821                             screenshotdestdir = os.path.join(destdir, d)
 822                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 823                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 824                             shutil.copy(f, screenshotdestdir)
 825
 826     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 827     for d in repofiles:
 828         if not os.path.isdir(d):
 829             continue
 830         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 831             if not os.path.isfile(f):
 832                 continue
 833             segments = f.split('/')
 834             packageName = segments[1]
 835             locale = segments[2]
 836             screenshotdir = segments[3]
 837             filename = os.path.basename(f)
 838             base, extension = common.get_extension(filename)
 839
 840             if packageName not in apps:
 841                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 842                                 % (filename, packageName))
 843                 continue
 844             graphics = _get_localized_dict(apps[packageName], locale)
 845
 846             if extension not in ALLOWED_EXTENSIONS:
 847                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 848             elif base in GRAPHIC_NAMES:
 849                 # there can only be zero or one of these per locale
 850                 graphics[base] = filename
 851             elif screenshotdir in SCREENSHOT_DIRS:
 852                 # there can any number of these per locale
 853                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 854                 if screenshotdir not in graphics:
 855                     graphics[screenshotdir] = []
 856                 graphics[screenshotdir].append(filename)
 857             else:
 858                 logging.warning('Unsupported graphics file found: ' + f)
 859
 860
 861 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 862     """Scan a repo for all files with an extension except APK/OBB
 863
 864     :param apkcache: current cached info about all repo files
 865     :param repodir: repo directory to scan
 866     :param knownapks: list of all known files, as per metadata.read_metadata
 867     :param use_date_from_file: use date from file (instead of current date)
 868                                for newly added files
 869     """
 870
 871     cachechanged = False
 872     repo_files = []
 873     repodir = repodir.encode('utf-8')
 874     for name in os.listdir(repodir):
 875         file_extension = common.get_file_extension(name)
 876         if file_extension == 'apk' or file_extension == 'obb':
 877             continue
 878         filename = os.path.join(repodir, name)
 879         name_utf8 = name.decode('utf-8')
 880         if filename.endswith(b'_src.tar.gz'):
 881             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 882             continue
 883         if not common.is_repo_file(filename):
 884             continue
 885         stat = os.stat(filename)
 886         if stat.st_size == 0:
 887             raise FDroidException(filename + ' is zero size!')
 888
 889         shasum = sha256sum(filename)
 890         usecache = False
 891         if name in apkcache:
 892             repo_file = apkcache[name]
 893             # added time is cached as tuple but used here as datetime instance
 894             if 'added' in repo_file:
 895                 a = repo_file['added']
 896                 if isinstance(a, datetime):
 897                     repo_file['added'] = a
 898                 else:
 899                     repo_file['added'] = datetime(*a[:6])
 900             if repo_file.get('hash') == shasum:
 901                 logging.debug("Reading " + name_utf8 + " from cache")
 902                 usecache = True
 903             else:
 904                 logging.debug("Ignoring stale cache data for " + name)
 905
 906         if not usecache:
 907             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 908             repo_file = collections.OrderedDict()
 909             repo_file['name'] = os.path.splitext(name_utf8)[0]
 910             # TODO rename apkname globally to something more generic
 911             repo_file['apkName'] = name_utf8
 912             repo_file['hash'] = shasum
 913             repo_file['hashType'] = 'sha256'
 914             repo_file['versionCode'] = 0
 915             repo_file['versionName'] = shasum
 916             # the static ID is the SHA256 unless it is set in the metadata
 917             repo_file['packageName'] = shasum
 918
 919             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 920             if m:
 921                 repo_file['packageName'] = m.group(1)
 922                 repo_file['versionCode'] = int(m.group(2))
 923             srcfilename = name + b'_src.tar.gz'
 924             if os.path.exists(os.path.join(repodir, srcfilename)):
 925                 repo_file['srcname'] = srcfilename.decode('utf-8')
 926             repo_file['size'] = stat.st_size
 927
 928             apkcache[name] = repo_file
 929             cachechanged = True
 930
 931         if use_date_from_file:
 932             timestamp = stat.st_ctime
 933             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 934         else:
 935             default_date_param = None
 936
 937         # Record in knownapks, getting the added date at the same time..
 938         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 939                                     default_date=default_date_param)
 940         if added:
 941             repo_file['added'] = added
 942
 943         repo_files.append(repo_file)
 944
 945     return repo_files, cachechanged
 946
 947
 948 def scan_apk(apk_file):
 949     """
 950     Scans an APK file and returns dictionary with metadata of the APK.
 951
 952     Attention: This does *not* verify that the APK signature is correct.
 953
 954     :param apk_file: The (ideally absolute) path to the APK file
 955     :raises BuildException
 956     :return A dict containing APK metadata
 957     """
 958     apk = {
 959         'hash': sha256sum(apk_file),
 960         'hashType': 'sha256',
 961         'uses-permission': [],
 962         'uses-permission-sdk-23': [],
 963         'features': [],
 964         'icons_src': {},
 965         'icons': {},
 966         'antiFeatures': set(),
 967     }
 968
 969     if SdkToolsPopen(['aapt', 'version'], output=False):
 970         scan_apk_aapt(apk, apk_file)
 971     else:
 972         scan_apk_androguard(apk, apk_file)
 973
 974     # Get the signature
 975     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 976     apk['sig'] = getsig(apk_file)
 977     if not apk['sig']:
 978         raise BuildException("Failed to get apk signature")
 979
 980     # Get size of the APK
 981     apk['size'] = os.path.getsize(apk_file)
 982
 983     if 'minSdkVersion' not in apk:
 984         logging.warning("No SDK version information found in {0}".format(apk_file))
 985         apk['minSdkVersion'] = 1
 986
 987     # Check for known vulnerabilities
 988     if has_known_vulnerability(apk_file):
 989         apk['antiFeatures'].add('KnownVuln')
 990
 991     return apk
 992
 993
 994 def scan_apk_aapt(apk, apkfile):
 995     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 996     if p.returncode != 0:
 997         if options.delete_unknown:
 998             if os.path.exists(apkfile):
 999                 logging.error("Failed to get apk information, deleting " + apkfile)
1000                 os.remove(apkfile)
1001             else:
1002                 logging.error("Could not find {0} to remove it".format(apkfile))
1003         else:
1004             logging.error("Failed to get apk information, skipping " + apkfile)
1005         raise BuildException("Invalid APK")
1006     for line in p.output.splitlines():
1007         if line.startswith("package:"):
1008             try:
1009                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1010                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1011                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1012             except Exception as e:
1013                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1014         elif line.startswith("application:"):
1015             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1016             # Keep path to non-dpi icon in case we need it
1017             match = re.match(APK_ICON_PAT_NODPI, line)
1018             if match:
1019                 apk['icons_src']['-1'] = match.group(1)
1020         elif line.startswith("launchable-activity:"):
1021             # Only use launchable-activity as fallback to application
1022             if not apk['name']:
1023                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1024             if '-1' not in apk['icons_src']:
1025                 match = re.match(APK_ICON_PAT_NODPI, line)
1026                 if match:
1027                     apk['icons_src']['-1'] = match.group(1)
1028         elif line.startswith("application-icon-"):
1029             match = re.match(APK_ICON_PAT, line)
1030             if match:
1031                 density = match.group(1)
1032                 path = match.group(2)
1033                 apk['icons_src'][density] = path
1034         elif line.startswith("sdkVersion:"):
1035             m = re.match(APK_SDK_VERSION_PAT, line)
1036             if m is None:
1037                 logging.error(line.replace('sdkVersion:', '')
1038                               + ' is not a valid minSdkVersion!')
1039             else:
1040                 apk['minSdkVersion'] = m.group(1)
1041                 # if target not set, default to min
1042                 if 'targetSdkVersion' not in apk:
1043                     apk['targetSdkVersion'] = m.group(1)
1044         elif line.startswith("targetSdkVersion:"):
1045             m = re.match(APK_SDK_VERSION_PAT, line)
1046             if m is None:
1047                 logging.error(line.replace('targetSdkVersion:', '')
1048                               + ' is not a valid targetSdkVersion!')
1049             else:
1050                 apk['targetSdkVersion'] = m.group(1)
1051         elif line.startswith("maxSdkVersion:"):
1052             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1053         elif line.startswith("native-code:"):
1054             apk['nativecode'] = []
1055             for arch in line[13:].split(' '):
1056                 apk['nativecode'].append(arch[1:-1])
1057         elif line.startswith('uses-permission:'):
1058             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1059             if perm_match['maxSdkVersion']:
1060                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1061             permission = UsesPermission(
1062                 perm_match['name'],
1063                 perm_match['maxSdkVersion']
1064             )
1065
1066             apk['uses-permission'].append(permission)
1067         elif line.startswith('uses-permission-sdk-23:'):
1068             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1069             if perm_match['maxSdkVersion']:
1070                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1071             permission_sdk_23 = UsesPermissionSdk23(
1072                 perm_match['name'],
1073                 perm_match['maxSdkVersion']
1074             )
1075
1076             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1077
1078         elif line.startswith('uses-feature:'):
1079             feature = re.match(APK_FEATURE_PAT, line).group(1)
1080             # Filter out this, it's only added with the latest SDK tools and
1081             # causes problems for lots of apps.
1082             if feature != "android.hardware.screen.portrait" \
1083                     and feature != "android.hardware.screen.landscape":
1084                 if feature.startswith("android.feature."):
1085                     feature = feature[16:]
1086                 apk['features'].add(feature)
1087
1088
1089 def scan_apk_androguard(apk, apkfile):
1090     try:
1091         from androguard.core.bytecodes.apk import APK
1092         apkobject = APK(apkfile)
1093         if apkobject.is_valid_APK():
1094             arsc = apkobject.get_android_resources()
1095         else:
1096             if options.delete_unknown:
1097                 if os.path.exists(apkfile):
1098                     logging.error("Failed to get apk information, deleting " + apkfile)
1099                     os.remove(apkfile)
1100                 else:
1101                     logging.error("Could not find {0} to remove it".format(apkfile))
1102             else:
1103                 logging.error("Failed to get apk information, skipping " + apkfile)
1104             raise BuildException("Invaild APK")
1105     except ImportError:
1106         raise FDroidException("androguard library is not installed and aapt not present")
1107     except FileNotFoundError:
1108         logging.error("Could not open apk file for analysis")
1109         raise BuildException("Invalid APK")
1110
1111     apk['packageName'] = apkobject.get_package()
1112     apk['versionCode'] = int(apkobject.get_androidversion_code())
1113     apk['versionName'] = apkobject.get_androidversion_name()
1114     if apk['versionName'][0] == "@":
1115         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1116         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1117         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1118     apk['name'] = apkobject.get_app_name()
1119
1120     if apkobject.get_max_sdk_version() is not None:
1121         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1122     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1123     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1124
1125     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1126     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1127
1128     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1129
1130     for file in apkobject.get_files():
1131         d_re = density_re.match(file)
1132         if d_re:
1133             folder = d_re.group(1).split('-')
1134             if len(folder) > 1:
1135                 resolution = folder[1]
1136             else:
1137                 resolution = 'mdpi'
1138             density = screen_resolutions[resolution]
1139             apk['icons_src'][density] = d_re.group(0)
1140
1141     if apk['icons_src'].get('-1') is None:
1142         apk['icons_src']['-1'] = apk['icons_src']['160']
1143
1144     arch_re = re.compile("^lib/(.*)/.*$")
1145     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1146     if len(arch) >= 1:
1147         apk['nativecode'] = []
1148         apk['nativecode'].extend(sorted(list(arch)))
1149
1150     xml = apkobject.get_android_manifest_xml()
1151
1152     for item in xml.getElementsByTagName('uses-permission'):
1153         name = str(item.getAttribute("android:name"))
1154         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1155         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1156         permission = UsesPermission(
1157             name,
1158             maxSdkVersion
1159         )
1160         apk['uses-permission'].append(permission)
1161
1162     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1163         name = str(item.getAttribute("android:name"))
1164         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1165         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1166         permission_sdk_23 = UsesPermissionSdk23(
1167             name,
1168             maxSdkVersion
1169         )
1170         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1171
1172     for item in xml.getElementsByTagName('uses-feature'):
1173         feature = str(item.getAttribute("android:name"))
1174         if feature != "android.hardware.screen.portrait" \
1175                 and feature != "android.hardware.screen.landscape":
1176             if feature.startswith("android.feature."):
1177                 feature = feature[16:]
1178         apk['features'].append(feature)
1179
1180
1181 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1182                 allow_disabled_algorithms=False, archive_bad_sig=False):
1183     """Processes the apk with the given filename in the given repo directory.
1184
1185     This also extracts the icons.
1186
1187     :param apkcache: current apk cache information
1188     :param apkfilename: the filename of the apk to scan
1189     :param repodir: repo directory to scan
1190     :param knownapks: known apks info
1191     :param use_date_from_apk: use date from APK (instead of current date)
1192                               for newly added APKs
1193     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1194                                       disabled algorithms in the signature (e.g. MD5)
1195     :param archive_bad_sig: move APKs with a bad signature to the archive
1196     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1197      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1198     """
1199
1200     apk = {}
1201     apkfile = os.path.join(repodir, apkfilename)
1202
1203     cachechanged = False
1204     usecache = False
1205     if apkfilename in apkcache:
1206         apk = apkcache[apkfilename]
1207         if apk.get('hash') == sha256sum(apkfile):
1208             logging.debug("Reading " + apkfilename + " from cache")
1209             usecache = True
1210         else:
1211             logging.debug("Ignoring stale cache data for " + apkfilename)
1212
1213     if not usecache:
1214         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1215
1216         try:
1217             apk = scan_apk(apkfile)
1218         except BuildException:
1219             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1220                             .format(apkfilename=apkfilename))
1221             return True, None, False
1222
1223         # Check for debuggable apks...
1224         if common.isApkAndDebuggable(apkfile):
1225             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1226
1227         if options.rename_apks:
1228             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1229             std_short_name = os.path.join(repodir, n)
1230             if apkfile != std_short_name:
1231                 if os.path.exists(std_short_name):
1232                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1233                     if apkfile != std_long_name:
1234                         if os.path.exists(std_long_name):
1235                             dupdir = os.path.join('duplicates', repodir)
1236                             if not os.path.isdir(dupdir):
1237                                 os.makedirs(dupdir, exist_ok=True)
1238                             dupfile = os.path.join('duplicates', std_long_name)
1239                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1240                             os.rename(apkfile, dupfile)
1241                             return True, None, False
1242                         else:
1243                             os.rename(apkfile, std_long_name)
1244                     apkfile = std_long_name
1245                 else:
1246                     os.rename(apkfile, std_short_name)
1247                     apkfile = std_short_name
1248                 apkfilename = apkfile[len(repodir) + 1:]
1249
1250         apk['apkName'] = apkfilename
1251         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1252         if os.path.exists(os.path.join(repodir, srcfilename)):
1253             apk['srcname'] = srcfilename
1254
1255         # verify the jar signature is correct, allow deprecated
1256         # algorithms only if the APK is in the archive.
1257         skipapk = False
1258         if not common.verify_apk_signature(apkfile):
1259             if repodir == 'archive' or allow_disabled_algorithms:
1260                 if common.verify_old_apk_signature(apkfile):
1261                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1262                 else:
1263                     skipapk = True
1264             else:
1265                 skipapk = True
1266
1267         if skipapk:
1268             if archive_bad_sig:
1269                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1270                 move_apk_between_sections(repodir, 'archive', apk)
1271             else:
1272                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1273             return True, None, False
1274
1275         apkzip = zipfile.ZipFile(apkfile, 'r')
1276
1277         # if an APK has files newer than the system time, suggest updating
1278         # the system clock.  This is useful for offline systems, used for
1279         # signing, which do not have another source of clock sync info. It
1280         # has to be more than 24 hours newer because ZIP/APK files do not
1281         # store timezone info
1282         manifest = apkzip.getinfo('AndroidManifest.xml')
1283         if manifest.date_time[1] == 0:  # month can't be zero
1284             logging.debug('AndroidManifest.xml has no date')
1285         else:
1286             dt_obj = datetime(*manifest.date_time)
1287             checkdt = dt_obj - timedelta(1)
1288             if datetime.today() < checkdt:
1289                 logging.warning('System clock is older than manifest in: '
1290                                 + apkfilename
1291                                 + '\nSet clock to that time using:\n'
1292                                 + 'sudo date -s "' + str(dt_obj) + '"')
1293
1294         # extract icons from APK zip file
1295         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1296         try:
1297             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1298         finally:
1299             apkzip.close()  # ensure that APK zip file gets closed
1300
1301         # resize existing icons for densities missing in the APK
1302         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1303
1304         if use_date_from_apk and manifest.date_time[1] != 0:
1305             default_date_param = datetime(*manifest.date_time)
1306         else:
1307             default_date_param = None
1308
1309         # Record in known apks, getting the added date at the same time..
1310         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1311                                     default_date=default_date_param)
1312         if added:
1313             apk['added'] = added
1314
1315         apkcache[apkfilename] = apk
1316         cachechanged = True
1317
1318     return False, apk, cachechanged
1319
1320
1321 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1322     """Processes the apks in the given repo directory.
1323
1324     This also extracts the icons.
1325
1326     :param apkcache: current apk cache information
1327     :param repodir: repo directory to scan
1328     :param knownapks: known apks info
1329     :param use_date_from_apk: use date from APK (instead of current date)
1330                               for newly added APKs
1331     :returns: (apks, cachechanged) where apks is a list of apk information,
1332               and cachechanged is True if the apkcache got changed.
1333     """
1334
1335     cachechanged = False
1336
1337     for icon_dir in get_all_icon_dirs(repodir):
1338         if os.path.exists(icon_dir):
1339             if options.clean:
1340                 shutil.rmtree(icon_dir)
1341                 os.makedirs(icon_dir)
1342         else:
1343             os.makedirs(icon_dir)
1344
1345     apks = []
1346     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1347         apkfilename = apkfile[len(repodir) + 1:]
1348         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1349         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1350                                              use_date_from_apk, ada, True)
1351         if skip:
1352             continue
1353         apks.append(apk)
1354         cachechanged = cachechanged or cachethis
1355
1356     return apks, cachechanged
1357
1358
1359 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1360     """
1361     Extracts icons from the given APK zip in various densities,
1362     saves them into given repo directory
1363     and stores their names in the APK metadata dictionary.
1364
1365     :param icon_filename: A string representing the icon's file name
1366     :param apk: A populated dictionary containing APK metadata.
1367                 Needs to have 'icons_src' key
1368     :param apkzip: An opened zipfile.ZipFile of the APK file
1369     :param repo_dir: The directory of the APK's repository
1370     :return: A list of icon densities that are missing
1371     """
1372     empty_densities = []
1373     for density in screen_densities:
1374         if density not in apk['icons_src']:
1375             empty_densities.append(density)
1376             continue
1377         icon_src = apk['icons_src'][density]
1378         icon_dir = get_icon_dir(repo_dir, density)
1379         icon_dest = os.path.join(icon_dir, icon_filename)
1380
1381         # Extract the icon files per density
1382         if icon_src.endswith('.xml'):
1383             png = os.path.basename(icon_src)[:-4] + '.png'
1384             for f in apkzip.namelist():
1385                 if f.endswith(png):
1386                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1387                     if m and screen_resolutions[m.group(2)] == density:
1388                         icon_src = f
1389             if icon_src.endswith('.xml'):
1390                 empty_densities.append(density)
1391                 continue
1392         try:
1393             with open(icon_dest, 'wb') as f:
1394                 f.write(get_icon_bytes(apkzip, icon_src))
1395             apk['icons'][density] = icon_filename
1396         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1397             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1398             del apk['icons_src'][density]
1399             empty_densities.append(density)
1400
1401     if '-1' in apk['icons_src']:
1402         icon_src = apk['icons_src']['-1']
1403         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1404         with open(icon_path, 'wb') as f:
1405             f.write(get_icon_bytes(apkzip, icon_src))
1406         try:
1407             im = Image.open(icon_path)
1408             dpi = px_to_dpi(im.size[0])
1409             for density in screen_densities:
1410                 if density in apk['icons']:
1411                     break
1412                 if density == screen_densities[-1] or dpi >= int(density):
1413                     apk['icons'][density] = icon_filename
1414                     shutil.move(icon_path,
1415                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1416                     empty_densities.remove(density)
1417                     break
1418         except Exception as e:
1419             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1420
1421     if apk['icons']:
1422         apk['icon'] = icon_filename
1423
1424     return empty_densities
1425
1426
1427 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1428     """
1429     Resize existing icons for densities missing in the APK to ensure all densities are available
1430
1431     :param empty_densities: A list of icon densities that are missing
1432     :param icon_filename: A string representing the icon's file name
1433     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1434     :param repo_dir: The directory of the APK's repository
1435     """
1436     # First try resizing down to not lose quality
1437     last_density = None
1438     for density in screen_densities:
1439         if density not in empty_densities:
1440             last_density = density
1441             continue
1442         if last_density is None:
1443             continue
1444         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1445
1446         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1447         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1448         fp = None
1449         try:
1450             fp = open(last_icon_path, 'rb')
1451             im = Image.open(fp)
1452
1453             size = dpi_to_px(density)
1454
1455             im.thumbnail((size, size), Image.ANTIALIAS)
1456             im.save(icon_path, "PNG")
1457             empty_densities.remove(density)
1458         except Exception as e:
1459             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1460         finally:
1461             if fp:
1462                 fp.close()
1463
1464     # Then just copy from the highest resolution available
1465     last_density = None
1466     for density in reversed(screen_densities):
1467         if density not in empty_densities:
1468             last_density = density
1469             continue
1470
1471         if last_density is None:
1472             continue
1473
1474         shutil.copyfile(
1475             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1476             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1477         )
1478         empty_densities.remove(density)
1479
1480     for density in screen_densities:
1481         icon_dir = get_icon_dir(repo_dir, density)
1482         icon_dest = os.path.join(icon_dir, icon_filename)
1483         resize_icon(icon_dest, density)
1484
1485     # Copy from icons-mdpi to icons since mdpi is the baseline density
1486     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1487     if os.path.isfile(baseline):
1488         apk['icons']['0'] = icon_filename
1489         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1490
1491
1492 def apply_info_from_latest_apk(apps, apks):
1493     """
1494     Some information from the apks needs to be applied up to the application level.
1495     When doing this, we use the info from the most recent version's apk.
1496     We deal with figuring out when the app was added and last updated at the same time.
1497     """
1498     for appid, app in apps.items():
1499         bestver = UNSET_VERSION_CODE
1500         for apk in apks:
1501             if apk['packageName'] == appid:
1502                 if apk['versionCode'] > bestver:
1503                     bestver = apk['versionCode']
1504                     bestapk = apk
1505
1506                 if 'added' in apk:
1507                     if not app.added or apk['added'] < app.added:
1508                         app.added = apk['added']
1509                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1510                         app.lastUpdated = apk['added']
1511
1512         if not app.added:
1513             logging.debug("Don't know when " + appid + " was added")
1514         if not app.lastUpdated:
1515             logging.debug("Don't know when " + appid + " was last updated")
1516
1517         if bestver == UNSET_VERSION_CODE:
1518
1519             if app.Name is None:
1520                 app.Name = app.AutoName or appid
1521             app.icon = None
1522             logging.debug("Application " + appid + " has no packages")
1523         else:
1524             if app.Name is None:
1525                 app.Name = bestapk['name']
1526             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1527             if app.CurrentVersionCode is None:
1528                 app.CurrentVersionCode = str(bestver)
1529
1530
1531 def make_categories_txt(repodir, categories):
1532     '''Write a category list in the repo to allow quick access'''
1533     catdata = ''
1534     for cat in sorted(categories):
1535         catdata += cat + '\n'
1536     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1537         f.write(catdata)
1538
1539
1540 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1541
1542     def filter_apk_list_sorted(apk_list):
1543         res = []
1544         for apk in apk_list:
1545             if apk['packageName'] == appid:
1546                 res.append(apk)
1547
1548         # Sort the apk list by version code. First is highest/newest.
1549         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1550
1551     for appid, app in apps.items():
1552
1553         if app.ArchivePolicy:
1554             keepversions = int(app.ArchivePolicy[:-9])
1555         else:
1556             keepversions = defaultkeepversions
1557
1558         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1559                       .format(appid, len(apks), keepversions, len(archapks)))
1560
1561         current_app_apks = filter_apk_list_sorted(apks)
1562         if len(current_app_apks) > keepversions:
1563             # Move back the ones we don't want.
1564             for apk in current_app_apks[keepversions:]:
1565                 move_apk_between_sections(repodir, archivedir, apk)
1566                 archapks.append(apk)
1567                 apks.remove(apk)
1568
1569         current_app_archapks = filter_apk_list_sorted(archapks)
1570         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1571             kept = 0
1572             # Move forward the ones we want again, except DisableAlgorithm
1573             for apk in current_app_archapks:
1574                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1575                     move_apk_between_sections(archivedir, repodir, apk)
1576                     archapks.remove(apk)
1577                     apks.append(apk)
1578                     kept += 1
1579                 if kept == keepversions:
1580                     break
1581
1582
1583 def move_apk_between_sections(from_dir, to_dir, apk):
1584     """move an APK from repo to archive or vice versa"""
1585
1586     def _move_file(from_dir, to_dir, filename, ignore_missing):
1587         from_path = os.path.join(from_dir, filename)
1588         if ignore_missing and not os.path.exists(from_path):
1589             return
1590         to_path = os.path.join(to_dir, filename)
1591         if not os.path.exists(to_dir):
1592             os.mkdir(to_dir)
1593         shutil.move(from_path, to_path)
1594
1595     if from_dir == to_dir:
1596         return
1597
1598     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1599     _move_file(from_dir, to_dir, apk['apkName'], False)
1600     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1601     for density in all_screen_densities:
1602         from_icon_dir = get_icon_dir(from_dir, density)
1603         to_icon_dir = get_icon_dir(to_dir, density)
1604         if density not in apk['icons']:
1605             continue
1606         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1607     if 'srcname' in apk:
1608         _move_file(from_dir, to_dir, apk['srcname'], False)
1609
1610
1611 def add_apks_to_per_app_repos(repodir, apks):
1612     apks_per_app = dict()
1613     for apk in apks:
1614         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1615         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1616         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1617         apks_per_app[apk['packageName']] = apk
1618
1619         if not os.path.exists(apk['per_app_icons']):
1620             logging.info('Adding new repo for only ' + apk['packageName'])
1621             os.makedirs(apk['per_app_icons'])
1622
1623         apkpath = os.path.join(repodir, apk['apkName'])
1624         shutil.copy(apkpath, apk['per_app_repo'])
1625         apksigpath = apkpath + '.sig'
1626         if os.path.exists(apksigpath):
1627             shutil.copy(apksigpath, apk['per_app_repo'])
1628         apkascpath = apkpath + '.asc'
1629         if os.path.exists(apkascpath):
1630             shutil.copy(apkascpath, apk['per_app_repo'])
1631
1632
1633 def create_metadata_from_template(apk):
1634     '''create a new metadata file using internal or external template
1635
1636     Generate warnings for apk's with no metadata (or create skeleton
1637     metadata files, if requested on the command line).  Though the
1638     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1639     since those impose things on the metadata file made from the
1640     template: field sort order, empty field value, formatting, etc.
1641     '''
1642
1643     import yaml
1644     if os.path.exists('template.yml'):
1645         with open('template.yml') as f:
1646             metatxt = f.read()
1647         if 'name' in apk and apk['name'] != '':
1648             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1649                              r'\1 ' + apk['name'],
1650                              metatxt,
1651                              flags=re.IGNORECASE | re.MULTILINE)
1652         else:
1653             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1654             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1655                              r'\1 ' + apk['packageName'],
1656                              metatxt,
1657                              flags=re.IGNORECASE | re.MULTILINE)
1658         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1659             f.write(metatxt)
1660     else:
1661         app = dict()
1662         app['Categories'] = [os.path.basename(os.getcwd())]
1663         # include some blanks as part of the template
1664         app['AuthorName'] = ''
1665         app['Summary'] = ''
1666         app['WebSite'] = ''
1667         app['IssueTracker'] = ''
1668         app['SourceCode'] = ''
1669         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1670         if 'name' in apk and apk['name'] != '':
1671             app['Name'] = apk['name']
1672         else:
1673             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1674             app['Name'] = apk['packageName']
1675         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1676             yaml.dump(app, f, default_flow_style=False)
1677     logging.info("Generated skeleton metadata for " + apk['packageName'])
1678
1679
1680 config = None
1681 options = None
1682
1683
1684 def main():
1685
1686     global config, options
1687
1688     # Parse command line...
1689     parser = ArgumentParser()
1690     common.setup_global_opts(parser)
1691     parser.add_argument("--create-key", action="store_true", default=False,
1692                         help=_("Create a repo signing key in a keystore"))
1693     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1694                         help=_("Create skeleton metadata files that are missing"))
1695     parser.add_argument("--delete-unknown", action="store_true", default=False,
1696                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1697     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1698                         help=_("Report on build data status"))
1699     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1700                         help=_("Interactively ask about things that need updating."))
1701     parser.add_argument("-I", "--icons", action="store_true", default=False,
1702                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1703     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1704                         help=_("Specify editor to use in interactive mode. Default ") +
1705                         "is /etc/alternatives/editor")
1706     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1707                         help=_("Update the wiki"))
1708     parser.add_argument("--pretty", action="store_true", default=False,
1709                         help=_("Produce human-readable index.xml"))
1710     parser.add_argument("--clean", action="store_true", default=False,
1711                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1712     parser.add_argument("--nosign", action="store_true", default=False,
1713                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1714     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1715                         help=_("Use date from APK instead of current time for newly added APKs"))
1716     parser.add_argument("--rename-apks", action="store_true", default=False,
1717                         help=_("Rename APK files that do not match package.name_123.apk"))
1718     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1719                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1720     metadata.add_metadata_arguments(parser)
1721     options = parser.parse_args()
1722     metadata.warnings_action = options.W
1723
1724     config = common.read_config(options)
1725
1726     if not ('jarsigner' in config and 'keytool' in config):
1727         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1728
1729     repodirs = ['repo']
1730     if config['archive_older'] != 0:
1731         repodirs.append('archive')
1732         if not os.path.exists('archive'):
1733             os.mkdir('archive')
1734
1735     if options.icons:
1736         resize_all_icons(repodirs)
1737         sys.exit(0)
1738
1739     if options.rename_apks:
1740         options.clean = True
1741
1742     # check that icons exist now, rather than fail at the end of `fdroid update`
1743     for k in ['repo_icon', 'archive_icon']:
1744         if k in config:
1745             if not os.path.exists(config[k]):
1746                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1747                 sys.exit(1)
1748
1749     # if the user asks to create a keystore, do it now, reusing whatever it can
1750     if options.create_key:
1751         if os.path.exists(config['keystore']):
1752             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1753             logging.critical("\t'" + config['keystore'] + "'")
1754             sys.exit(1)
1755
1756         if 'repo_keyalias' not in config:
1757             config['repo_keyalias'] = socket.getfqdn()
1758             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1759         if 'keydname' not in config:
1760             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1761             common.write_to_config(config, 'keydname', config['keydname'])
1762         if 'keystore' not in config:
1763             config['keystore'] = common.default_config['keystore']
1764             common.write_to_config(config, 'keystore', config['keystore'])
1765
1766         password = common.genpassword()
1767         if 'keystorepass' not in config:
1768             config['keystorepass'] = password
1769             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1770         if 'keypass' not in config:
1771             config['keypass'] = password
1772             common.write_to_config(config, 'keypass', config['keypass'])
1773         common.genkeystore(config)
1774
1775     # Get all apps...
1776     apps = metadata.read_metadata()
1777
1778     # Generate a list of categories...
1779     categories = set()
1780     for app in apps.values():
1781         categories.update(app.Categories)
1782
1783     # Read known apks data (will be updated and written back when we've finished)
1784     knownapks = common.KnownApks()
1785
1786     # Get APK cache
1787     apkcache = get_cache()
1788
1789     # Delete builds for disabled apps
1790     delete_disabled_builds(apps, apkcache, repodirs)
1791
1792     # Scan all apks in the main repo
1793     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1794
1795     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1796                                            options.use_date_from_apk)
1797     cachechanged = cachechanged or fcachechanged
1798     apks += files
1799     for apk in apks:
1800         if apk['packageName'] not in apps:
1801             if options.create_metadata:
1802                 create_metadata_from_template(apk)
1803                 apps = metadata.read_metadata()
1804             else:
1805                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1806                 if options.delete_unknown:
1807                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1808                     rmf = os.path.join(repodirs[0], apk['apkName'])
1809                     if not os.path.exists(rmf):
1810                         logging.error("Could not find {0} to remove it".format(rmf))
1811                     else:
1812                         os.remove(rmf)
1813                 else:
1814                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1815
1816     copy_triple_t_store_metadata(apps)
1817     insert_obbs(repodirs[0], apps, apks)
1818     insert_localized_app_metadata(apps)
1819     translate_per_build_anti_features(apps, apks)
1820
1821     # Scan the archive repo for apks as well
1822     if len(repodirs) > 1:
1823         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1824         if cc:
1825             cachechanged = True
1826     else:
1827         archapks = []
1828
1829     # Apply information from latest apks to the application and update dates
1830     apply_info_from_latest_apk(apps, apks + archapks)
1831
1832     # Sort the app list by name, then the web site doesn't have to by default.
1833     # (we had to wait until we'd scanned the apks to do this, because mostly the
1834     # name comes from there!)
1835     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1836
1837     # APKs are placed into multiple repos based on the app package, providing
1838     # per-app subscription feeds for nightly builds and things like it
1839     if config['per_app_repos']:
1840         add_apks_to_per_app_repos(repodirs[0], apks)
1841         for appid, app in apps.items():
1842             repodir = os.path.join(appid, 'fdroid', 'repo')
1843             appdict = dict()
1844             appdict[appid] = app
1845             if os.path.isdir(repodir):
1846                 index.make(appdict, [appid], apks, repodir, False)
1847             else:
1848                 logging.info('Skipping index generation for ' + appid)
1849         return
1850
1851     if len(repodirs) > 1:
1852         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1853
1854     # Make the index for the main repo...
1855     index.make(apps, sortedids, apks, repodirs[0], False)
1856     make_categories_txt(repodirs[0], categories)
1857
1858     # If there's an archive repo,  make the index for it. We already scanned it
1859     # earlier on.
1860     if len(repodirs) > 1:
1861         index.make(apps, sortedids, archapks, repodirs[1], True)
1862
1863     git_remote = config.get('binary_transparency_remote')
1864     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1865         from . import btlog
1866         btlog.make_binary_transparency_log(repodirs)
1867
1868     if config['update_stats']:
1869         # Update known apks info...
1870         knownapks.writeifchanged()
1871
1872         # Generate latest apps data for widget
1873         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1874             data = ''
1875             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1876                 for line in f:
1877                     appid = line.rstrip()
1878                     data += appid + "\t"
1879                     app = apps[appid]
1880                     data += app.Name + "\t"
1881                     if app.icon is not None:
1882                         data += app.icon + "\t"
1883                     data += app.License + "\n"
1884             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1885                 f.write(data)
1886
1887     if cachechanged:
1888         write_cache(apkcache)
1889
1890     # Update the wiki...
1891     if options.wiki:
1892         update_wiki(apps, sortedids, apks + archapks)
1893
1894     logging.info(_("Finished"))
1895
1896
1897 if __name__ == "__main__":
1898     main()