fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 from datetime import datetime, timedelta
  32 from argparse import ArgumentParser
  33
  34 import collections
  35 from binascii import hexlify
  36
  37 from PIL import Image
  38 import logging
  39
  40 from . import _
  41 from . import common
  42 from . import index
  43 from . import metadata
  44 from .common import SdkToolsPopen
  45 from .exception import BuildException, FDroidException
  46
  47 METADATA_VERSION = 19
  48
  49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  50 UNSET_VERSION_CODE = -0x100000000
  51
  52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  59 APK_PERMISSION_PAT = \
  60     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  62
  63 screen_densities = ['640', '480', '320', '240', '160', '120']
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "undefined": '-1',
  72     "anydpi": '65534',
  73     "nodpi": '65535'
  74 }
  75
  76 all_screen_densities = ['0'] + screen_densities
  77
  78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  80
  81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  84                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  85
  86
  87 def dpi_to_px(density):
  88     return (int(density) * 48) / 160
  89
  90
  91 def px_to_dpi(px):
  92     return (int(px) * 160) / 48
  93
  94
  95 def get_icon_dir(repodir, density):
  96     if density == '0':
  97         return os.path.join(repodir, "icons")
  98     return os.path.join(repodir, "icons-%s" % density)
  99
 100
 101 def get_icon_dirs(repodir):
 102     for density in screen_densities:
 103         yield get_icon_dir(repodir, density)
 104
 105
 106 def get_all_icon_dirs(repodir):
 107     for density in all_screen_densities:
 108         yield get_icon_dir(repodir, density)
 109
 110
 111 def update_wiki(apps, sortedids, apks):
 112     """Update the wiki
 113
 114     :param apps: fully populated list of all applications
 115     :param apks: all apks, except...
 116     """
 117     logging.info("Updating wiki")
 118     wikicat = 'Apps'
 119     wikiredircat = 'App Redirects'
 120     import mwclient
 121     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 122                          path=config['wiki_path'])
 123     site.login(config['wiki_user'], config['wiki_password'])
 124     generated_pages = {}
 125     generated_redirects = {}
 126
 127     for appid in sortedids:
 128         app = metadata.App(apps[appid])
 129
 130         wikidata = ''
 131         if app.Disabled:
 132             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 133         if app.AntiFeatures:
 134             for af in app.AntiFeatures:
 135                 wikidata += '{{AntiFeature|' + af + '}}\n'
 136         if app.RequiresRoot:
 137             requiresroot = 'Yes'
 138         else:
 139             requiresroot = 'No'
 140         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 141             appid,
 142             app.Name,
 143             app.added.strftime('%Y-%m-%d') if app.added else '',
 144             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 145             app.SourceCode,
 146             app.IssueTracker,
 147             app.WebSite,
 148             app.Changelog,
 149             app.Donate,
 150             app.FlattrID,
 151             app.Bitcoin,
 152             app.Litecoin,
 153             app.License,
 154             requiresroot,
 155             app.AuthorName,
 156             app.AuthorEmail)
 157
 158         if app.Provides:
 159             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 160
 161         wikidata += app.Summary
 162         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 163
 164         wikidata += "=Description=\n"
 165         wikidata += metadata.description_wiki(app.Description) + "\n"
 166
 167         wikidata += "=Maintainer Notes=\n"
 168         if app.MaintainerNotes:
 169             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 170         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 171
 172         # Get a list of all packages for this application...
 173         apklist = []
 174         gotcurrentver = False
 175         cantupdate = False
 176         buildfails = False
 177         for apk in apks:
 178             if apk['packageName'] == appid:
 179                 if str(apk['versionCode']) == app.CurrentVersionCode:
 180                     gotcurrentver = True
 181                 apklist.append(apk)
 182         # Include ones we can't build, as a special case...
 183         for build in app.builds:
 184             if build.disable:
 185                 if build.versionCode == app.CurrentVersionCode:
 186                     cantupdate = True
 187                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 188                 apklist.append({'versionCode': int(build.versionCode),
 189                                 'versionName': build.versionName,
 190                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 191                                 })
 192             else:
 193                 builtit = False
 194                 for apk in apklist:
 195                     if apk['versionCode'] == int(build.versionCode):
 196                         builtit = True
 197                         break
 198                 if not builtit:
 199                     buildfails = True
 200                     apklist.append({'versionCode': int(build.versionCode),
 201                                     'versionName': build.versionName,
 202                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 203                                     })
 204         if app.CurrentVersionCode == '0':
 205             cantupdate = True
 206         # Sort with most recent first...
 207         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 208
 209         wikidata += "=Versions=\n"
 210         if len(apklist) == 0:
 211             wikidata += "We currently have no versions of this app available."
 212         elif not gotcurrentver:
 213             wikidata += "We don't have the current version of this app."
 214         else:
 215             wikidata += "We have the current version of this app."
 216         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 217         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 218         if len(app.NoSourceSince) > 0:
 219             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 220         if len(app.CurrentVersion) > 0:
 221             wikidata += "The current (recommended) version is " + app.CurrentVersion
 222             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 223         validapks = 0
 224         for apk in apklist:
 225             wikidata += "==" + apk['versionName'] + "==\n"
 226
 227             if 'buildproblem' in apk:
 228                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 229             else:
 230                 validapks += 1
 231                 wikidata += "This version is built and signed by "
 232                 if 'srcname' in apk:
 233                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 234                 else:
 235                     wikidata += "the original developer.\n\n"
 236             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 237
 238         wikidata += '\n[[Category:' + wikicat + ']]\n'
 239         if len(app.NoSourceSince) > 0:
 240             wikidata += '\n[[Category:Apps missing source code]]\n'
 241         if validapks == 0 and not app.Disabled:
 242             wikidata += '\n[[Category:Apps with no packages]]\n'
 243         if cantupdate and not app.Disabled:
 244             wikidata += "\n[[Category:Apps we cannot update]]\n"
 245         if buildfails and not app.Disabled:
 246             wikidata += "\n[[Category:Apps with failing builds]]\n"
 247         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 248             wikidata += '\n[[Category:Apps to Update]]\n'
 249         if app.Disabled:
 250             wikidata += '\n[[Category:Apps that are disabled]]\n'
 251         if app.UpdateCheckMode == 'None' and not app.Disabled:
 252             wikidata += '\n[[Category:Apps with no update check]]\n'
 253         for appcat in app.Categories:
 254             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 255
 256         # We can't have underscores in the page name, even if they're in
 257         # the package ID, because MediaWiki messes with them...
 258         pagename = appid.replace('_', ' ')
 259
 260         # Drop a trailing newline, because mediawiki is going to drop it anyway
 261         # and it we don't we'll think the page has changed when it hasn't...
 262         if wikidata.endswith('\n'):
 263             wikidata = wikidata[:-1]
 264
 265         generated_pages[pagename] = wikidata
 266
 267         # Make a redirect from the name to the ID too, unless there's
 268         # already an existing page with the name and it isn't a redirect.
 269         noclobber = False
 270         apppagename = app.Name.replace('_', ' ')
 271         apppagename = apppagename.replace('{', '')
 272         apppagename = apppagename.replace('}', ' ')
 273         apppagename = apppagename.replace(':', ' ')
 274         apppagename = apppagename.replace('[', ' ')
 275         apppagename = apppagename.replace(']', ' ')
 276         # Drop double spaces caused mostly by replacing ':' above
 277         apppagename = apppagename.replace('  ', ' ')
 278         for expagename in site.allpages(prefix=apppagename,
 279                                         filterredir='nonredirects',
 280                                         generator=False):
 281             if expagename == apppagename:
 282                 noclobber = True
 283         # Another reason not to make the redirect page is if the app name
 284         # is the same as it's ID, because that will overwrite the real page
 285         # with an redirect to itself! (Although it seems like an odd
 286         # scenario this happens a lot, e.g. where there is metadata but no
 287         # builds or binaries to extract a name from.
 288         if apppagename == pagename:
 289             noclobber = True
 290         if not noclobber:
 291             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 292
 293     for tcat, genp in [(wikicat, generated_pages),
 294                        (wikiredircat, generated_redirects)]:
 295         catpages = site.Pages['Category:' + tcat]
 296         existingpages = []
 297         for page in catpages:
 298             existingpages.append(page.name)
 299             if page.name in genp:
 300                 pagetxt = page.edit()
 301                 if pagetxt != genp[page.name]:
 302                     logging.debug("Updating modified page " + page.name)
 303                     page.save(genp[page.name], summary='Auto-updated')
 304                 else:
 305                     logging.debug("Page " + page.name + " is unchanged")
 306             else:
 307                 logging.warn("Deleting page " + page.name)
 308                 page.delete('No longer published')
 309         for pagename, text in genp.items():
 310             logging.debug("Checking " + pagename)
 311             if pagename not in existingpages:
 312                 logging.debug("Creating page " + pagename)
 313                 try:
 314                     newpage = site.Pages[pagename]
 315                     newpage.save(text, summary='Auto-created')
 316                 except Exception as e:
 317                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 318
 319     # Purge server cache to ensure counts are up to date
 320     site.pages['Repository Maintenance'].purge()
 321
 322
 323 def delete_disabled_builds(apps, apkcache, repodirs):
 324     """Delete disabled build outputs.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodirs: the repo directories to process
 329     """
 330     for appid, app in apps.items():
 331         for build in app['builds']:
 332             if not build.disable:
 333                 continue
 334             apkfilename = common.get_release_filename(app, build)
 335             iconfilename = "%s.%s.png" % (
 336                 appid,
 337                 build.versionCode)
 338             for repodir in repodirs:
 339                 files = [
 340                     os.path.join(repodir, apkfilename),
 341                     os.path.join(repodir, apkfilename + '.asc'),
 342                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 343                 ]
 344                 for density in all_screen_densities:
 345                     repo_dir = get_icon_dir(repodir, density)
 346                     files.append(os.path.join(repo_dir, iconfilename))
 347
 348                 for f in files:
 349                     if os.path.exists(f):
 350                         logging.info("Deleting disabled build output " + f)
 351                         os.remove(f)
 352             if apkfilename in apkcache:
 353                 del apkcache[apkfilename]
 354
 355
 356 def resize_icon(iconpath, density):
 357
 358     if not os.path.isfile(iconpath):
 359         return
 360
 361     fp = None
 362     try:
 363         fp = open(iconpath, 'rb')
 364         im = Image.open(fp)
 365         size = dpi_to_px(density)
 366
 367         if any(length > size for length in im.size):
 368             oldsize = im.size
 369             im.thumbnail((size, size), Image.ANTIALIAS)
 370             logging.debug("%s was too large at %s - new size is %s" % (
 371                 iconpath, oldsize, im.size))
 372             im.save(iconpath, "PNG")
 373
 374     except Exception as e:
 375         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 376
 377     finally:
 378         if fp:
 379             fp.close()
 380
 381
 382 def resize_all_icons(repodirs):
 383     """Resize all icons that exceed the max size
 384
 385     :param repodirs: the repo directories to process
 386     """
 387     for repodir in repodirs:
 388         for density in screen_densities:
 389             icon_dir = get_icon_dir(repodir, density)
 390             icon_glob = os.path.join(icon_dir, '*.png')
 391             for iconpath in glob.glob(icon_glob):
 392                 resize_icon(iconpath, density)
 393
 394
 395 def getsig(apkpath):
 396     """ Get the signing certificate of an apk. To get the same md5 has that
 397     Android gets, we encode the .RSA certificate in a specific format and pass
 398     it hex-encoded to the md5 digest algorithm.
 399
 400     :param apkpath: path to the apk
 401     :returns: A string containing the md5 of the signature of the apk or None
 402               if an error occurred.
 403     """
 404
 405     with zipfile.ZipFile(apkpath, 'r') as apk:
 406         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 407
 408         if len(certs) < 1:
 409             logging.error("Found no signing certificates on %s" % apkpath)
 410             return None
 411         if len(certs) > 1:
 412             logging.error("Found multiple signing certificates on %s" % apkpath)
 413             return None
 414
 415         cert = apk.read(certs[0])
 416
 417     cert_encoded = common.get_certificate(cert)
 418
 419     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 420
 421
 422 def get_cache_file():
 423     return os.path.join('tmp', 'apkcache')
 424
 425
 426 def get_cache():
 427     """Get the cached dict of the APK index
 428
 429     Gather information about all the apk files in the repo directory,
 430     using cached data if possible. Some of the index operations take a
 431     long time, like calculating the SHA-256 and verifying the APK
 432     signature.
 433
 434     The cache is invalidated if the metadata version is different, or
 435     the 'allow_disabled_algorithms' config/option is different.  In
 436     those cases, there is no easy way to know what has changed from
 437     the cache, so just rerun the whole thing.
 438
 439     :return: apkcache
 440
 441     """
 442     apkcachefile = get_cache_file()
 443     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 444     if not options.clean and os.path.exists(apkcachefile):
 445         with open(apkcachefile, 'rb') as cf:
 446             apkcache = pickle.load(cf, encoding='utf-8')
 447         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 448            or apkcache.get('allow_disabled_algorithms') != ada:
 449             apkcache = {}
 450     else:
 451         apkcache = {}
 452
 453     apkcache["METADATA_VERSION"] = METADATA_VERSION
 454     apkcache['allow_disabled_algorithms'] = ada
 455
 456     return apkcache
 457
 458
 459 def write_cache(apkcache):
 460     apkcachefile = get_cache_file()
 461     cache_path = os.path.dirname(apkcachefile)
 462     if not os.path.exists(cache_path):
 463         os.makedirs(cache_path)
 464     with open(apkcachefile, 'wb') as cf:
 465         pickle.dump(apkcache, cf)
 466
 467
 468 def get_icon_bytes(apkzip, iconsrc):
 469     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 470     try:
 471         return apkzip.read(iconsrc)
 472     except KeyError:
 473         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 474
 475
 476 def sha256sum(filename):
 477     '''Calculate the sha256 of the given file'''
 478     sha = hashlib.sha256()
 479     with open(filename, 'rb') as f:
 480         while True:
 481             t = f.read(16384)
 482             if len(t) == 0:
 483                 break
 484             sha.update(t)
 485     return sha.hexdigest()
 486
 487
 488 def has_known_vulnerability(filename):
 489     """checks for known vulnerabilities in the APK
 490
 491     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 492     version.  Google also enforces this:
 493     https://support.google.com/faqs/answer/6376725?hl=en
 494
 495     Checks whether there are more than one classes.dex or AndroidManifest.xml
 496     files, which is invalid and an essential part of the "Master Key" attack.
 497
 498     http://www.saurik.com/id/17
 499     """
 500
 501     # statically load this pattern
 502     if not hasattr(has_known_vulnerability, "pattern"):
 503         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 504
 505     files_in_apk = set()
 506     with zipfile.ZipFile(filename) as zf:
 507         for name in zf.namelist():
 508             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 509                 lib = zf.open(name)
 510                 while True:
 511                     chunk = lib.read(4096)
 512                     if chunk == b'':
 513                         break
 514                     m = has_known_vulnerability.pattern.search(chunk)
 515                     if m:
 516                         version = m.group(1).decode('ascii')
 517                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 518                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 519                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 520                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 521                         else:
 522                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 523                             return True
 524                         break
 525             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 526                 if name in files_in_apk:
 527                     return True
 528                 files_in_apk.add(name)
 529
 530     return False
 531
 532
 533 def insert_obbs(repodir, apps, apks):
 534     """Scans the .obb files in a given repo directory and adds them to the
 535     relevant APK instances.  OBB files have versionCodes like APK
 536     files, and they are loosely associated.  If there is an OBB file
 537     present, then any APK with the same or higher versionCode will use
 538     that OBB file.  There are two OBB types: main and patch, each APK
 539     can only have only have one of each.
 540
 541     https://developer.android.com/google/play/expansion-files.html
 542
 543     :param repodir: repo directory to scan
 544     :param apps: list of current, valid apps
 545     :param apks: current information on all APKs
 546
 547     """
 548
 549     def obbWarnDelete(f, msg):
 550         logging.warning(msg + f)
 551         if options.delete_unknown:
 552             logging.error("Deleting unknown file: " + f)
 553             os.remove(f)
 554
 555     obbs = []
 556     java_Integer_MIN_VALUE = -pow(2, 31)
 557     currentPackageNames = apps.keys()
 558     for f in glob.glob(os.path.join(repodir, '*.obb')):
 559         obbfile = os.path.basename(f)
 560         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 561         chunks = obbfile.split('.')
 562         if chunks[0] != 'main' and chunks[0] != 'patch':
 563             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 564             continue
 565         if not re.match(r'^-?[0-9]+$', chunks[1]):
 566             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 567             continue
 568         versionCode = int(chunks[1])
 569         packagename = ".".join(chunks[2:-1])
 570
 571         highestVersionCode = java_Integer_MIN_VALUE
 572         if packagename not in currentPackageNames:
 573             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 574             continue
 575         for apk in apks:
 576             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 577                 highestVersionCode = apk['versionCode']
 578         if versionCode > highestVersionCode:
 579             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 580                           + ') than any APK: ')
 581             continue
 582         obbsha256 = sha256sum(f)
 583         obbs.append((packagename, versionCode, obbfile, obbsha256))
 584
 585     for apk in apks:
 586         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 587             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 588                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 589                     apk['obbMainFile'] = obbfile
 590                     apk['obbMainFileSha256'] = obbsha256
 591                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 592                     apk['obbPatchFile'] = obbfile
 593                     apk['obbPatchFileSha256'] = obbsha256
 594             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 595                 break
 596
 597
 598 def translate_per_build_anti_features(apps, apks):
 599     """Grab the anti-features list from the build metadata
 600
 601     For most Anti-Features, they are really most applicable per-APK,
 602     not for an app.  An app can fix a vulnerability, add/remove
 603     tracking, etc.  This reads the 'antifeatures' list from the Build
 604     entries in the fdroiddata metadata file, then transforms it into
 605     the 'antiFeatures' list of unique items for the index.
 606
 607     The field key is all lower case in the metadata file to match the
 608     rest of the Build fields.  It is 'antiFeatures' camel case in the
 609     implementation, index, and fdroidclient since it is translated
 610     from the build 'antifeatures' field, not directly included.
 611
 612     """
 613
 614     antiFeatures = dict()
 615     for packageName, app in apps.items():
 616         d = dict()
 617         for build in app['builds']:
 618             afl = build.get('antifeatures')
 619             if afl:
 620                 d[int(build.versionCode)] = afl
 621         if len(d) > 0:
 622             antiFeatures[packageName] = d
 623
 624     for apk in apks:
 625         d = antiFeatures.get(apk['packageName'])
 626         if d:
 627             afl = d.get(apk['versionCode'])
 628             if afl:
 629                 apk['antiFeatures'].update(afl)
 630
 631
 632 def _get_localized_dict(app, locale):
 633     '''get the dict to add localized store metadata to'''
 634     if 'localized' not in app:
 635         app['localized'] = collections.OrderedDict()
 636     if locale not in app['localized']:
 637         app['localized'][locale] = collections.OrderedDict()
 638     return app['localized'][locale]
 639
 640
 641 def _set_localized_text_entry(app, locale, key, f):
 642     limit = config['char_limits'][key]
 643     localized = _get_localized_dict(app, locale)
 644     with open(f) as fp:
 645         text = fp.read()[:limit]
 646         if len(text) > 0:
 647             localized[key] = text
 648
 649
 650 def _set_author_entry(app, key, f):
 651     limit = config['char_limits']['author']
 652     with open(f) as fp:
 653         text = fp.read()[:limit]
 654         if len(text) > 0:
 655             app[key] = text
 656
 657
 658 def copy_triple_t_store_metadata(apps):
 659     """Include store metadata from the app's source repo
 660
 661     The Triple-T Gradle Play Publisher is a plugin that has a standard
 662     file layout for all of the metadata and graphics that the Google
 663     Play Store accepts.  Since F-Droid has the git repo, it can just
 664     pluck those files directly.  This method reads any text files into
 665     the app dict, then copies any graphics into the fdroid repo
 666     directory structure.
 667
 668     This needs to be run before insert_localized_app_metadata() so that
 669     the graphics files that are copied into the fdroid repo get
 670     properly indexed.
 671
 672     https://github.com/Triple-T/gradle-play-publisher#upload-images
 673     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 674
 675     """
 676
 677     if not os.path.isdir('build'):
 678         return  # nothing to do
 679
 680     for packageName, app in apps.items():
 681         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 682             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 683             for root, dirs, files in os.walk(d):
 684                 segments = root.split('/')
 685                 locale = segments[-2]
 686                 for f in files:
 687                     if f == 'fulldescription':
 688                         _set_localized_text_entry(app, locale, 'description',
 689                                                   os.path.join(root, f))
 690                         continue
 691                     elif f == 'shortdescription':
 692                         _set_localized_text_entry(app, locale, 'summary',
 693                                                   os.path.join(root, f))
 694                         continue
 695                     elif f == 'title':
 696                         _set_localized_text_entry(app, locale, 'name',
 697                                                   os.path.join(root, f))
 698                         continue
 699                     elif f == 'video':
 700                         _set_localized_text_entry(app, locale, 'video',
 701                                                   os.path.join(root, f))
 702                         continue
 703                     elif f == 'whatsnew':
 704                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 705                                                   os.path.join(root, f))
 706                         continue
 707                     elif f == 'contactEmail':
 708                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 709                         continue
 710                     elif f == 'contactPhone':
 711                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 712                         continue
 713                     elif f == 'contactWebsite':
 714                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 715                         continue
 716
 717                     base, extension = common.get_extension(f)
 718                     dirname = os.path.basename(root)
 719                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 720                         if segments[-2] == 'listing':
 721                             locale = segments[-3]
 722                         else:
 723                             locale = segments[-2]
 724                         destdir = os.path.join('repo', packageName, locale)
 725                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 726                         sourcefile = os.path.join(root, f)
 727                         destfile = os.path.join(destdir, dirname + '.' + extension)
 728                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 729                         shutil.copy(sourcefile, destfile)
 730
 731
 732 def insert_localized_app_metadata(apps):
 733     """scans standard locations for graphics and localized text
 734
 735     Scans for localized description files, store graphics, and
 736     screenshot PNG files in statically defined screenshots directory
 737     and adds them to the app metadata.  The screenshots and graphic
 738     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 739     and must be in the following layout:
 740     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 741
 742     repo/packageName/locale/featureGraphic.png
 743     repo/packageName/locale/phoneScreenshots/1.png
 744     repo/packageName/locale/phoneScreenshots/2.png
 745
 746     The changelog files must be text files named with the versionCode
 747     ending with ".txt" and must be in the following layout:
 748     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 749
 750     repo/packageName/locale/changelogs/12345.txt
 751
 752     This will scan the each app's source repo then the metadata/ dir
 753     for these standard locations of changelog files.  If it finds
 754     them, they will be added to the dict of all packages, with the
 755     versions in the metadata/ folder taking precendence over the what
 756     is in the app's source repo.
 757
 758     Where "packageName" is the app's packageName and "locale" is the locale
 759     of the graphics, e.g. what language they are in, using the IETF RFC5646
 760     format (en-US, fr-CA, es-MX, etc).
 761
 762     This will also scan the app's git for a fastlane folder, and the
 763     metadata/ folder and the apps' source repos for standard locations
 764     of graphic and screenshot files.  If it finds them, it will copy
 765     them into the repo.  The fastlane files follow this pattern:
 766     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 767
 768     """
 769
 770     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 771     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 772     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 773
 774     for srcd in sorted(sourcedirs):
 775         if not os.path.isdir(srcd):
 776             continue
 777         for root, dirs, files in os.walk(srcd):
 778             segments = root.split('/')
 779             packageName = segments[1]
 780             if packageName not in apps:
 781                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 782                 continue
 783             locale = segments[-1]
 784             destdir = os.path.join('repo', packageName, locale)
 785             for f in files:
 786                 if f in ('description.txt', 'full_description.txt'):
 787                     _set_localized_text_entry(apps[packageName], locale, 'description',
 788                                               os.path.join(root, f))
 789                     continue
 790                 elif f in ('summary.txt', 'short_description.txt'):
 791                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 792                                               os.path.join(root, f))
 793                     continue
 794                 elif f in ('name.txt', 'title.txt'):
 795                     _set_localized_text_entry(apps[packageName], locale, 'name',
 796                                               os.path.join(root, f))
 797                     continue
 798                 elif f == 'video.txt':
 799                     _set_localized_text_entry(apps[packageName], locale, 'video',
 800                                               os.path.join(root, f))
 801                     continue
 802                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 803                     locale = segments[-2]
 804                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 805                                               os.path.join(root, f))
 806                     continue
 807
 808                 base, extension = common.get_extension(f)
 809                 if locale == 'images':
 810                     locale = segments[-2]
 811                     destdir = os.path.join('repo', packageName, locale)
 812                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 813                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 814                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 815                     shutil.copy(os.path.join(root, f), destdir)
 816             for d in dirs:
 817                 if d in SCREENSHOT_DIRS:
 818                     for f in glob.glob(os.path.join(root, d, '*.*')):
 819                         _, extension = common.get_extension(f)
 820                         if extension in ALLOWED_EXTENSIONS:
 821                             screenshotdestdir = os.path.join(destdir, d)
 822                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 823                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 824                             shutil.copy(f, screenshotdestdir)
 825
 826     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 827     for d in repofiles:
 828         if not os.path.isdir(d):
 829             continue
 830         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 831             if not os.path.isfile(f):
 832                 continue
 833             segments = f.split('/')
 834             packageName = segments[1]
 835             locale = segments[2]
 836             screenshotdir = segments[3]
 837             filename = os.path.basename(f)
 838             base, extension = common.get_extension(filename)
 839
 840             if packageName not in apps:
 841                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 842                                 % (filename, packageName))
 843                 continue
 844             graphics = _get_localized_dict(apps[packageName], locale)
 845
 846             if extension not in ALLOWED_EXTENSIONS:
 847                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 848             elif base in GRAPHIC_NAMES:
 849                 # there can only be zero or one of these per locale
 850                 graphics[base] = filename
 851             elif screenshotdir in SCREENSHOT_DIRS:
 852                 # there can any number of these per locale
 853                 logging.debug('adding to ' + screenshotdir + ': ' + f)
 854                 if screenshotdir not in graphics:
 855                     graphics[screenshotdir] = []
 856                 graphics[screenshotdir].append(filename)
 857             else:
 858                 logging.warning('Unsupported graphics file found: ' + f)
 859
 860
 861 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 862     """Scan a repo for all files with an extension except APK/OBB
 863
 864     :param apkcache: current cached info about all repo files
 865     :param repodir: repo directory to scan
 866     :param knownapks: list of all known files, as per metadata.read_metadata
 867     :param use_date_from_file: use date from file (instead of current date)
 868                                for newly added files
 869     """
 870
 871     cachechanged = False
 872     repo_files = []
 873     repodir = repodir.encode('utf-8')
 874     for name in os.listdir(repodir):
 875         file_extension = common.get_file_extension(name)
 876         if file_extension == 'apk' or file_extension == 'obb':
 877             continue
 878         filename = os.path.join(repodir, name)
 879         name_utf8 = name.decode('utf-8')
 880         if filename.endswith(b'_src.tar.gz'):
 881             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
 882             continue
 883         if not common.is_repo_file(filename):
 884             continue
 885         stat = os.stat(filename)
 886         if stat.st_size == 0:
 887             raise FDroidException(filename + ' is zero size!')
 888
 889         shasum = sha256sum(filename)
 890         usecache = False
 891         if name in apkcache:
 892             repo_file = apkcache[name]
 893             # added time is cached as tuple but used here as datetime instance
 894             if 'added' in repo_file:
 895                 a = repo_file['added']
 896                 if isinstance(a, datetime):
 897                     repo_file['added'] = a
 898                 else:
 899                     repo_file['added'] = datetime(*a[:6])
 900             if repo_file.get('hash') == shasum:
 901                 logging.debug("Reading " + name_utf8 + " from cache")
 902                 usecache = True
 903             else:
 904                 logging.debug("Ignoring stale cache data for " + name_utf8)
 905
 906         if not usecache:
 907             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 908             repo_file = collections.OrderedDict()
 909             repo_file['name'] = os.path.splitext(name_utf8)[0]
 910             # TODO rename apkname globally to something more generic
 911             repo_file['apkName'] = name_utf8
 912             repo_file['hash'] = shasum
 913             repo_file['hashType'] = 'sha256'
 914             repo_file['versionCode'] = 0
 915             repo_file['versionName'] = shasum
 916             # the static ID is the SHA256 unless it is set in the metadata
 917             repo_file['packageName'] = shasum
 918
 919             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
 920             if m:
 921                 repo_file['packageName'] = m.group(1)
 922                 repo_file['versionCode'] = int(m.group(2))
 923             srcfilename = name + b'_src.tar.gz'
 924             if os.path.exists(os.path.join(repodir, srcfilename)):
 925                 repo_file['srcname'] = srcfilename.decode('utf-8')
 926             repo_file['size'] = stat.st_size
 927
 928             apkcache[name] = repo_file
 929             cachechanged = True
 930
 931         if use_date_from_file:
 932             timestamp = stat.st_ctime
 933             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 934         else:
 935             default_date_param = None
 936
 937         # Record in knownapks, getting the added date at the same time..
 938         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 939                                     default_date=default_date_param)
 940         if added:
 941             repo_file['added'] = added
 942
 943         repo_files.append(repo_file)
 944
 945     return repo_files, cachechanged
 946
 947
 948 def scan_apk(apk_file):
 949     """
 950     Scans an APK file and returns dictionary with metadata of the APK.
 951
 952     Attention: This does *not* verify that the APK signature is correct.
 953
 954     :param apk_file: The (ideally absolute) path to the APK file
 955     :raises BuildException
 956     :return A dict containing APK metadata
 957     """
 958     apk = {
 959         'hash': sha256sum(apk_file),
 960         'hashType': 'sha256',
 961         'uses-permission': [],
 962         'uses-permission-sdk-23': [],
 963         'features': [],
 964         'icons_src': {},
 965         'icons': {},
 966         'antiFeatures': set(),
 967     }
 968
 969     if SdkToolsPopen(['aapt', 'version'], output=False):
 970         scan_apk_aapt(apk, apk_file)
 971     else:
 972         scan_apk_androguard(apk, apk_file)
 973
 974     # Get the signature, or rather the signing key fingerprints
 975     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 976     apk['sig'] = getsig(apk_file)
 977     if not apk['sig']:
 978         raise BuildException("Failed to get apk signature")
 979     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
 980                                                                apk_file))
 981     if not apk.get('signer'):
 982         raise BuildException("Failed to get apk signing key fingerprint")
 983
 984     # Get size of the APK
 985     apk['size'] = os.path.getsize(apk_file)
 986
 987     if 'minSdkVersion' not in apk:
 988         logging.warning("No SDK version information found in {0}".format(apk_file))
 989         apk['minSdkVersion'] = 1
 990
 991     # Check for known vulnerabilities
 992     if has_known_vulnerability(apk_file):
 993         apk['antiFeatures'].add('KnownVuln')
 994
 995     return apk
 996
 997
 998 def scan_apk_aapt(apk, apkfile):
 999     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1000     if p.returncode != 0:
1001         if options.delete_unknown:
1002             if os.path.exists(apkfile):
1003                 logging.error("Failed to get apk information, deleting " + apkfile)
1004                 os.remove(apkfile)
1005             else:
1006                 logging.error("Could not find {0} to remove it".format(apkfile))
1007         else:
1008             logging.error("Failed to get apk information, skipping " + apkfile)
1009         raise BuildException("Invalid APK")
1010     for line in p.output.splitlines():
1011         if line.startswith("package:"):
1012             try:
1013                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1014                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1015                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1016             except Exception as e:
1017                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1018         elif line.startswith("application:"):
1019             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1020             # Keep path to non-dpi icon in case we need it
1021             match = re.match(APK_ICON_PAT_NODPI, line)
1022             if match:
1023                 apk['icons_src']['-1'] = match.group(1)
1024         elif line.startswith("launchable-activity:"):
1025             # Only use launchable-activity as fallback to application
1026             if not apk['name']:
1027                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1028             if '-1' not in apk['icons_src']:
1029                 match = re.match(APK_ICON_PAT_NODPI, line)
1030                 if match:
1031                     apk['icons_src']['-1'] = match.group(1)
1032         elif line.startswith("application-icon-"):
1033             match = re.match(APK_ICON_PAT, line)
1034             if match:
1035                 density = match.group(1)
1036                 path = match.group(2)
1037                 apk['icons_src'][density] = path
1038         elif line.startswith("sdkVersion:"):
1039             m = re.match(APK_SDK_VERSION_PAT, line)
1040             if m is None:
1041                 logging.error(line.replace('sdkVersion:', '')
1042                               + ' is not a valid minSdkVersion!')
1043             else:
1044                 apk['minSdkVersion'] = m.group(1)
1045                 # if target not set, default to min
1046                 if 'targetSdkVersion' not in apk:
1047                     apk['targetSdkVersion'] = m.group(1)
1048         elif line.startswith("targetSdkVersion:"):
1049             m = re.match(APK_SDK_VERSION_PAT, line)
1050             if m is None:
1051                 logging.error(line.replace('targetSdkVersion:', '')
1052                               + ' is not a valid targetSdkVersion!')
1053             else:
1054                 apk['targetSdkVersion'] = m.group(1)
1055         elif line.startswith("maxSdkVersion:"):
1056             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1057         elif line.startswith("native-code:"):
1058             apk['nativecode'] = []
1059             for arch in line[13:].split(' '):
1060                 apk['nativecode'].append(arch[1:-1])
1061         elif line.startswith('uses-permission:'):
1062             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1063             if perm_match['maxSdkVersion']:
1064                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1065             permission = UsesPermission(
1066                 perm_match['name'],
1067                 perm_match['maxSdkVersion']
1068             )
1069
1070             apk['uses-permission'].append(permission)
1071         elif line.startswith('uses-permission-sdk-23:'):
1072             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1073             if perm_match['maxSdkVersion']:
1074                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1075             permission_sdk_23 = UsesPermissionSdk23(
1076                 perm_match['name'],
1077                 perm_match['maxSdkVersion']
1078             )
1079
1080             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1081
1082         elif line.startswith('uses-feature:'):
1083             feature = re.match(APK_FEATURE_PAT, line).group(1)
1084             # Filter out this, it's only added with the latest SDK tools and
1085             # causes problems for lots of apps.
1086             if feature != "android.hardware.screen.portrait" \
1087                     and feature != "android.hardware.screen.landscape":
1088                 if feature.startswith("android.feature."):
1089                     feature = feature[16:]
1090                 apk['features'].add(feature)
1091
1092
1093 def scan_apk_androguard(apk, apkfile):
1094     try:
1095         from androguard.core.bytecodes.apk import APK
1096         apkobject = APK(apkfile)
1097         if apkobject.is_valid_APK():
1098             arsc = apkobject.get_android_resources()
1099         else:
1100             if options.delete_unknown:
1101                 if os.path.exists(apkfile):
1102                     logging.error("Failed to get apk information, deleting " + apkfile)
1103                     os.remove(apkfile)
1104                 else:
1105                     logging.error("Could not find {0} to remove it".format(apkfile))
1106             else:
1107                 logging.error("Failed to get apk information, skipping " + apkfile)
1108             raise BuildException("Invaild APK")
1109     except ImportError:
1110         raise FDroidException("androguard library is not installed and aapt not present")
1111     except FileNotFoundError:
1112         logging.error("Could not open apk file for analysis")
1113         raise BuildException("Invalid APK")
1114
1115     apk['packageName'] = apkobject.get_package()
1116     apk['versionCode'] = int(apkobject.get_androidversion_code())
1117     apk['versionName'] = apkobject.get_androidversion_name()
1118     if apk['versionName'][0] == "@":
1119         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1120         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1121         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1122     apk['name'] = apkobject.get_app_name()
1123
1124     if apkobject.get_max_sdk_version() is not None:
1125         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1126     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1127     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1128
1129     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1130     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1131
1132     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1133
1134     for file in apkobject.get_files():
1135         d_re = density_re.match(file)
1136         if d_re:
1137             folder = d_re.group(1).split('-')
1138             if len(folder) > 1:
1139                 resolution = folder[1]
1140             else:
1141                 resolution = 'mdpi'
1142             density = screen_resolutions[resolution]
1143             apk['icons_src'][density] = d_re.group(0)
1144
1145     if apk['icons_src'].get('-1') is None:
1146         apk['icons_src']['-1'] = apk['icons_src']['160']
1147
1148     arch_re = re.compile("^lib/(.*)/.*$")
1149     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1150     if len(arch) >= 1:
1151         apk['nativecode'] = []
1152         apk['nativecode'].extend(sorted(list(arch)))
1153
1154     xml = apkobject.get_android_manifest_xml()
1155
1156     for item in xml.getElementsByTagName('uses-permission'):
1157         name = str(item.getAttribute("android:name"))
1158         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1159         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1160         permission = UsesPermission(
1161             name,
1162             maxSdkVersion
1163         )
1164         apk['uses-permission'].append(permission)
1165
1166     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1167         name = str(item.getAttribute("android:name"))
1168         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1169         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1170         permission_sdk_23 = UsesPermissionSdk23(
1171             name,
1172             maxSdkVersion
1173         )
1174         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1175
1176     for item in xml.getElementsByTagName('uses-feature'):
1177         feature = str(item.getAttribute("android:name"))
1178         if feature != "android.hardware.screen.portrait" \
1179                 and feature != "android.hardware.screen.landscape":
1180             if feature.startswith("android.feature."):
1181                 feature = feature[16:]
1182         apk['features'].append(feature)
1183
1184
1185 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1186                 allow_disabled_algorithms=False, archive_bad_sig=False):
1187     """Processes the apk with the given filename in the given repo directory.
1188
1189     This also extracts the icons.
1190
1191     :param apkcache: current apk cache information
1192     :param apkfilename: the filename of the apk to scan
1193     :param repodir: repo directory to scan
1194     :param knownapks: known apks info
1195     :param use_date_from_apk: use date from APK (instead of current date)
1196                               for newly added APKs
1197     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1198                                       disabled algorithms in the signature (e.g. MD5)
1199     :param archive_bad_sig: move APKs with a bad signature to the archive
1200     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1201      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1202     """
1203
1204     apk = {}
1205     apkfile = os.path.join(repodir, apkfilename)
1206
1207     cachechanged = False
1208     usecache = False
1209     if apkfilename in apkcache:
1210         apk = apkcache[apkfilename]
1211         if apk.get('hash') == sha256sum(apkfile):
1212             logging.debug("Reading " + apkfilename + " from cache")
1213             usecache = True
1214         else:
1215             logging.debug("Ignoring stale cache data for " + apkfilename)
1216
1217     if not usecache:
1218         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1219
1220         try:
1221             apk = scan_apk(apkfile)
1222         except BuildException:
1223             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1224                             .format(apkfilename=apkfilename))
1225             return True, None, False
1226
1227         # Check for debuggable apks...
1228         if common.isApkAndDebuggable(apkfile):
1229             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1230
1231         if options.rename_apks:
1232             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1233             std_short_name = os.path.join(repodir, n)
1234             if apkfile != std_short_name:
1235                 if os.path.exists(std_short_name):
1236                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1237                     if apkfile != std_long_name:
1238                         if os.path.exists(std_long_name):
1239                             dupdir = os.path.join('duplicates', repodir)
1240                             if not os.path.isdir(dupdir):
1241                                 os.makedirs(dupdir, exist_ok=True)
1242                             dupfile = os.path.join('duplicates', std_long_name)
1243                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1244                             os.rename(apkfile, dupfile)
1245                             return True, None, False
1246                         else:
1247                             os.rename(apkfile, std_long_name)
1248                     apkfile = std_long_name
1249                 else:
1250                     os.rename(apkfile, std_short_name)
1251                     apkfile = std_short_name
1252                 apkfilename = apkfile[len(repodir) + 1:]
1253
1254         apk['apkName'] = apkfilename
1255         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1256         if os.path.exists(os.path.join(repodir, srcfilename)):
1257             apk['srcname'] = srcfilename
1258
1259         # verify the jar signature is correct, allow deprecated
1260         # algorithms only if the APK is in the archive.
1261         skipapk = False
1262         if not common.verify_apk_signature(apkfile):
1263             if repodir == 'archive' or allow_disabled_algorithms:
1264                 if common.verify_old_apk_signature(apkfile):
1265                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1266                 else:
1267                     skipapk = True
1268             else:
1269                 skipapk = True
1270
1271         if skipapk:
1272             if archive_bad_sig:
1273                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1274                 move_apk_between_sections(repodir, 'archive', apk)
1275             else:
1276                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1277             return True, None, False
1278
1279         apkzip = zipfile.ZipFile(apkfile, 'r')
1280
1281         # if an APK has files newer than the system time, suggest updating
1282         # the system clock.  This is useful for offline systems, used for
1283         # signing, which do not have another source of clock sync info. It
1284         # has to be more than 24 hours newer because ZIP/APK files do not
1285         # store timezone info
1286         manifest = apkzip.getinfo('AndroidManifest.xml')
1287         if manifest.date_time[1] == 0:  # month can't be zero
1288             logging.debug('AndroidManifest.xml has no date')
1289         else:
1290             dt_obj = datetime(*manifest.date_time)
1291             checkdt = dt_obj - timedelta(1)
1292             if datetime.today() < checkdt:
1293                 logging.warning('System clock is older than manifest in: '
1294                                 + apkfilename
1295                                 + '\nSet clock to that time using:\n'
1296                                 + 'sudo date -s "' + str(dt_obj) + '"')
1297
1298         # extract icons from APK zip file
1299         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1300         try:
1301             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1302         finally:
1303             apkzip.close()  # ensure that APK zip file gets closed
1304
1305         # resize existing icons for densities missing in the APK
1306         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1307
1308         if use_date_from_apk and manifest.date_time[1] != 0:
1309             default_date_param = datetime(*manifest.date_time)
1310         else:
1311             default_date_param = None
1312
1313         # Record in known apks, getting the added date at the same time..
1314         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1315                                     default_date=default_date_param)
1316         if added:
1317             apk['added'] = added
1318
1319         apkcache[apkfilename] = apk
1320         cachechanged = True
1321
1322     return False, apk, cachechanged
1323
1324
1325 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1326     """Processes the apks in the given repo directory.
1327
1328     This also extracts the icons.
1329
1330     :param apkcache: current apk cache information
1331     :param repodir: repo directory to scan
1332     :param knownapks: known apks info
1333     :param use_date_from_apk: use date from APK (instead of current date)
1334                               for newly added APKs
1335     :returns: (apks, cachechanged) where apks is a list of apk information,
1336               and cachechanged is True if the apkcache got changed.
1337     """
1338
1339     cachechanged = False
1340
1341     for icon_dir in get_all_icon_dirs(repodir):
1342         if os.path.exists(icon_dir):
1343             if options.clean:
1344                 shutil.rmtree(icon_dir)
1345                 os.makedirs(icon_dir)
1346         else:
1347             os.makedirs(icon_dir)
1348
1349     apks = []
1350     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1351         apkfilename = apkfile[len(repodir) + 1:]
1352         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1353         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1354                                              use_date_from_apk, ada, True)
1355         if skip:
1356             continue
1357         apks.append(apk)
1358         cachechanged = cachechanged or cachethis
1359
1360     return apks, cachechanged
1361
1362
1363 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1364     """
1365     Extracts icons from the given APK zip in various densities,
1366     saves them into given repo directory
1367     and stores their names in the APK metadata dictionary.
1368
1369     :param icon_filename: A string representing the icon's file name
1370     :param apk: A populated dictionary containing APK metadata.
1371                 Needs to have 'icons_src' key
1372     :param apkzip: An opened zipfile.ZipFile of the APK file
1373     :param repo_dir: The directory of the APK's repository
1374     :return: A list of icon densities that are missing
1375     """
1376     empty_densities = []
1377     for density in screen_densities:
1378         if density not in apk['icons_src']:
1379             empty_densities.append(density)
1380             continue
1381         icon_src = apk['icons_src'][density]
1382         icon_dir = get_icon_dir(repo_dir, density)
1383         icon_dest = os.path.join(icon_dir, icon_filename)
1384
1385         # Extract the icon files per density
1386         if icon_src.endswith('.xml'):
1387             png = os.path.basename(icon_src)[:-4] + '.png'
1388             for f in apkzip.namelist():
1389                 if f.endswith(png):
1390                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1391                     if m and screen_resolutions[m.group(2)] == density:
1392                         icon_src = f
1393             if icon_src.endswith('.xml'):
1394                 empty_densities.append(density)
1395                 continue
1396         try:
1397             with open(icon_dest, 'wb') as f:
1398                 f.write(get_icon_bytes(apkzip, icon_src))
1399             apk['icons'][density] = icon_filename
1400         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1401             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1402             del apk['icons_src'][density]
1403             empty_densities.append(density)
1404
1405     if '-1' in apk['icons_src']:
1406         icon_src = apk['icons_src']['-1']
1407         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1408         with open(icon_path, 'wb') as f:
1409             f.write(get_icon_bytes(apkzip, icon_src))
1410         try:
1411             im = Image.open(icon_path)
1412             dpi = px_to_dpi(im.size[0])
1413             for density in screen_densities:
1414                 if density in apk['icons']:
1415                     break
1416                 if density == screen_densities[-1] or dpi >= int(density):
1417                     apk['icons'][density] = icon_filename
1418                     shutil.move(icon_path,
1419                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1420                     empty_densities.remove(density)
1421                     break
1422         except Exception as e:
1423             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1424
1425     if apk['icons']:
1426         apk['icon'] = icon_filename
1427
1428     return empty_densities
1429
1430
1431 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1432     """
1433     Resize existing icons for densities missing in the APK to ensure all densities are available
1434
1435     :param empty_densities: A list of icon densities that are missing
1436     :param icon_filename: A string representing the icon's file name
1437     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1438     :param repo_dir: The directory of the APK's repository
1439     """
1440     # First try resizing down to not lose quality
1441     last_density = None
1442     for density in screen_densities:
1443         if density not in empty_densities:
1444             last_density = density
1445             continue
1446         if last_density is None:
1447             continue
1448         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1449
1450         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1451         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1452         fp = None
1453         try:
1454             fp = open(last_icon_path, 'rb')
1455             im = Image.open(fp)
1456
1457             size = dpi_to_px(density)
1458
1459             im.thumbnail((size, size), Image.ANTIALIAS)
1460             im.save(icon_path, "PNG")
1461             empty_densities.remove(density)
1462         except Exception as e:
1463             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1464         finally:
1465             if fp:
1466                 fp.close()
1467
1468     # Then just copy from the highest resolution available
1469     last_density = None
1470     for density in reversed(screen_densities):
1471         if density not in empty_densities:
1472             last_density = density
1473             continue
1474
1475         if last_density is None:
1476             continue
1477
1478         shutil.copyfile(
1479             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1480             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1481         )
1482         empty_densities.remove(density)
1483
1484     for density in screen_densities:
1485         icon_dir = get_icon_dir(repo_dir, density)
1486         icon_dest = os.path.join(icon_dir, icon_filename)
1487         resize_icon(icon_dest, density)
1488
1489     # Copy from icons-mdpi to icons since mdpi is the baseline density
1490     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1491     if os.path.isfile(baseline):
1492         apk['icons']['0'] = icon_filename
1493         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1494
1495
1496 def apply_info_from_latest_apk(apps, apks):
1497     """
1498     Some information from the apks needs to be applied up to the application level.
1499     When doing this, we use the info from the most recent version's apk.
1500     We deal with figuring out when the app was added and last updated at the same time.
1501     """
1502     for appid, app in apps.items():
1503         bestver = UNSET_VERSION_CODE
1504         for apk in apks:
1505             if apk['packageName'] == appid:
1506                 if apk['versionCode'] > bestver:
1507                     bestver = apk['versionCode']
1508                     bestapk = apk
1509
1510                 if 'added' in apk:
1511                     if not app.added or apk['added'] < app.added:
1512                         app.added = apk['added']
1513                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1514                         app.lastUpdated = apk['added']
1515
1516         if not app.added:
1517             logging.debug("Don't know when " + appid + " was added")
1518         if not app.lastUpdated:
1519             logging.debug("Don't know when " + appid + " was last updated")
1520
1521         if bestver == UNSET_VERSION_CODE:
1522
1523             if app.Name is None:
1524                 app.Name = app.AutoName or appid
1525             app.icon = None
1526             logging.debug("Application " + appid + " has no packages")
1527         else:
1528             if app.Name is None:
1529                 app.Name = bestapk['name']
1530             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1531             if app.CurrentVersionCode is None:
1532                 app.CurrentVersionCode = str(bestver)
1533
1534
1535 def make_categories_txt(repodir, categories):
1536     '''Write a category list in the repo to allow quick access'''
1537     catdata = ''
1538     for cat in sorted(categories):
1539         catdata += cat + '\n'
1540     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1541         f.write(catdata)
1542
1543
1544 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1545
1546     def filter_apk_list_sorted(apk_list):
1547         res = []
1548         for apk in apk_list:
1549             if apk['packageName'] == appid:
1550                 res.append(apk)
1551
1552         # Sort the apk list by version code. First is highest/newest.
1553         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1554
1555     for appid, app in apps.items():
1556
1557         if app.ArchivePolicy:
1558             keepversions = int(app.ArchivePolicy[:-9])
1559         else:
1560             keepversions = defaultkeepversions
1561
1562         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1563                       .format(appid, len(apks), keepversions, len(archapks)))
1564
1565         current_app_apks = filter_apk_list_sorted(apks)
1566         if len(current_app_apks) > keepversions:
1567             # Move back the ones we don't want.
1568             for apk in current_app_apks[keepversions:]:
1569                 move_apk_between_sections(repodir, archivedir, apk)
1570                 archapks.append(apk)
1571                 apks.remove(apk)
1572
1573         current_app_archapks = filter_apk_list_sorted(archapks)
1574         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1575             kept = 0
1576             # Move forward the ones we want again, except DisableAlgorithm
1577             for apk in current_app_archapks:
1578                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1579                     move_apk_between_sections(archivedir, repodir, apk)
1580                     archapks.remove(apk)
1581                     apks.append(apk)
1582                     kept += 1
1583                 if kept == keepversions:
1584                     break
1585
1586
1587 def move_apk_between_sections(from_dir, to_dir, apk):
1588     """move an APK from repo to archive or vice versa"""
1589
1590     def _move_file(from_dir, to_dir, filename, ignore_missing):
1591         from_path = os.path.join(from_dir, filename)
1592         if ignore_missing and not os.path.exists(from_path):
1593             return
1594         to_path = os.path.join(to_dir, filename)
1595         if not os.path.exists(to_dir):
1596             os.mkdir(to_dir)
1597         shutil.move(from_path, to_path)
1598
1599     if from_dir == to_dir:
1600         return
1601
1602     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1603     _move_file(from_dir, to_dir, apk['apkName'], False)
1604     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1605     for density in all_screen_densities:
1606         from_icon_dir = get_icon_dir(from_dir, density)
1607         to_icon_dir = get_icon_dir(to_dir, density)
1608         if density not in apk['icons']:
1609             continue
1610         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1611     if 'srcname' in apk:
1612         _move_file(from_dir, to_dir, apk['srcname'], False)
1613
1614
1615 def add_apks_to_per_app_repos(repodir, apks):
1616     apks_per_app = dict()
1617     for apk in apks:
1618         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1619         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1620         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1621         apks_per_app[apk['packageName']] = apk
1622
1623         if not os.path.exists(apk['per_app_icons']):
1624             logging.info('Adding new repo for only ' + apk['packageName'])
1625             os.makedirs(apk['per_app_icons'])
1626
1627         apkpath = os.path.join(repodir, apk['apkName'])
1628         shutil.copy(apkpath, apk['per_app_repo'])
1629         apksigpath = apkpath + '.sig'
1630         if os.path.exists(apksigpath):
1631             shutil.copy(apksigpath, apk['per_app_repo'])
1632         apkascpath = apkpath + '.asc'
1633         if os.path.exists(apkascpath):
1634             shutil.copy(apkascpath, apk['per_app_repo'])
1635
1636
1637 def create_metadata_from_template(apk):
1638     '''create a new metadata file using internal or external template
1639
1640     Generate warnings for apk's with no metadata (or create skeleton
1641     metadata files, if requested on the command line).  Though the
1642     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1643     since those impose things on the metadata file made from the
1644     template: field sort order, empty field value, formatting, etc.
1645     '''
1646
1647     import yaml
1648     if os.path.exists('template.yml'):
1649         with open('template.yml') as f:
1650             metatxt = f.read()
1651         if 'name' in apk and apk['name'] != '':
1652             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1653                              r'\1 ' + apk['name'],
1654                              metatxt,
1655                              flags=re.IGNORECASE | re.MULTILINE)
1656         else:
1657             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1658             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1659                              r'\1 ' + apk['packageName'],
1660                              metatxt,
1661                              flags=re.IGNORECASE | re.MULTILINE)
1662         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1663             f.write(metatxt)
1664     else:
1665         app = dict()
1666         app['Categories'] = [os.path.basename(os.getcwd())]
1667         # include some blanks as part of the template
1668         app['AuthorName'] = ''
1669         app['Summary'] = ''
1670         app['WebSite'] = ''
1671         app['IssueTracker'] = ''
1672         app['SourceCode'] = ''
1673         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1674         if 'name' in apk and apk['name'] != '':
1675             app['Name'] = apk['name']
1676         else:
1677             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1678             app['Name'] = apk['packageName']
1679         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1680             yaml.dump(app, f, default_flow_style=False)
1681     logging.info("Generated skeleton metadata for " + apk['packageName'])
1682
1683
1684 config = None
1685 options = None
1686
1687
1688 def main():
1689
1690     global config, options
1691
1692     # Parse command line...
1693     parser = ArgumentParser()
1694     common.setup_global_opts(parser)
1695     parser.add_argument("--create-key", action="store_true", default=False,
1696                         help=_("Create a repo signing key in a keystore"))
1697     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1698                         help=_("Create skeleton metadata files that are missing"))
1699     parser.add_argument("--delete-unknown", action="store_true", default=False,
1700                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1701     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1702                         help=_("Report on build data status"))
1703     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1704                         help=_("Interactively ask about things that need updating."))
1705     parser.add_argument("-I", "--icons", action="store_true", default=False,
1706                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1707     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1708                         help=_("Specify editor to use in interactive mode. Default ") +
1709                         "is /etc/alternatives/editor")
1710     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1711                         help=_("Update the wiki"))
1712     parser.add_argument("--pretty", action="store_true", default=False,
1713                         help=_("Produce human-readable index.xml"))
1714     parser.add_argument("--clean", action="store_true", default=False,
1715                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1716     parser.add_argument("--nosign", action="store_true", default=False,
1717                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1718     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1719                         help=_("Use date from APK instead of current time for newly added APKs"))
1720     parser.add_argument("--rename-apks", action="store_true", default=False,
1721                         help=_("Rename APK files that do not match package.name_123.apk"))
1722     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1723                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1724     metadata.add_metadata_arguments(parser)
1725     options = parser.parse_args()
1726     metadata.warnings_action = options.W
1727
1728     config = common.read_config(options)
1729
1730     if not ('jarsigner' in config and 'keytool' in config):
1731         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1732
1733     repodirs = ['repo']
1734     if config['archive_older'] != 0:
1735         repodirs.append('archive')
1736         if not os.path.exists('archive'):
1737             os.mkdir('archive')
1738
1739     if options.icons:
1740         resize_all_icons(repodirs)
1741         sys.exit(0)
1742
1743     if options.rename_apks:
1744         options.clean = True
1745
1746     # check that icons exist now, rather than fail at the end of `fdroid update`
1747     for k in ['repo_icon', 'archive_icon']:
1748         if k in config:
1749             if not os.path.exists(config[k]):
1750                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1751                 sys.exit(1)
1752
1753     # if the user asks to create a keystore, do it now, reusing whatever it can
1754     if options.create_key:
1755         if os.path.exists(config['keystore']):
1756             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1757             logging.critical("\t'" + config['keystore'] + "'")
1758             sys.exit(1)
1759
1760         if 'repo_keyalias' not in config:
1761             config['repo_keyalias'] = socket.getfqdn()
1762             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1763         if 'keydname' not in config:
1764             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1765             common.write_to_config(config, 'keydname', config['keydname'])
1766         if 'keystore' not in config:
1767             config['keystore'] = common.default_config['keystore']
1768             common.write_to_config(config, 'keystore', config['keystore'])
1769
1770         password = common.genpassword()
1771         if 'keystorepass' not in config:
1772             config['keystorepass'] = password
1773             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1774         if 'keypass' not in config:
1775             config['keypass'] = password
1776             common.write_to_config(config, 'keypass', config['keypass'])
1777         common.genkeystore(config)
1778
1779     # Get all apps...
1780     apps = metadata.read_metadata()
1781
1782     # Generate a list of categories...
1783     categories = set()
1784     for app in apps.values():
1785         categories.update(app.Categories)
1786
1787     # Read known apks data (will be updated and written back when we've finished)
1788     knownapks = common.KnownApks()
1789
1790     # Get APK cache
1791     apkcache = get_cache()
1792
1793     # Delete builds for disabled apps
1794     delete_disabled_builds(apps, apkcache, repodirs)
1795
1796     # Scan all apks in the main repo
1797     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1798
1799     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1800                                            options.use_date_from_apk)
1801     cachechanged = cachechanged or fcachechanged
1802     apks += files
1803     for apk in apks:
1804         if apk['packageName'] not in apps:
1805             if options.create_metadata:
1806                 create_metadata_from_template(apk)
1807                 apps = metadata.read_metadata()
1808             else:
1809                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1810                 if options.delete_unknown:
1811                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1812                     rmf = os.path.join(repodirs[0], apk['apkName'])
1813                     if not os.path.exists(rmf):
1814                         logging.error("Could not find {0} to remove it".format(rmf))
1815                     else:
1816                         os.remove(rmf)
1817                 else:
1818                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1819
1820     copy_triple_t_store_metadata(apps)
1821     insert_obbs(repodirs[0], apps, apks)
1822     insert_localized_app_metadata(apps)
1823     translate_per_build_anti_features(apps, apks)
1824
1825     # Scan the archive repo for apks as well
1826     if len(repodirs) > 1:
1827         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1828         if cc:
1829             cachechanged = True
1830     else:
1831         archapks = []
1832
1833     # Apply information from latest apks to the application and update dates
1834     apply_info_from_latest_apk(apps, apks + archapks)
1835
1836     # Sort the app list by name, then the web site doesn't have to by default.
1837     # (we had to wait until we'd scanned the apks to do this, because mostly the
1838     # name comes from there!)
1839     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1840
1841     # APKs are placed into multiple repos based on the app package, providing
1842     # per-app subscription feeds for nightly builds and things like it
1843     if config['per_app_repos']:
1844         add_apks_to_per_app_repos(repodirs[0], apks)
1845         for appid, app in apps.items():
1846             repodir = os.path.join(appid, 'fdroid', 'repo')
1847             appdict = dict()
1848             appdict[appid] = app
1849             if os.path.isdir(repodir):
1850                 index.make(appdict, [appid], apks, repodir, False)
1851             else:
1852                 logging.info('Skipping index generation for ' + appid)
1853         return
1854
1855     if len(repodirs) > 1:
1856         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1857
1858     # Make the index for the main repo...
1859     index.make(apps, sortedids, apks, repodirs[0], False)
1860     make_categories_txt(repodirs[0], categories)
1861
1862     # If there's an archive repo,  make the index for it. We already scanned it
1863     # earlier on.
1864     if len(repodirs) > 1:
1865         index.make(apps, sortedids, archapks, repodirs[1], True)
1866
1867     git_remote = config.get('binary_transparency_remote')
1868     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1869         from . import btlog
1870         btlog.make_binary_transparency_log(repodirs)
1871
1872     if config['update_stats']:
1873         # Update known apks info...
1874         knownapks.writeifchanged()
1875
1876         # Generate latest apps data for widget
1877         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1878             data = ''
1879             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1880                 for line in f:
1881                     appid = line.rstrip()
1882                     data += appid + "\t"
1883                     app = apps[appid]
1884                     data += app.Name + "\t"
1885                     if app.icon is not None:
1886                         data += app.icon + "\t"
1887                     data += app.License + "\n"
1888             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1889                 f.write(data)
1890
1891     if cachechanged:
1892         write_cache(apkcache)
1893
1894     # Update the wiki...
1895     if options.wiki:
1896         update_wiki(apps, sortedids, apks + archapks)
1897
1898     logging.info(_("Finished"))
1899
1900
1901 if __name__ == "__main__":
1902     main()