fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*)'")
  57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  58 APK_PERMISSION_PAT = \
  59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  61
  62 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
  63 # resolutions must end with 'dpi'
  64 screen_resolutions = {
  65     "xxxhdpi": '640',
  66     "xxhdpi": '480',
  67     "xhdpi": '320',
  68     "hdpi": '240',
  69     "mdpi": '160',
  70     "ldpi": '120',
  71     "tvdpi": '213',
  72     "undefineddpi": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0' or density == '65534':
 100         return os.path.join(repodir, "icons")
 101     else:
 102         return os.path.join(repodir, "icons-%s" % density)
 103
 104
 105 def get_icon_dirs(repodir):
 106     for density in screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def get_all_icon_dirs(repodir):
 111     for density in all_screen_densities:
 112         yield get_icon_dir(repodir, density)
 113
 114
 115 def update_wiki(apps, sortedids, apks):
 116     """Update the wiki
 117
 118     :param apps: fully populated list of all applications
 119     :param apks: all apks, except...
 120     """
 121     logging.info("Updating wiki")
 122     wikicat = 'Apps'
 123     wikiredircat = 'App Redirects'
 124     import mwclient
 125     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 126                          path=config['wiki_path'])
 127     site.login(config['wiki_user'], config['wiki_password'])
 128     generated_pages = {}
 129     generated_redirects = {}
 130
 131     for appid in sortedids:
 132         app = metadata.App(apps[appid])
 133
 134         wikidata = ''
 135         if app.Disabled:
 136             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 137         if app.AntiFeatures:
 138             for af in sorted(app.AntiFeatures):
 139                 wikidata += '{{AntiFeature|' + af + '}}\n'
 140         if app.RequiresRoot:
 141             requiresroot = 'Yes'
 142         else:
 143             requiresroot = 'No'
 144         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 145             appid,
 146             app.Name,
 147             app.added.strftime('%Y-%m-%d') if app.added else '',
 148             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 149             app.SourceCode,
 150             app.IssueTracker,
 151             app.WebSite,
 152             app.Changelog,
 153             app.Donate,
 154             app.FlattrID,
 155             app.LiberapayID,
 156             app.Bitcoin,
 157             app.Litecoin,
 158             app.License,
 159             requiresroot,
 160             app.AuthorName,
 161             app.AuthorEmail)
 162
 163         if app.Provides:
 164             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 165
 166         wikidata += app.Summary
 167         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 168
 169         wikidata += "=Description=\n"
 170         wikidata += metadata.description_wiki(app.Description) + "\n"
 171
 172         wikidata += "=Maintainer Notes=\n"
 173         if app.MaintainerNotes:
 174             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 175         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 176
 177         # Get a list of all packages for this application...
 178         apklist = []
 179         gotcurrentver = False
 180         cantupdate = False
 181         buildfails = False
 182         for apk in apks:
 183             if apk['packageName'] == appid:
 184                 if str(apk['versionCode']) == app.CurrentVersionCode:
 185                     gotcurrentver = True
 186                 apklist.append(apk)
 187         # Include ones we can't build, as a special case...
 188         for build in app.builds:
 189             if build.disable:
 190                 if build.versionCode == app.CurrentVersionCode:
 191                     cantupdate = True
 192                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 193                 apklist.append({'versionCode': int(build.versionCode),
 194                                 'versionName': build.versionName,
 195                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 196                                 })
 197             else:
 198                 builtit = False
 199                 for apk in apklist:
 200                     if apk['versionCode'] == int(build.versionCode):
 201                         builtit = True
 202                         break
 203                 if not builtit:
 204                     buildfails = True
 205                     apklist.append({'versionCode': int(build.versionCode),
 206                                     'versionName': build.versionName,
 207                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 208                                     })
 209         if app.CurrentVersionCode == '0':
 210             cantupdate = True
 211         # Sort with most recent first...
 212         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 213
 214         wikidata += "=Versions=\n"
 215         if len(apklist) == 0:
 216             wikidata += "We currently have no versions of this app available."
 217         elif not gotcurrentver:
 218             wikidata += "We don't have the current version of this app."
 219         else:
 220             wikidata += "We have the current version of this app."
 221         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 222         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 223         if len(app.NoSourceSince) > 0:
 224             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 225         if len(app.CurrentVersion) > 0:
 226             wikidata += "The current (recommended) version is " + app.CurrentVersion
 227             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 228         validapks = 0
 229         for apk in apklist:
 230             wikidata += "==" + apk['versionName'] + "==\n"
 231
 232             if 'buildproblem' in apk:
 233                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 234             else:
 235                 validapks += 1
 236                 wikidata += "This version is built and signed by "
 237                 if 'srcname' in apk:
 238                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 239                 else:
 240                     wikidata += "the original developer.\n\n"
 241             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 242
 243         wikidata += '\n[[Category:' + wikicat + ']]\n'
 244         if len(app.NoSourceSince) > 0:
 245             wikidata += '\n[[Category:Apps missing source code]]\n'
 246         if validapks == 0 and not app.Disabled:
 247             wikidata += '\n[[Category:Apps with no packages]]\n'
 248         if cantupdate and not app.Disabled:
 249             wikidata += "\n[[Category:Apps we cannot update]]\n"
 250         if buildfails and not app.Disabled:
 251             wikidata += "\n[[Category:Apps with failing builds]]\n"
 252         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 253             wikidata += '\n[[Category:Apps to Update]]\n'
 254         if app.Disabled:
 255             wikidata += '\n[[Category:Apps that are disabled]]\n'
 256         if app.UpdateCheckMode == 'None' and not app.Disabled:
 257             wikidata += '\n[[Category:Apps with no update check]]\n'
 258         for appcat in app.Categories:
 259             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 260
 261         # We can't have underscores in the page name, even if they're in
 262         # the package ID, because MediaWiki messes with them...
 263         pagename = appid.replace('_', ' ')
 264
 265         # Drop a trailing newline, because mediawiki is going to drop it anyway
 266         # and it we don't we'll think the page has changed when it hasn't...
 267         if wikidata.endswith('\n'):
 268             wikidata = wikidata[:-1]
 269
 270         generated_pages[pagename] = wikidata
 271
 272         # Make a redirect from the name to the ID too, unless there's
 273         # already an existing page with the name and it isn't a redirect.
 274         noclobber = False
 275         apppagename = app.Name.replace('_', ' ')
 276         apppagename = apppagename.replace('{', '')
 277         apppagename = apppagename.replace('}', ' ')
 278         apppagename = apppagename.replace(':', ' ')
 279         apppagename = apppagename.replace('[', ' ')
 280         apppagename = apppagename.replace(']', ' ')
 281         # Drop double spaces caused mostly by replacing ':' above
 282         apppagename = apppagename.replace('  ', ' ')
 283         for expagename in site.allpages(prefix=apppagename,
 284                                         filterredir='nonredirects',
 285                                         generator=False):
 286             if expagename == apppagename:
 287                 noclobber = True
 288         # Another reason not to make the redirect page is if the app name
 289         # is the same as it's ID, because that will overwrite the real page
 290         # with an redirect to itself! (Although it seems like an odd
 291         # scenario this happens a lot, e.g. where there is metadata but no
 292         # builds or binaries to extract a name from.
 293         if apppagename == pagename:
 294             noclobber = True
 295         if not noclobber:
 296             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 297
 298     for tcat, genp in [(wikicat, generated_pages),
 299                        (wikiredircat, generated_redirects)]:
 300         catpages = site.Pages['Category:' + tcat]
 301         existingpages = []
 302         for page in catpages:
 303             existingpages.append(page.name)
 304             if page.name in genp:
 305                 pagetxt = page.edit()
 306                 if pagetxt != genp[page.name]:
 307                     logging.debug("Updating modified page " + page.name)
 308                     page.save(genp[page.name], summary='Auto-updated')
 309                 else:
 310                     logging.debug("Page " + page.name + " is unchanged")
 311             else:
 312                 logging.warn("Deleting page " + page.name)
 313                 page.delete('No longer published')
 314         for pagename, text in genp.items():
 315             logging.debug("Checking " + pagename)
 316             if pagename not in existingpages:
 317                 logging.debug("Creating page " + pagename)
 318                 try:
 319                     newpage = site.Pages[pagename]
 320                     newpage.save(text, summary='Auto-created')
 321                 except Exception as e:
 322                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 323
 324     # Purge server cache to ensure counts are up to date
 325     site.Pages['Repository Maintenance'].purge()
 326
 327     # Write a page with the last build log for this version code
 328     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 329     newpage = site.Pages[wiki_page_path]
 330     txt = ''
 331     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 332     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 333     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 334     txt += common.get_git_describe_link()
 335     txt += "\n\n"
 336     txt += common.get_android_tools_version_log()
 337     newpage.save(txt, summary='Run log')
 338     newpage = site.Pages['update']
 339     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 340
 341
 342 def delete_disabled_builds(apps, apkcache, repodirs):
 343     """Delete disabled build outputs.
 344
 345     :param apps: list of all applications, as per metadata.read_metadata
 346     :param apkcache: current apk cache information
 347     :param repodirs: the repo directories to process
 348     """
 349     for appid, app in apps.items():
 350         for build in app['builds']:
 351             if not build.disable:
 352                 continue
 353             apkfilename = common.get_release_filename(app, build)
 354             iconfilename = "%s.%s.png" % (
 355                 appid,
 356                 build.versionCode)
 357             for repodir in repodirs:
 358                 files = [
 359                     os.path.join(repodir, apkfilename),
 360                     os.path.join(repodir, apkfilename + '.asc'),
 361                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 362                 ]
 363                 for density in all_screen_densities:
 364                     repo_dir = get_icon_dir(repodir, density)
 365                     files.append(os.path.join(repo_dir, iconfilename))
 366
 367                 for f in files:
 368                     if os.path.exists(f):
 369                         logging.info("Deleting disabled build output " + f)
 370                         os.remove(f)
 371             if apkfilename in apkcache:
 372                 del apkcache[apkfilename]
 373
 374
 375 def resize_icon(iconpath, density):
 376
 377     if not os.path.isfile(iconpath):
 378         return
 379
 380     fp = None
 381     try:
 382         fp = open(iconpath, 'rb')
 383         im = Image.open(fp)
 384         size = dpi_to_px(density)
 385
 386         if any(length > size for length in im.size):
 387             oldsize = im.size
 388             im.thumbnail((size, size), Image.ANTIALIAS)
 389             logging.debug("%s was too large at %s - new size is %s" % (
 390                 iconpath, oldsize, im.size))
 391             im.save(iconpath, "PNG", optimize=True,
 392                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 393
 394     except Exception as e:
 395         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 396
 397     finally:
 398         if fp:
 399             fp.close()
 400
 401
 402 def resize_all_icons(repodirs):
 403     """Resize all icons that exceed the max size
 404
 405     :param repodirs: the repo directories to process
 406     """
 407     for repodir in repodirs:
 408         for density in screen_densities:
 409             icon_dir = get_icon_dir(repodir, density)
 410             icon_glob = os.path.join(icon_dir, '*.png')
 411             for iconpath in glob.glob(icon_glob):
 412                 resize_icon(iconpath, density)
 413
 414
 415 def getsig(apkpath):
 416     """ Get the signing certificate of an apk. To get the same md5 has that
 417     Android gets, we encode the .RSA certificate in a specific format and pass
 418     it hex-encoded to the md5 digest algorithm.
 419
 420     :param apkpath: path to the apk
 421     :returns: A string containing the md5 of the signature of the apk or None
 422               if an error occurred.
 423     """
 424
 425     with zipfile.ZipFile(apkpath, 'r') as apk:
 426         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 427
 428         if len(certs) < 1:
 429             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 430             return None
 431         if len(certs) > 1:
 432             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 433             return None
 434
 435         cert = apk.read(certs[0])
 436
 437     cert_encoded = common.get_certificate(cert)
 438
 439     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 440
 441
 442 def get_cache_file():
 443     return os.path.join('tmp', 'apkcache')
 444
 445
 446 def get_cache():
 447     """Get the cached dict of the APK index
 448
 449     Gather information about all the apk files in the repo directory,
 450     using cached data if possible. Some of the index operations take a
 451     long time, like calculating the SHA-256 and verifying the APK
 452     signature.
 453
 454     The cache is invalidated if the metadata version is different, or
 455     the 'allow_disabled_algorithms' config/option is different.  In
 456     those cases, there is no easy way to know what has changed from
 457     the cache, so just rerun the whole thing.
 458
 459     :return: apkcache
 460
 461     """
 462     apkcachefile = get_cache_file()
 463     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 464     if not options.clean and os.path.exists(apkcachefile):
 465         with open(apkcachefile, 'rb') as cf:
 466             apkcache = pickle.load(cf, encoding='utf-8')
 467         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 468            or apkcache.get('allow_disabled_algorithms') != ada:
 469             apkcache = {}
 470     else:
 471         apkcache = {}
 472
 473     apkcache["METADATA_VERSION"] = METADATA_VERSION
 474     apkcache['allow_disabled_algorithms'] = ada
 475
 476     return apkcache
 477
 478
 479 def write_cache(apkcache):
 480     apkcachefile = get_cache_file()
 481     cache_path = os.path.dirname(apkcachefile)
 482     if not os.path.exists(cache_path):
 483         os.makedirs(cache_path)
 484     with open(apkcachefile, 'wb') as cf:
 485         pickle.dump(apkcache, cf)
 486
 487
 488 def get_icon_bytes(apkzip, iconsrc):
 489     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 490     try:
 491         return apkzip.read(iconsrc)
 492     except KeyError:
 493         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 494
 495
 496 def sha256sum(filename):
 497     '''Calculate the sha256 of the given file'''
 498     sha = hashlib.sha256()
 499     with open(filename, 'rb') as f:
 500         while True:
 501             t = f.read(16384)
 502             if len(t) == 0:
 503                 break
 504             sha.update(t)
 505     return sha.hexdigest()
 506
 507
 508 def has_known_vulnerability(filename):
 509     """checks for known vulnerabilities in the APK
 510
 511     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 512     version.  Google also enforces this:
 513     https://support.google.com/faqs/answer/6376725?hl=en
 514
 515     Checks whether there are more than one classes.dex or AndroidManifest.xml
 516     files, which is invalid and an essential part of the "Master Key" attack.
 517     http://www.saurik.com/id/17
 518
 519     Janus is similar to Master Key but is perhaps easier to scan for.
 520     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 521     """
 522
 523     found_vuln = False
 524
 525     # statically load this pattern
 526     if not hasattr(has_known_vulnerability, "pattern"):
 527         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 528
 529     with open(filename.encode(), 'rb') as fp:
 530         first4 = fp.read(4)
 531     if first4 != b'\x50\x4b\x03\x04':
 532         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 533                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 534                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 535
 536     files_in_apk = set()
 537     with zipfile.ZipFile(filename) as zf:
 538         for name in zf.namelist():
 539             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 540                 lib = zf.open(name)
 541                 while True:
 542                     chunk = lib.read(4096)
 543                     if chunk == b'':
 544                         break
 545                     m = has_known_vulnerability.pattern.search(chunk)
 546                     if m:
 547                         version = m.group(1).decode('ascii')
 548                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 549                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 550                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 551                             logging.debug(_('"{path}" contains recent {name} ({version})')
 552                                           .format(path=filename, name=name, version=version))
 553                         else:
 554                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 555                                             .format(path=filename, name=name, version=version))
 556                             found_vuln = True
 557                         break
 558             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 559                 if name in files_in_apk:
 560                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 561                                     .format(apkfilename=filename, name=name))
 562                     found_vuln = True
 563                 files_in_apk.add(name)
 564     return found_vuln
 565
 566
 567 def insert_obbs(repodir, apps, apks):
 568     """Scans the .obb files in a given repo directory and adds them to the
 569     relevant APK instances.  OBB files have versionCodes like APK
 570     files, and they are loosely associated.  If there is an OBB file
 571     present, then any APK with the same or higher versionCode will use
 572     that OBB file.  There are two OBB types: main and patch, each APK
 573     can only have only have one of each.
 574
 575     https://developer.android.com/google/play/expansion-files.html
 576
 577     :param repodir: repo directory to scan
 578     :param apps: list of current, valid apps
 579     :param apks: current information on all APKs
 580
 581     """
 582
 583     def obbWarnDelete(f, msg):
 584         logging.warning(msg + ' ' + f)
 585         if options.delete_unknown:
 586             logging.error(_("Deleting unknown file: {path}").format(path=f))
 587             os.remove(f)
 588
 589     obbs = []
 590     java_Integer_MIN_VALUE = -pow(2, 31)
 591     currentPackageNames = apps.keys()
 592     for f in glob.glob(os.path.join(repodir, '*.obb')):
 593         obbfile = os.path.basename(f)
 594         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 595         chunks = obbfile.split('.')
 596         if chunks[0] != 'main' and chunks[0] != 'patch':
 597             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 598             continue
 599         if not re.match(r'^-?[0-9]+$', chunks[1]):
 600             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 601                           .format(name=chunks[0]))
 602             continue
 603         versionCode = int(chunks[1])
 604         packagename = ".".join(chunks[2:-1])
 605
 606         highestVersionCode = java_Integer_MIN_VALUE
 607         if packagename not in currentPackageNames:
 608             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 609             continue
 610         for apk in apks:
 611             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 612                 highestVersionCode = apk['versionCode']
 613         if versionCode > highestVersionCode:
 614             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 615                           .format(integer=str(versionCode)))
 616             continue
 617         obbsha256 = sha256sum(f)
 618         obbs.append((packagename, versionCode, obbfile, obbsha256))
 619
 620     for apk in apks:
 621         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 622             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 623                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 624                     apk['obbMainFile'] = obbfile
 625                     apk['obbMainFileSha256'] = obbsha256
 626                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 627                     apk['obbPatchFile'] = obbfile
 628                     apk['obbPatchFileSha256'] = obbsha256
 629             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 630                 break
 631
 632
 633 def translate_per_build_anti_features(apps, apks):
 634     """Grab the anti-features list from the build metadata
 635
 636     For most Anti-Features, they are really most applicable per-APK,
 637     not for an app.  An app can fix a vulnerability, add/remove
 638     tracking, etc.  This reads the 'antifeatures' list from the Build
 639     entries in the fdroiddata metadata file, then transforms it into
 640     the 'antiFeatures' list of unique items for the index.
 641
 642     The field key is all lower case in the metadata file to match the
 643     rest of the Build fields.  It is 'antiFeatures' camel case in the
 644     implementation, index, and fdroidclient since it is translated
 645     from the build 'antifeatures' field, not directly included.
 646
 647     """
 648
 649     antiFeatures = dict()
 650     for packageName, app in apps.items():
 651         d = dict()
 652         for build in app['builds']:
 653             afl = build.get('antifeatures')
 654             if afl:
 655                 d[int(build.versionCode)] = afl
 656         if len(d) > 0:
 657             antiFeatures[packageName] = d
 658
 659     for apk in apks:
 660         d = antiFeatures.get(apk['packageName'])
 661         if d:
 662             afl = d.get(apk['versionCode'])
 663             if afl:
 664                 apk['antiFeatures'].update(afl)
 665
 666
 667 def _get_localized_dict(app, locale):
 668     '''get the dict to add localized store metadata to'''
 669     if 'localized' not in app:
 670         app['localized'] = collections.OrderedDict()
 671     if locale not in app['localized']:
 672         app['localized'][locale] = collections.OrderedDict()
 673     return app['localized'][locale]
 674
 675
 676 def _set_localized_text_entry(app, locale, key, f):
 677     limit = config['char_limits'][key]
 678     localized = _get_localized_dict(app, locale)
 679     with open(f) as fp:
 680         text = fp.read()[:limit]
 681         if len(text) > 0:
 682             localized[key] = text
 683
 684
 685 def _set_author_entry(app, key, f):
 686     limit = config['char_limits']['author']
 687     with open(f) as fp:
 688         text = fp.read()[:limit]
 689         if len(text) > 0:
 690             app[key] = text
 691
 692
 693 def _strip_and_copy_image(inpath, outpath):
 694     """Remove any metadata from image and copy it to new path
 695
 696     Sadly, image metadata like EXIF can be used to exploit devices.
 697     It is not used at all in the F-Droid ecosystem, so its much safer
 698     just to remove it entirely.
 699
 700     """
 701
 702     extension = common.get_extension(inpath)[1]
 703     if os.path.isdir(outpath):
 704         outpath = os.path.join(outpath, os.path.basename(inpath))
 705     if extension == 'png':
 706         with open(inpath, 'rb') as fp:
 707             in_image = Image.open(fp)
 708             in_image.save(outpath, "PNG", optimize=True,
 709                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 710     elif extension == 'jpg' or extension == 'jpeg':
 711         with open(inpath, 'rb') as fp:
 712             in_image = Image.open(fp)
 713             data = list(in_image.getdata())
 714             out_image = Image.new(in_image.mode, in_image.size)
 715         out_image.putdata(data)
 716         out_image.save(outpath, "JPEG", optimize=True)
 717     else:
 718         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 719                               .format(extension=extension))
 720
 721
 722 def copy_triple_t_store_metadata(apps):
 723     """Include store metadata from the app's source repo
 724
 725     The Triple-T Gradle Play Publisher is a plugin that has a standard
 726     file layout for all of the metadata and graphics that the Google
 727     Play Store accepts.  Since F-Droid has the git repo, it can just
 728     pluck those files directly.  This method reads any text files into
 729     the app dict, then copies any graphics into the fdroid repo
 730     directory structure.
 731
 732     This needs to be run before insert_localized_app_metadata() so that
 733     the graphics files that are copied into the fdroid repo get
 734     properly indexed.
 735
 736     https://github.com/Triple-T/gradle-play-publisher#upload-images
 737     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 738
 739     """
 740
 741     if not os.path.isdir('build'):
 742         return  # nothing to do
 743
 744     for packageName, app in apps.items():
 745         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 746             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 747             for root, dirs, files in os.walk(d):
 748                 segments = root.split('/')
 749                 locale = segments[-2]
 750                 for f in files:
 751                     if f == 'fulldescription':
 752                         _set_localized_text_entry(app, locale, 'description',
 753                                                   os.path.join(root, f))
 754                         continue
 755                     elif f == 'shortdescription':
 756                         _set_localized_text_entry(app, locale, 'summary',
 757                                                   os.path.join(root, f))
 758                         continue
 759                     elif f == 'title':
 760                         _set_localized_text_entry(app, locale, 'name',
 761                                                   os.path.join(root, f))
 762                         continue
 763                     elif f == 'video':
 764                         _set_localized_text_entry(app, locale, 'video',
 765                                                   os.path.join(root, f))
 766                         continue
 767                     elif f == 'whatsnew':
 768                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 769                                                   os.path.join(root, f))
 770                         continue
 771                     elif f == 'contactEmail':
 772                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 773                         continue
 774                     elif f == 'contactPhone':
 775                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 776                         continue
 777                     elif f == 'contactWebsite':
 778                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 779                         continue
 780
 781                     base, extension = common.get_extension(f)
 782                     dirname = os.path.basename(root)
 783                     if extension in ALLOWED_EXTENSIONS \
 784                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 785                         if segments[-2] == 'listing':
 786                             locale = segments[-3]
 787                         else:
 788                             locale = segments[-2]
 789                         destdir = os.path.join('repo', packageName, locale, dirname)
 790                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 791                         sourcefile = os.path.join(root, f)
 792                         destfile = os.path.join(destdir, os.path.basename(f))
 793                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 794                         _strip_and_copy_image(sourcefile, destfile)
 795
 796
 797 def insert_localized_app_metadata(apps):
 798     """scans standard locations for graphics and localized text
 799
 800     Scans for localized description files, store graphics, and
 801     screenshot PNG files in statically defined screenshots directory
 802     and adds them to the app metadata.  The screenshots and graphic
 803     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 804     and must be in the following layout:
 805     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 806
 807     repo/packageName/locale/featureGraphic.png
 808     repo/packageName/locale/phoneScreenshots/1.png
 809     repo/packageName/locale/phoneScreenshots/2.png
 810
 811     The changelog files must be text files named with the versionCode
 812     ending with ".txt" and must be in the following layout:
 813     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 814
 815     repo/packageName/locale/changelogs/12345.txt
 816
 817     This will scan the each app's source repo then the metadata/ dir
 818     for these standard locations of changelog files.  If it finds
 819     them, they will be added to the dict of all packages, with the
 820     versions in the metadata/ folder taking precendence over the what
 821     is in the app's source repo.
 822
 823     Where "packageName" is the app's packageName and "locale" is the locale
 824     of the graphics, e.g. what language they are in, using the IETF RFC5646
 825     format (en-US, fr-CA, es-MX, etc).
 826
 827     This will also scan the app's git for a fastlane folder, and the
 828     metadata/ folder and the apps' source repos for standard locations
 829     of graphic and screenshot files.  If it finds them, it will copy
 830     them into the repo.  The fastlane files follow this pattern:
 831     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 832
 833     """
 834
 835     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 837     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 838     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 839
 840     for srcd in sorted(sourcedirs):
 841         if not os.path.isdir(srcd):
 842             continue
 843         for root, dirs, files in os.walk(srcd):
 844             segments = root.split('/')
 845             packageName = segments[1]
 846             if packageName not in apps:
 847                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 848                 continue
 849             locale = segments[-1]
 850             destdir = os.path.join('repo', packageName, locale)
 851
 852             # flavours specified in build receipt
 853             build_flavours = ""
 854             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 855                     and 'gradle' in apps[packageName].builds[-1]:
 856                 build_flavours = apps[packageName].builds[-1].gradle
 857
 858             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 859                 logging.debug("ignoring due to wrong flavour")
 860                 continue
 861
 862             for f in files:
 863                 if f in ('description.txt', 'full_description.txt'):
 864                     _set_localized_text_entry(apps[packageName], locale, 'description',
 865                                               os.path.join(root, f))
 866                     continue
 867                 elif f in ('summary.txt', 'short_description.txt'):
 868                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 869                                               os.path.join(root, f))
 870                     continue
 871                 elif f in ('name.txt', 'title.txt'):
 872                     _set_localized_text_entry(apps[packageName], locale, 'name',
 873                                               os.path.join(root, f))
 874                     continue
 875                 elif f == 'video.txt':
 876                     _set_localized_text_entry(apps[packageName], locale, 'video',
 877                                               os.path.join(root, f))
 878                     continue
 879                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 880                     locale = segments[-2]
 881                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 882                                               os.path.join(root, f))
 883                     continue
 884
 885                 base, extension = common.get_extension(f)
 886                 if locale == 'images':
 887                     locale = segments[-2]
 888                     destdir = os.path.join('repo', packageName, locale)
 889                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 890                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 891                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 892                     _strip_and_copy_image(os.path.join(root, f), destdir)
 893             for d in dirs:
 894                 if d in SCREENSHOT_DIRS:
 895                     if locale == 'images':
 896                         locale = segments[-2]
 897                         destdir = os.path.join('repo', packageName, locale)
 898                     for f in glob.glob(os.path.join(root, d, '*.*')):
 899                         _ignored, extension = common.get_extension(f)
 900                         if extension in ALLOWED_EXTENSIONS:
 901                             screenshotdestdir = os.path.join(destdir, d)
 902                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 903                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 904                             _strip_and_copy_image(f, screenshotdestdir)
 905
 906     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 907     for d in repofiles:
 908         if not os.path.isdir(d):
 909             continue
 910         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 911             if not os.path.isfile(f):
 912                 continue
 913             segments = f.split('/')
 914             packageName = segments[1]
 915             locale = segments[2]
 916             screenshotdir = segments[3]
 917             filename = os.path.basename(f)
 918             base, extension = common.get_extension(filename)
 919
 920             if packageName not in apps:
 921                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 922                                 .format(path=filename, name=packageName))
 923                 continue
 924             graphics = _get_localized_dict(apps[packageName], locale)
 925
 926             if extension not in ALLOWED_EXTENSIONS:
 927                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 928             elif base in GRAPHIC_NAMES:
 929                 # there can only be zero or one of these per locale
 930                 graphics[base] = filename
 931             elif screenshotdir in SCREENSHOT_DIRS:
 932                 # there can any number of these per locale
 933                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 934                 if screenshotdir not in graphics:
 935                     graphics[screenshotdir] = []
 936                 graphics[screenshotdir].append(filename)
 937             else:
 938                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 939
 940
 941 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 942     """Scan a repo for all files with an extension except APK/OBB
 943
 944     :param apkcache: current cached info about all repo files
 945     :param repodir: repo directory to scan
 946     :param knownapks: list of all known files, as per metadata.read_metadata
 947     :param use_date_from_file: use date from file (instead of current date)
 948                                for newly added files
 949     """
 950
 951     cachechanged = False
 952     repo_files = []
 953     repodir = repodir.encode('utf-8')
 954     for name in os.listdir(repodir):
 955         file_extension = common.get_file_extension(name)
 956         if file_extension == 'apk' or file_extension == 'obb':
 957             continue
 958         filename = os.path.join(repodir, name)
 959         name_utf8 = name.decode('utf-8')
 960         if filename.endswith(b'_src.tar.gz'):
 961             logging.debug(_('skipping source tarball: {path}')
 962                           .format(path=filename.decode('utf-8')))
 963             continue
 964         if not common.is_repo_file(filename):
 965             continue
 966         stat = os.stat(filename)
 967         if stat.st_size == 0:
 968             raise FDroidException(_('{path} is zero size!')
 969                                   .format(path=filename))
 970
 971         shasum = sha256sum(filename)
 972         usecache = False
 973         if name in apkcache:
 974             repo_file = apkcache[name]
 975             # added time is cached as tuple but used here as datetime instance
 976             if 'added' in repo_file:
 977                 a = repo_file['added']
 978                 if isinstance(a, datetime):
 979                     repo_file['added'] = a
 980                 else:
 981                     repo_file['added'] = datetime(*a[:6])
 982             if repo_file.get('hash') == shasum:
 983                 logging.debug(_("Reading {apkfilename} from cache")
 984                               .format(apkfilename=name_utf8))
 985                 usecache = True
 986             else:
 987                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 988                               .format(apkfilename=name_utf8))
 989
 990         if not usecache:
 991             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 992             repo_file = collections.OrderedDict()
 993             repo_file['name'] = os.path.splitext(name_utf8)[0]
 994             # TODO rename apkname globally to something more generic
 995             repo_file['apkName'] = name_utf8
 996             repo_file['hash'] = shasum
 997             repo_file['hashType'] = 'sha256'
 998             repo_file['versionCode'] = 0
 999             repo_file['versionName'] = shasum
1000             # the static ID is the SHA256 unless it is set in the metadata
1001             repo_file['packageName'] = shasum
1002
1003             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1004             if m:
1005                 repo_file['packageName'] = m.group(1)
1006                 repo_file['versionCode'] = int(m.group(2))
1007             srcfilename = name + b'_src.tar.gz'
1008             if os.path.exists(os.path.join(repodir, srcfilename)):
1009                 repo_file['srcname'] = srcfilename.decode('utf-8')
1010             repo_file['size'] = stat.st_size
1011
1012             apkcache[name] = repo_file
1013             cachechanged = True
1014
1015         if use_date_from_file:
1016             timestamp = stat.st_ctime
1017             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1018         else:
1019             default_date_param = None
1020
1021         # Record in knownapks, getting the added date at the same time..
1022         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1023                                     default_date=default_date_param)
1024         if added:
1025             repo_file['added'] = added
1026
1027         repo_files.append(repo_file)
1028
1029     return repo_files, cachechanged
1030
1031
1032 def scan_apk(apk_file):
1033     """
1034     Scans an APK file and returns dictionary with metadata of the APK.
1035
1036     Attention: This does *not* verify that the APK signature is correct.
1037
1038     :param apk_file: The (ideally absolute) path to the APK file
1039     :raises BuildException
1040     :return A dict containing APK metadata
1041     """
1042     apk = {
1043         'hash': sha256sum(apk_file),
1044         'hashType': 'sha256',
1045         'uses-permission': [],
1046         'uses-permission-sdk-23': [],
1047         'features': [],
1048         'icons_src': {},
1049         'icons': {},
1050         'antiFeatures': set(),
1051     }
1052
1053     if common.use_androguard():
1054         scan_apk_androguard(apk, apk_file)
1055     else:
1056         scan_apk_aapt(apk, apk_file)
1057
1058     # Get the signature, or rather the signing key fingerprints
1059     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1060     apk['sig'] = getsig(apk_file)
1061     if not apk['sig']:
1062         raise BuildException("Failed to get apk signature")
1063     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1064                                                                apk_file))
1065     if not apk.get('signer'):
1066         raise BuildException("Failed to get apk signing key fingerprint")
1067
1068     # Get size of the APK
1069     apk['size'] = os.path.getsize(apk_file)
1070
1071     if 'minSdkVersion' not in apk:
1072         logging.warning("No SDK version information found in {0}".format(apk_file))
1073         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
1074     if 'targetSdkVersion' not in apk:
1075         apk['targetSdkVersion'] = apk['minSdkVersion']
1076
1077     # Check for known vulnerabilities
1078     if has_known_vulnerability(apk_file):
1079         apk['antiFeatures'].add('KnownVuln')
1080
1081     return apk
1082
1083
1084 def _get_apk_icons_src(apkfile, icon_name):
1085     """Extract the paths to the app icon in all available densities
1086
1087     """
1088     icons_src = dict()
1089     density_re = re.compile('^res/(.*)/{}\.(png|xml)$'.format(icon_name))
1090     with zipfile.ZipFile(apkfile) as zf:
1091         for filename in zf.namelist():
1092             m = density_re.match(filename)
1093             if m:
1094                 folder = m.group(1).split('-')
1095                 if len(folder) > 1 and folder[1].endswith('dpi'):
1096                     density = screen_resolutions[folder[1]]
1097                 else:
1098                     density = '160'
1099                 icons_src[density] = m.group(0)
1100     if icons_src.get('-1') is None and '160' in icons_src:
1101         icons_src['-1'] = icons_src['160']
1102     return icons_src
1103
1104
1105 def scan_apk_aapt(apk, apkfile):
1106     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1107     if p.returncode != 0:
1108         if options.delete_unknown:
1109             if os.path.exists(apkfile):
1110                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1111                 os.remove(apkfile)
1112             else:
1113                 logging.error("Could not find {0} to remove it".format(apkfile))
1114         else:
1115             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1116         raise BuildException(_("Invalid APK"))
1117     icon_name = None
1118     for line in p.output.splitlines():
1119         if line.startswith("package:"):
1120             try:
1121                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1122                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1123                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1124             except Exception as e:
1125                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1126         elif line.startswith("application:"):
1127             m = re.match(APK_LABEL_ICON_PAT, line)
1128             if m:
1129                 apk['name'] = m.group(1)
1130                 icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
1131         elif not apk.get('name') and line.startswith("launchable-activity:"):
1132             # Only use launchable-activity as fallback to application
1133             apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)
1134         elif line.startswith("sdkVersion:"):
1135             m = re.match(APK_SDK_VERSION_PAT, line)
1136             if m is None:
1137                 logging.error(line.replace('sdkVersion:', '')
1138                               + ' is not a valid minSdkVersion!')
1139             else:
1140                 apk['minSdkVersion'] = m.group(1)
1141         elif line.startswith("targetSdkVersion:"):
1142             m = re.match(APK_SDK_VERSION_PAT, line)
1143             if m is None:
1144                 logging.error(line.replace('targetSdkVersion:', '')
1145                               + ' is not a valid targetSdkVersion!')
1146             else:
1147                 apk['targetSdkVersion'] = m.group(1)
1148         elif line.startswith("maxSdkVersion:"):
1149             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1150         elif line.startswith("native-code:"):
1151             apk['nativecode'] = []
1152             for arch in line[13:].split(' '):
1153                 apk['nativecode'].append(arch[1:-1])
1154         elif line.startswith('uses-permission:'):
1155             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1156             if perm_match['maxSdkVersion']:
1157                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1158             permission = UsesPermission(
1159                 perm_match['name'],
1160                 perm_match['maxSdkVersion']
1161             )
1162
1163             apk['uses-permission'].append(permission)
1164         elif line.startswith('uses-permission-sdk-23:'):
1165             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1166             if perm_match['maxSdkVersion']:
1167                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1168             permission_sdk_23 = UsesPermissionSdk23(
1169                 perm_match['name'],
1170                 perm_match['maxSdkVersion']
1171             )
1172
1173             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1174
1175         elif line.startswith('uses-feature:'):
1176             feature = re.match(APK_FEATURE_PAT, line).group(1)
1177             # Filter out this, it's only added with the latest SDK tools and
1178             # causes problems for lots of apps.
1179             if feature != "android.hardware.screen.portrait" \
1180                     and feature != "android.hardware.screen.landscape":
1181                 if feature.startswith("android.feature."):
1182                     feature = feature[16:]
1183                 apk['features'].add(feature)
1184     apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1185
1186
1187 def scan_apk_androguard(apk, apkfile):
1188     try:
1189         from androguard.core.bytecodes.apk import APK
1190         apkobject = APK(apkfile)
1191         if apkobject.is_valid_APK():
1192             arsc = apkobject.get_android_resources()
1193         else:
1194             if options.delete_unknown:
1195                 if os.path.exists(apkfile):
1196                     logging.error(_("Failed to get apk information, deleting {path}")
1197                                   .format(path=apkfile))
1198                     os.remove(apkfile)
1199                 else:
1200                     logging.error(_("Could not find {path} to remove it")
1201                                   .format(path=apkfile))
1202             else:
1203                 logging.error(_("Failed to get apk information, skipping {path}")
1204                               .format(path=apkfile))
1205             raise BuildException(_("Invalid APK"))
1206     except ImportError:
1207         raise FDroidException("androguard library is not installed and aapt not present")
1208     except FileNotFoundError:
1209         logging.error(_("Could not open apk file for analysis"))
1210         raise BuildException(_("Invalid APK"))
1211
1212     apk['packageName'] = apkobject.get_package()
1213     apk['versionCode'] = int(apkobject.get_androidversion_code())
1214     apk['versionName'] = apkobject.get_androidversion_name()
1215     if apk['versionName'][0] == "@":
1216         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1217         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1218         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1219     apk['name'] = apkobject.get_app_name()
1220
1221     if apkobject.get_max_sdk_version() is not None:
1222         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1223     if apkobject.get_min_sdk_version() is not None:
1224         apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1225     if apkobject.get_target_sdk_version() is not None:
1226         apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1227
1228     icon_id_str = apkobject.get_element("application", "icon")
1229     if icon_id_str:
1230         icon_id = int(icon_id_str.replace("@", "0x"), 16)
1231         resource_id = arsc.get_id(apk['packageName'], icon_id)
1232         if resource_id:
1233             icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1234         else:
1235             icon_name = os.path.splitext(os.path.basename(apkobject.get_app_icon()))[0]
1236         apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
1237
1238     arch_re = re.compile("^lib/(.*)/.*$")
1239     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1240     if len(arch) >= 1:
1241         apk['nativecode'] = []
1242         apk['nativecode'].extend(sorted(list(arch)))
1243
1244     xml = apkobject.get_android_manifest_xml()
1245     xmlns = xml.nsmap.get('android')
1246     if not xmlns:
1247         xmlns = 'http://schemas.android.com/apk/res/android'
1248
1249     for item in xml.findall('uses-permission'):
1250         name = str(item.attrib['{' + xmlns + '}name'])
1251         maxSdkVersion = item.attrib.get('{' + xmlns + '}maxSdkVersion')
1252         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1253         permission = UsesPermission(
1254             name,
1255             maxSdkVersion
1256         )
1257         apk['uses-permission'].append(permission)
1258     for name, maxSdkVersion in apkobject.get_uses_implied_permission_list():
1259         permission = UsesPermission(
1260             name,
1261             maxSdkVersion
1262         )
1263         apk['uses-permission'].append(permission)
1264
1265     for item in xml.findall('uses-permission-sdk-23'):
1266         name = str(item.attrib['{' + xmlns + '}name'])
1267         maxSdkVersion = item.attrib.get('{' + xmlns + '}maxSdkVersion')
1268         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1269         permission_sdk_23 = UsesPermissionSdk23(
1270             name,
1271             maxSdkVersion
1272         )
1273         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1274
1275     for item in xml.findall('uses-feature'):
1276         key = '{' + xmlns + '}name'
1277         if key not in item.attrib:
1278             continue
1279         feature = str(item.attrib[key])
1280         if feature != "android.hardware.screen.portrait" \
1281                 and feature != "android.hardware.screen.landscape":
1282             if feature.startswith("android.feature."):
1283                 feature = feature[16:]
1284         required = item.attrib.get('{' + xmlns + '}required')
1285         if required is None or required == 'true':
1286             apk['features'].append(feature)
1287
1288
1289 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1290                 allow_disabled_algorithms=False, archive_bad_sig=False):
1291     """Processes the apk with the given filename in the given repo directory.
1292
1293     This also extracts the icons.
1294
1295     :param apkcache: current apk cache information
1296     :param apkfilename: the filename of the apk to scan
1297     :param repodir: repo directory to scan
1298     :param knownapks: known apks info
1299     :param use_date_from_apk: use date from APK (instead of current date)
1300                               for newly added APKs
1301     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1302                                       disabled algorithms in the signature (e.g. MD5)
1303     :param archive_bad_sig: move APKs with a bad signature to the archive
1304     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1305      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1306     """
1307
1308     apk = {}
1309     apkfile = os.path.join(repodir, apkfilename)
1310
1311     cachechanged = False
1312     usecache = False
1313     if apkfilename in apkcache:
1314         apk = apkcache[apkfilename]
1315         if apk.get('hash') == sha256sum(apkfile):
1316             logging.debug(_("Reading {apkfilename} from cache")
1317                           .format(apkfilename=apkfilename))
1318             usecache = True
1319         else:
1320             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1321                           .format(apkfilename=apkfilename))
1322
1323     if not usecache:
1324         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1325
1326         try:
1327             apk = scan_apk(apkfile)
1328         except BuildException:
1329             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1330                             .format(apkfilename=apkfilename))
1331             return True, None, False
1332
1333         # Check for debuggable apks...
1334         if common.is_apk_and_debuggable(apkfile):
1335             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1336
1337         if options.rename_apks:
1338             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1339             std_short_name = os.path.join(repodir, n)
1340             if apkfile != std_short_name:
1341                 if os.path.exists(std_short_name):
1342                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1343                     if apkfile != std_long_name:
1344                         if os.path.exists(std_long_name):
1345                             dupdir = os.path.join('duplicates', repodir)
1346                             if not os.path.isdir(dupdir):
1347                                 os.makedirs(dupdir, exist_ok=True)
1348                             dupfile = os.path.join('duplicates', std_long_name)
1349                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1350                             os.rename(apkfile, dupfile)
1351                             return True, None, False
1352                         else:
1353                             os.rename(apkfile, std_long_name)
1354                     apkfile = std_long_name
1355                 else:
1356                     os.rename(apkfile, std_short_name)
1357                     apkfile = std_short_name
1358                 apkfilename = apkfile[len(repodir) + 1:]
1359
1360         apk['apkName'] = apkfilename
1361         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1362         if os.path.exists(os.path.join(repodir, srcfilename)):
1363             apk['srcname'] = srcfilename
1364
1365         # verify the jar signature is correct, allow deprecated
1366         # algorithms only if the APK is in the archive.
1367         skipapk = False
1368         if not common.verify_apk_signature(apkfile):
1369             if repodir == 'archive' or allow_disabled_algorithms:
1370                 if common.verify_old_apk_signature(apkfile):
1371                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1372                 else:
1373                     skipapk = True
1374             else:
1375                 skipapk = True
1376
1377         if skipapk:
1378             if archive_bad_sig:
1379                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1380                                 .format(apkfilename=apkfilename))
1381                 move_apk_between_sections(repodir, 'archive', apk)
1382             else:
1383                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1384                                 .format(apkfilename=apkfilename))
1385             return True, None, False
1386
1387         apkzip = zipfile.ZipFile(apkfile, 'r')
1388
1389         manifest = apkzip.getinfo('AndroidManifest.xml')
1390         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1391         if (1980, 0, 0) != manifest.date_time[0:3]:
1392             try:
1393                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1394             except ValueError as e:
1395                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1396                                 .format(apkfilename=apkfile) + str(e))
1397
1398         # extract icons from APK zip file
1399         iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
1400         try:
1401             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1402         finally:
1403             apkzip.close()  # ensure that APK zip file gets closed
1404
1405         # resize existing icons for densities missing in the APK
1406         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1407
1408         if use_date_from_apk and manifest.date_time[1] != 0:
1409             default_date_param = datetime(*manifest.date_time)
1410         else:
1411             default_date_param = None
1412
1413         # Record in known apks, getting the added date at the same time..
1414         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1415                                     default_date=default_date_param)
1416         if added:
1417             apk['added'] = added
1418
1419         apkcache[apkfilename] = apk
1420         cachechanged = True
1421
1422     return False, apk, cachechanged
1423
1424
1425 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1426     """Processes the apks in the given repo directory.
1427
1428     This also extracts the icons.
1429
1430     :param apkcache: current apk cache information
1431     :param repodir: repo directory to scan
1432     :param knownapks: known apks info
1433     :param use_date_from_apk: use date from APK (instead of current date)
1434                               for newly added APKs
1435     :returns: (apks, cachechanged) where apks is a list of apk information,
1436               and cachechanged is True if the apkcache got changed.
1437     """
1438
1439     cachechanged = False
1440
1441     for icon_dir in get_all_icon_dirs(repodir):
1442         if os.path.exists(icon_dir):
1443             if options.clean:
1444                 shutil.rmtree(icon_dir)
1445                 os.makedirs(icon_dir)
1446         else:
1447             os.makedirs(icon_dir)
1448
1449     apks = []
1450     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1451         apkfilename = apkfile[len(repodir) + 1:]
1452         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1453         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1454                                              use_date_from_apk, ada, True)
1455         if skip:
1456             continue
1457         apks.append(apk)
1458         cachechanged = cachechanged or cachethis
1459
1460     return apks, cachechanged
1461
1462
1463 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1464     """Extracts PNG icons from an APK with the supported pixel densities
1465
1466     Extracts icons from the given APK zip in various densities, saves
1467     them into given repo directory and stores their names in the APK
1468     metadata dictionary.  If the icon is an XML icon, then this tries
1469     to find PNG icon that can replace it.
1470
1471     :param icon_filename: A string representing the icon's file name
1472     :param apk: A populated dictionary containing APK metadata.
1473                 Needs to have 'icons_src' key
1474     :param apkzip: An opened zipfile.ZipFile of the APK file
1475     :param repo_dir: The directory of the APK's repository
1476     :return: A list of icon densities that are missing
1477
1478     """
1479     res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1480     pngs = dict()
1481     for f in apkzip.namelist():
1482         m = res_name_re.match(f)
1483         if m and m.group(4) == 'png':
1484             density = screen_resolutions[m.group(2)]
1485             pngs[m.group(3) + '/' + density] = m.group(0)
1486
1487     icon_type = None
1488     empty_densities = []
1489     for density in screen_densities:
1490         if density not in apk['icons_src']:
1491             empty_densities.append(density)
1492             continue
1493         icon_src = apk['icons_src'][density]
1494         icon_dir = get_icon_dir(repo_dir, density)
1495         icon_type = '.png'
1496
1497         # Extract the icon files per density
1498         if icon_src.endswith('.xml'):
1499             m = res_name_re.match(icon_src)
1500             if m:
1501                 name = pngs.get(m.group(3) + '/' + str(density))
1502                 if name:
1503                     icon_src = name
1504             if icon_src.endswith('.xml'):
1505                 empty_densities.append(density)
1506                 icon_type = '.xml'
1507         icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
1508
1509         try:
1510             with open(icon_dest, 'wb') as f:
1511                 f.write(get_icon_bytes(apkzip, icon_src))
1512             apk['icons'][density] = icon_filename + icon_type
1513         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1514             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1515             del apk['icons_src'][density]
1516             empty_densities.append(density)
1517
1518     # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
1519     if '-1' in apk['icons_src']:
1520         icon_src = apk['icons_src']['-1']
1521         icon_type = icon_src[-4:]
1522         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
1523         with open(icon_path, 'wb') as f:
1524             f.write(get_icon_bytes(apkzip, icon_src))
1525         if icon_type == '.png':
1526             im = None
1527             try:
1528                 im = Image.open(icon_path)
1529                 dpi = px_to_dpi(im.size[0])
1530                 for density in screen_densities:
1531                     if density in apk['icons']:
1532                         break
1533                     if density == screen_densities[-1] or dpi >= int(density):
1534                         apk['icons'][density] = icon_filename + icon_type
1535                         shutil.move(icon_path,
1536                                     os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
1537                         empty_densities.remove(density)
1538                         break
1539             except Exception as e:
1540                 logging.warning(_("Failed reading {path}: {error}")
1541                                 .format(path=icon_path, error=e))
1542             finally:
1543                 if im and hasattr(im, 'close'):
1544                     im.close()
1545
1546     if apk['icons']:
1547         apk['icon'] = icon_filename + icon_type
1548
1549     return empty_densities
1550
1551
1552 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1553     """
1554     Resize existing PNG icons for densities missing in the APK to ensure all densities are available
1555
1556     :param empty_densities: A list of icon densities that are missing
1557     :param icon_filename: A string representing the icon's file name
1558     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1559     :param repo_dir: The directory of the APK's repository
1560
1561     """
1562     icon_filename += '.png'
1563     # First try resizing down to not lose quality
1564     last_density = None
1565     for density in screen_densities:
1566         if density == '65534':  # not possible to generate 'anydpi' from other densities
1567             continue
1568         if density not in empty_densities:
1569             last_density = density
1570             continue
1571         if last_density is None:
1572             continue
1573         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1574
1575         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1576         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1577         fp = None
1578         try:
1579             fp = open(last_icon_path, 'rb')
1580             im = Image.open(fp)
1581
1582             size = dpi_to_px(density)
1583
1584             im.thumbnail((size, size), Image.ANTIALIAS)
1585             im.save(icon_path, "PNG", optimize=True,
1586                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1587             empty_densities.remove(density)
1588         except Exception as e:
1589             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1590         finally:
1591             if fp:
1592                 fp.close()
1593
1594     # Then just copy from the highest resolution available
1595     last_density = None
1596     for density in reversed(screen_densities):
1597         if density not in empty_densities:
1598             last_density = density
1599             continue
1600
1601         if last_density is None:
1602             continue
1603
1604         shutil.copyfile(
1605             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1606             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1607         )
1608         empty_densities.remove(density)
1609
1610     for density in screen_densities:
1611         icon_dir = get_icon_dir(repo_dir, density)
1612         icon_dest = os.path.join(icon_dir, icon_filename)
1613         resize_icon(icon_dest, density)
1614
1615     # Copy from icons-mdpi to icons since mdpi is the baseline density
1616     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1617     if os.path.isfile(baseline):
1618         apk['icons']['0'] = icon_filename
1619         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1620
1621
1622 def apply_info_from_latest_apk(apps, apks):
1623     """
1624     Some information from the apks needs to be applied up to the application level.
1625     When doing this, we use the info from the most recent version's apk.
1626     We deal with figuring out when the app was added and last updated at the same time.
1627     """
1628     for appid, app in apps.items():
1629         bestver = UNSET_VERSION_CODE
1630         for apk in apks:
1631             if apk['packageName'] == appid:
1632                 if apk['versionCode'] > bestver:
1633                     bestver = apk['versionCode']
1634                     bestapk = apk
1635
1636                 if 'added' in apk:
1637                     if not app.added or apk['added'] < app.added:
1638                         app.added = apk['added']
1639                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1640                         app.lastUpdated = apk['added']
1641
1642         if not app.added:
1643             logging.debug("Don't know when " + appid + " was added")
1644         if not app.lastUpdated:
1645             logging.debug("Don't know when " + appid + " was last updated")
1646
1647         if bestver == UNSET_VERSION_CODE:
1648
1649             if app.Name is None:
1650                 app.Name = app.AutoName or appid
1651             app.icon = None
1652             logging.debug("Application " + appid + " has no packages")
1653         else:
1654             if app.Name is None:
1655                 app.Name = bestapk['name']
1656             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1657             if app.CurrentVersionCode is None:
1658                 app.CurrentVersionCode = str(bestver)
1659
1660
1661 def make_categories_txt(repodir, categories):
1662     '''Write a category list in the repo to allow quick access'''
1663     catdata = ''
1664     for cat in sorted(categories):
1665         catdata += cat + '\n'
1666     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1667         f.write(catdata)
1668
1669
1670 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1671
1672     def filter_apk_list_sorted(apk_list):
1673         res = []
1674         for apk in apk_list:
1675             if apk['packageName'] == appid:
1676                 res.append(apk)
1677
1678         # Sort the apk list by version code. First is highest/newest.
1679         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1680
1681     for appid, app in apps.items():
1682
1683         if app.ArchivePolicy:
1684             keepversions = int(app.ArchivePolicy[:-9])
1685         else:
1686             keepversions = defaultkeepversions
1687
1688         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1689                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1690
1691         current_app_apks = filter_apk_list_sorted(apks)
1692         if len(current_app_apks) > keepversions:
1693             # Move back the ones we don't want.
1694             for apk in current_app_apks[keepversions:]:
1695                 move_apk_between_sections(repodir, archivedir, apk)
1696                 archapks.append(apk)
1697                 apks.remove(apk)
1698
1699         current_app_archapks = filter_apk_list_sorted(archapks)
1700         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1701             kept = 0
1702             # Move forward the ones we want again, except DisableAlgorithm
1703             for apk in current_app_archapks:
1704                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1705                     move_apk_between_sections(archivedir, repodir, apk)
1706                     archapks.remove(apk)
1707                     apks.append(apk)
1708                     kept += 1
1709                 if kept == keepversions:
1710                     break
1711
1712
1713 def move_apk_between_sections(from_dir, to_dir, apk):
1714     """move an APK from repo to archive or vice versa"""
1715
1716     def _move_file(from_dir, to_dir, filename, ignore_missing):
1717         from_path = os.path.join(from_dir, filename)
1718         if ignore_missing and not os.path.exists(from_path):
1719             return
1720         to_path = os.path.join(to_dir, filename)
1721         if not os.path.exists(to_dir):
1722             os.mkdir(to_dir)
1723         shutil.move(from_path, to_path)
1724
1725     if from_dir == to_dir:
1726         return
1727
1728     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1729     _move_file(from_dir, to_dir, apk['apkName'], False)
1730     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1731     for density in all_screen_densities:
1732         from_icon_dir = get_icon_dir(from_dir, density)
1733         to_icon_dir = get_icon_dir(to_dir, density)
1734         if density not in apk.get('icons', []):
1735             continue
1736         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1737     if 'srcname' in apk:
1738         _move_file(from_dir, to_dir, apk['srcname'], False)
1739
1740
1741 def add_apks_to_per_app_repos(repodir, apks):
1742     apks_per_app = dict()
1743     for apk in apks:
1744         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1745         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1746         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1747         apks_per_app[apk['packageName']] = apk
1748
1749         if not os.path.exists(apk['per_app_icons']):
1750             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1751             os.makedirs(apk['per_app_icons'])
1752
1753         apkpath = os.path.join(repodir, apk['apkName'])
1754         shutil.copy(apkpath, apk['per_app_repo'])
1755         apksigpath = apkpath + '.sig'
1756         if os.path.exists(apksigpath):
1757             shutil.copy(apksigpath, apk['per_app_repo'])
1758         apkascpath = apkpath + '.asc'
1759         if os.path.exists(apkascpath):
1760             shutil.copy(apkascpath, apk['per_app_repo'])
1761
1762
1763 def create_metadata_from_template(apk):
1764     '''create a new metadata file using internal or external template
1765
1766     Generate warnings for apk's with no metadata (or create skeleton
1767     metadata files, if requested on the command line).  Though the
1768     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1769     since those impose things on the metadata file made from the
1770     template: field sort order, empty field value, formatting, etc.
1771     '''
1772
1773     import yaml
1774     if os.path.exists('template.yml'):
1775         with open('template.yml') as f:
1776             metatxt = f.read()
1777         if 'name' in apk and apk['name'] != '':
1778             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1779                              r'\1 ' + apk['name'],
1780                              metatxt,
1781                              flags=re.IGNORECASE | re.MULTILINE)
1782         else:
1783             logging.warning(_('{appid} does not have a name! Using package name instead.')
1784                             .format(appid=apk['packageName']))
1785             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1786                              r'\1 ' + apk['packageName'],
1787                              metatxt,
1788                              flags=re.IGNORECASE | re.MULTILINE)
1789         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1790             f.write(metatxt)
1791     else:
1792         app = dict()
1793         app['Categories'] = [os.path.basename(os.getcwd())]
1794         # include some blanks as part of the template
1795         app['AuthorName'] = ''
1796         app['Summary'] = ''
1797         app['WebSite'] = ''
1798         app['IssueTracker'] = ''
1799         app['SourceCode'] = ''
1800         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1801         if 'name' in apk and apk['name'] != '':
1802             app['Name'] = apk['name']
1803         else:
1804             logging.warning(_('{appid} does not have a name! Using package name instead.')
1805                             .format(appid=apk['packageName']))
1806             app['Name'] = apk['packageName']
1807         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1808             yaml.dump(app, f, default_flow_style=False)
1809     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1810
1811
1812 config = None
1813 options = None
1814 start_timestamp = time.gmtime()
1815
1816
1817 def main():
1818
1819     global config, options
1820
1821     # Parse command line...
1822     parser = ArgumentParser()
1823     common.setup_global_opts(parser)
1824     parser.add_argument("--create-key", action="store_true", default=False,
1825                         help=_("Add a repo signing key to an unsigned repo"))
1826     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1827                         help=_("Add skeleton metadata files for APKs that are missing them"))
1828     parser.add_argument("--delete-unknown", action="store_true", default=False,
1829                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1830     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1831                         help=_("Report on build data status"))
1832     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1833                         help=_("Interactively ask about things that need updating."))
1834     parser.add_argument("-I", "--icons", action="store_true", default=False,
1835                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1836     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1837                         help=_("Specify editor to use in interactive mode. Default " +
1838                                "is {path}").format(path='/etc/alternatives/editor'))
1839     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1840                         help=_("Update the wiki"))
1841     parser.add_argument("--pretty", action="store_true", default=False,
1842                         help=_("Produce human-readable XML/JSON for index files"))
1843     parser.add_argument("--clean", action="store_true", default=False,
1844                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1845     parser.add_argument("--nosign", action="store_true", default=False,
1846                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1847     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1848                         help=_("Use date from APK instead of current time for newly added APKs"))
1849     parser.add_argument("--rename-apks", action="store_true", default=False,
1850                         help=_("Rename APK files that do not match package.name_123.apk"))
1851     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1852                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1853     metadata.add_metadata_arguments(parser)
1854     options = parser.parse_args()
1855     metadata.warnings_action = options.W
1856
1857     config = common.read_config(options)
1858
1859     if not ('jarsigner' in config and 'keytool' in config):
1860         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1861
1862     repodirs = ['repo']
1863     if config['archive_older'] != 0:
1864         repodirs.append('archive')
1865         if not os.path.exists('archive'):
1866             os.mkdir('archive')
1867
1868     if options.icons:
1869         resize_all_icons(repodirs)
1870         sys.exit(0)
1871
1872     if options.rename_apks:
1873         options.clean = True
1874
1875     # check that icons exist now, rather than fail at the end of `fdroid update`
1876     for k in ['repo_icon', 'archive_icon']:
1877         if k in config:
1878             if not os.path.exists(config[k]):
1879                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1880                                  .format(name=k, path=config[k]))
1881                 sys.exit(1)
1882
1883     # if the user asks to create a keystore, do it now, reusing whatever it can
1884     if options.create_key:
1885         if os.path.exists(config['keystore']):
1886             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1887             logging.critical("\t'" + config['keystore'] + "'")
1888             sys.exit(1)
1889
1890         if 'repo_keyalias' not in config:
1891             config['repo_keyalias'] = socket.getfqdn()
1892             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1893         if 'keydname' not in config:
1894             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1895             common.write_to_config(config, 'keydname', config['keydname'])
1896         if 'keystore' not in config:
1897             config['keystore'] = common.default_config['keystore']
1898             common.write_to_config(config, 'keystore', config['keystore'])
1899
1900         password = common.genpassword()
1901         if 'keystorepass' not in config:
1902             config['keystorepass'] = password
1903             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1904         if 'keypass' not in config:
1905             config['keypass'] = password
1906             common.write_to_config(config, 'keypass', config['keypass'])
1907         common.genkeystore(config)
1908
1909     # Get all apps...
1910     apps = metadata.read_metadata()
1911
1912     # Generate a list of categories...
1913     categories = set()
1914     for app in apps.values():
1915         categories.update(app.Categories)
1916
1917     # Read known apks data (will be updated and written back when we've finished)
1918     knownapks = common.KnownApks()
1919
1920     # Get APK cache
1921     apkcache = get_cache()
1922
1923     # Delete builds for disabled apps
1924     delete_disabled_builds(apps, apkcache, repodirs)
1925
1926     # Scan all apks in the main repo
1927     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1928
1929     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1930                                            options.use_date_from_apk)
1931     cachechanged = cachechanged or fcachechanged
1932     apks += files
1933     for apk in apks:
1934         if apk['packageName'] not in apps:
1935             if options.create_metadata:
1936                 create_metadata_from_template(apk)
1937                 apps = metadata.read_metadata()
1938             else:
1939                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1940                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1941                 if options.delete_unknown:
1942                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1943                                  .format(apkfilename=apk['apkName']))
1944                     rmf = os.path.join(repodirs[0], apk['apkName'])
1945                     if not os.path.exists(rmf):
1946                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1947                     else:
1948                         os.remove(rmf)
1949                 else:
1950                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1951
1952     copy_triple_t_store_metadata(apps)
1953     insert_obbs(repodirs[0], apps, apks)
1954     insert_localized_app_metadata(apps)
1955     translate_per_build_anti_features(apps, apks)
1956
1957     # Scan the archive repo for apks as well
1958     if len(repodirs) > 1:
1959         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1960         if cc:
1961             cachechanged = True
1962     else:
1963         archapks = []
1964
1965     # Apply information from latest apks to the application and update dates
1966     apply_info_from_latest_apk(apps, apks + archapks)
1967
1968     # Sort the app list by name, then the web site doesn't have to by default.
1969     # (we had to wait until we'd scanned the apks to do this, because mostly the
1970     # name comes from there!)
1971     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1972
1973     # APKs are placed into multiple repos based on the app package, providing
1974     # per-app subscription feeds for nightly builds and things like it
1975     if config['per_app_repos']:
1976         add_apks_to_per_app_repos(repodirs[0], apks)
1977         for appid, app in apps.items():
1978             repodir = os.path.join(appid, 'fdroid', 'repo')
1979             appdict = dict()
1980             appdict[appid] = app
1981             if os.path.isdir(repodir):
1982                 index.make(appdict, [appid], apks, repodir, False)
1983             else:
1984                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1985         return
1986
1987     if len(repodirs) > 1:
1988         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1989
1990     # Make the index for the main repo...
1991     index.make(apps, sortedids, apks, repodirs[0], False)
1992     make_categories_txt(repodirs[0], categories)
1993
1994     # If there's an archive repo,  make the index for it. We already scanned it
1995     # earlier on.
1996     if len(repodirs) > 1:
1997         index.make(apps, sortedids, archapks, repodirs[1], True)
1998
1999     git_remote = config.get('binary_transparency_remote')
2000     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
2001         from . import btlog
2002         btlog.make_binary_transparency_log(repodirs)
2003
2004     if config['update_stats']:
2005         # Update known apks info...
2006         knownapks.writeifchanged()
2007
2008         # Generate latest apps data for widget
2009         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
2010             data = ''
2011             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
2012                 for line in f:
2013                     appid = line.rstrip()
2014                     data += appid + "\t"
2015                     app = apps[appid]
2016                     data += app.Name + "\t"
2017                     if app.icon is not None:
2018                         data += app.icon + "\t"
2019                     data += app.License + "\n"
2020             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
2021                 f.write(data)
2022
2023     if cachechanged:
2024         write_cache(apkcache)
2025
2026     # Update the wiki...
2027     if options.wiki:
2028         update_wiki(apps, sortedids, apks + archapks)
2029
2030     logging.info(_("Finished"))
2031
2032
2033 if __name__ == "__main__":
2034     main()