fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import time
  32 from datetime import datetime
  33 from argparse import ArgumentParser
  34
  35 import collections
  36 from binascii import hexlify
  37
  38 from PIL import Image, PngImagePlugin
  39 import logging
  40
  41 from . import _
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46 from .exception import BuildException, FDroidException
  47
  48 METADATA_VERSION = 19
  49
  50 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  51 UNSET_VERSION_CODE = -0x100000000
  52
  53 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  54 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  55 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  56 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  57 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  58 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  59 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  60 APK_PERMISSION_PAT = \
  61     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  62 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  63
  64 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
  65 screen_resolutions = {
  66     "xxxhdpi": '640',
  67     "xxhdpi": '480',
  68     "xhdpi": '320',
  69     "hdpi": '240',
  70     "mdpi": '160',
  71     "ldpi": '120',
  72     "undefined": '-1',
  73     "anydpi": '65534',
  74     "nodpi": '65535'
  75 }
  76
  77 all_screen_densities = ['0'] + screen_densities
  78
  79 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  80 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  81
  82 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
  83 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
  84 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
  85                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
  86
  87 BLANK_PNG_INFO = PngImagePlugin.PngInfo()
  88
  89
  90 def dpi_to_px(density):
  91     return (int(density) * 48) / 160
  92
  93
  94 def px_to_dpi(px):
  95     return (int(px) * 160) / 48
  96
  97
  98 def get_icon_dir(repodir, density):
  99     if density == '0' or density == '65534':
 100         return os.path.join(repodir, "icons")
 101     else:
 102         return os.path.join(repodir, "icons-%s" % density)
 103
 104
 105 def get_icon_dirs(repodir):
 106     for density in screen_densities:
 107         yield get_icon_dir(repodir, density)
 108
 109
 110 def get_all_icon_dirs(repodir):
 111     for density in all_screen_densities:
 112         yield get_icon_dir(repodir, density)
 113
 114
 115 def update_wiki(apps, sortedids, apks):
 116     """Update the wiki
 117
 118     :param apps: fully populated list of all applications
 119     :param apks: all apks, except...
 120     """
 121     logging.info("Updating wiki")
 122     wikicat = 'Apps'
 123     wikiredircat = 'App Redirects'
 124     import mwclient
 125     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 126                          path=config['wiki_path'])
 127     site.login(config['wiki_user'], config['wiki_password'])
 128     generated_pages = {}
 129     generated_redirects = {}
 130
 131     for appid in sortedids:
 132         app = metadata.App(apps[appid])
 133
 134         wikidata = ''
 135         if app.Disabled:
 136             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 137         if app.AntiFeatures:
 138             for af in sorted(app.AntiFeatures):
 139                 wikidata += '{{AntiFeature|' + af + '}}\n'
 140         if app.RequiresRoot:
 141             requiresroot = 'Yes'
 142         else:
 143             requiresroot = 'No'
 144         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 145             appid,
 146             app.Name,
 147             app.added.strftime('%Y-%m-%d') if app.added else '',
 148             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 149             app.SourceCode,
 150             app.IssueTracker,
 151             app.WebSite,
 152             app.Changelog,
 153             app.Donate,
 154             app.FlattrID,
 155             app.LiberapayID,
 156             app.Bitcoin,
 157             app.Litecoin,
 158             app.License,
 159             requiresroot,
 160             app.AuthorName,
 161             app.AuthorEmail)
 162
 163         if app.Provides:
 164             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 165
 166         wikidata += app.Summary
 167         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 168
 169         wikidata += "=Description=\n"
 170         wikidata += metadata.description_wiki(app.Description) + "\n"
 171
 172         wikidata += "=Maintainer Notes=\n"
 173         if app.MaintainerNotes:
 174             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 175         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 176
 177         # Get a list of all packages for this application...
 178         apklist = []
 179         gotcurrentver = False
 180         cantupdate = False
 181         buildfails = False
 182         for apk in apks:
 183             if apk['packageName'] == appid:
 184                 if str(apk['versionCode']) == app.CurrentVersionCode:
 185                     gotcurrentver = True
 186                 apklist.append(apk)
 187         # Include ones we can't build, as a special case...
 188         for build in app.builds:
 189             if build.disable:
 190                 if build.versionCode == app.CurrentVersionCode:
 191                     cantupdate = True
 192                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 193                 apklist.append({'versionCode': int(build.versionCode),
 194                                 'versionName': build.versionName,
 195                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 196                                 })
 197             else:
 198                 builtit = False
 199                 for apk in apklist:
 200                     if apk['versionCode'] == int(build.versionCode):
 201                         builtit = True
 202                         break
 203                 if not builtit:
 204                     buildfails = True
 205                     apklist.append({'versionCode': int(build.versionCode),
 206                                     'versionName': build.versionName,
 207                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 208                                     })
 209         if app.CurrentVersionCode == '0':
 210             cantupdate = True
 211         # Sort with most recent first...
 212         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 213
 214         wikidata += "=Versions=\n"
 215         if len(apklist) == 0:
 216             wikidata += "We currently have no versions of this app available."
 217         elif not gotcurrentver:
 218             wikidata += "We don't have the current version of this app."
 219         else:
 220             wikidata += "We have the current version of this app."
 221         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 222         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 223         if len(app.NoSourceSince) > 0:
 224             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 225         if len(app.CurrentVersion) > 0:
 226             wikidata += "The current (recommended) version is " + app.CurrentVersion
 227             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 228         validapks = 0
 229         for apk in apklist:
 230             wikidata += "==" + apk['versionName'] + "==\n"
 231
 232             if 'buildproblem' in apk:
 233                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 234             else:
 235                 validapks += 1
 236                 wikidata += "This version is built and signed by "
 237                 if 'srcname' in apk:
 238                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 239                 else:
 240                     wikidata += "the original developer.\n\n"
 241             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 242
 243         wikidata += '\n[[Category:' + wikicat + ']]\n'
 244         if len(app.NoSourceSince) > 0:
 245             wikidata += '\n[[Category:Apps missing source code]]\n'
 246         if validapks == 0 and not app.Disabled:
 247             wikidata += '\n[[Category:Apps with no packages]]\n'
 248         if cantupdate and not app.Disabled:
 249             wikidata += "\n[[Category:Apps we cannot update]]\n"
 250         if buildfails and not app.Disabled:
 251             wikidata += "\n[[Category:Apps with failing builds]]\n"
 252         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 253             wikidata += '\n[[Category:Apps to Update]]\n'
 254         if app.Disabled:
 255             wikidata += '\n[[Category:Apps that are disabled]]\n'
 256         if app.UpdateCheckMode == 'None' and not app.Disabled:
 257             wikidata += '\n[[Category:Apps with no update check]]\n'
 258         for appcat in app.Categories:
 259             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 260
 261         # We can't have underscores in the page name, even if they're in
 262         # the package ID, because MediaWiki messes with them...
 263         pagename = appid.replace('_', ' ')
 264
 265         # Drop a trailing newline, because mediawiki is going to drop it anyway
 266         # and it we don't we'll think the page has changed when it hasn't...
 267         if wikidata.endswith('\n'):
 268             wikidata = wikidata[:-1]
 269
 270         generated_pages[pagename] = wikidata
 271
 272         # Make a redirect from the name to the ID too, unless there's
 273         # already an existing page with the name and it isn't a redirect.
 274         noclobber = False
 275         apppagename = app.Name.replace('_', ' ')
 276         apppagename = apppagename.replace('{', '')
 277         apppagename = apppagename.replace('}', ' ')
 278         apppagename = apppagename.replace(':', ' ')
 279         apppagename = apppagename.replace('[', ' ')
 280         apppagename = apppagename.replace(']', ' ')
 281         # Drop double spaces caused mostly by replacing ':' above
 282         apppagename = apppagename.replace('  ', ' ')
 283         for expagename in site.allpages(prefix=apppagename,
 284                                         filterredir='nonredirects',
 285                                         generator=False):
 286             if expagename == apppagename:
 287                 noclobber = True
 288         # Another reason not to make the redirect page is if the app name
 289         # is the same as it's ID, because that will overwrite the real page
 290         # with an redirect to itself! (Although it seems like an odd
 291         # scenario this happens a lot, e.g. where there is metadata but no
 292         # builds or binaries to extract a name from.
 293         if apppagename == pagename:
 294             noclobber = True
 295         if not noclobber:
 296             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 297
 298     for tcat, genp in [(wikicat, generated_pages),
 299                        (wikiredircat, generated_redirects)]:
 300         catpages = site.Pages['Category:' + tcat]
 301         existingpages = []
 302         for page in catpages:
 303             existingpages.append(page.name)
 304             if page.name in genp:
 305                 pagetxt = page.edit()
 306                 if pagetxt != genp[page.name]:
 307                     logging.debug("Updating modified page " + page.name)
 308                     page.save(genp[page.name], summary='Auto-updated')
 309                 else:
 310                     logging.debug("Page " + page.name + " is unchanged")
 311             else:
 312                 logging.warn("Deleting page " + page.name)
 313                 page.delete('No longer published')
 314         for pagename, text in genp.items():
 315             logging.debug("Checking " + pagename)
 316             if pagename not in existingpages:
 317                 logging.debug("Creating page " + pagename)
 318                 try:
 319                     newpage = site.Pages[pagename]
 320                     newpage.save(text, summary='Auto-created')
 321                 except Exception as e:
 322                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 323
 324     # Purge server cache to ensure counts are up to date
 325     site.Pages['Repository Maintenance'].purge()
 326
 327     # Write a page with the last build log for this version code
 328     wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
 329     newpage = site.Pages[wiki_page_path]
 330     txt = ''
 331     txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
 332     txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
 333     txt += "* completed at " + common.get_wiki_timestamp() + '\n'
 334     txt += "\n\n"
 335     txt += common.get_android_tools_version_log()
 336     newpage.save(txt, summary='Run log')
 337     newpage = site.Pages['update']
 338     newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 339
 340
 341 def delete_disabled_builds(apps, apkcache, repodirs):
 342     """Delete disabled build outputs.
 343
 344     :param apps: list of all applications, as per metadata.read_metadata
 345     :param apkcache: current apk cache information
 346     :param repodirs: the repo directories to process
 347     """
 348     for appid, app in apps.items():
 349         for build in app['builds']:
 350             if not build.disable:
 351                 continue
 352             apkfilename = common.get_release_filename(app, build)
 353             iconfilename = "%s.%s.png" % (
 354                 appid,
 355                 build.versionCode)
 356             for repodir in repodirs:
 357                 files = [
 358                     os.path.join(repodir, apkfilename),
 359                     os.path.join(repodir, apkfilename + '.asc'),
 360                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 361                 ]
 362                 for density in all_screen_densities:
 363                     repo_dir = get_icon_dir(repodir, density)
 364                     files.append(os.path.join(repo_dir, iconfilename))
 365
 366                 for f in files:
 367                     if os.path.exists(f):
 368                         logging.info("Deleting disabled build output " + f)
 369                         os.remove(f)
 370             if apkfilename in apkcache:
 371                 del apkcache[apkfilename]
 372
 373
 374 def resize_icon(iconpath, density):
 375
 376     if not os.path.isfile(iconpath):
 377         return
 378
 379     fp = None
 380     try:
 381         fp = open(iconpath, 'rb')
 382         im = Image.open(fp)
 383         size = dpi_to_px(density)
 384
 385         if any(length > size for length in im.size):
 386             oldsize = im.size
 387             im.thumbnail((size, size), Image.ANTIALIAS)
 388             logging.debug("%s was too large at %s - new size is %s" % (
 389                 iconpath, oldsize, im.size))
 390             im.save(iconpath, "PNG", optimize=True,
 391                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
 392
 393     except Exception as e:
 394         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 395
 396     finally:
 397         if fp:
 398             fp.close()
 399
 400
 401 def resize_all_icons(repodirs):
 402     """Resize all icons that exceed the max size
 403
 404     :param repodirs: the repo directories to process
 405     """
 406     for repodir in repodirs:
 407         for density in screen_densities:
 408             icon_dir = get_icon_dir(repodir, density)
 409             icon_glob = os.path.join(icon_dir, '*.png')
 410             for iconpath in glob.glob(icon_glob):
 411                 resize_icon(iconpath, density)
 412
 413
 414 def getsig(apkpath):
 415     """ Get the signing certificate of an apk. To get the same md5 has that
 416     Android gets, we encode the .RSA certificate in a specific format and pass
 417     it hex-encoded to the md5 digest algorithm.
 418
 419     :param apkpath: path to the apk
 420     :returns: A string containing the md5 of the signature of the apk or None
 421               if an error occurred.
 422     """
 423
 424     with zipfile.ZipFile(apkpath, 'r') as apk:
 425         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 426
 427         if len(certs) < 1:
 428             logging.error(_("No signing certificates found in {path}").format(path=apkpath))
 429             return None
 430         if len(certs) > 1:
 431             logging.error(_("Found multiple signing certificates in {path}").format(path=apkpath))
 432             return None
 433
 434         cert = apk.read(certs[0])
 435
 436     cert_encoded = common.get_certificate(cert)
 437
 438     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 439
 440
 441 def get_cache_file():
 442     return os.path.join('tmp', 'apkcache')
 443
 444
 445 def get_cache():
 446     """Get the cached dict of the APK index
 447
 448     Gather information about all the apk files in the repo directory,
 449     using cached data if possible. Some of the index operations take a
 450     long time, like calculating the SHA-256 and verifying the APK
 451     signature.
 452
 453     The cache is invalidated if the metadata version is different, or
 454     the 'allow_disabled_algorithms' config/option is different.  In
 455     those cases, there is no easy way to know what has changed from
 456     the cache, so just rerun the whole thing.
 457
 458     :return: apkcache
 459
 460     """
 461     apkcachefile = get_cache_file()
 462     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 463     if not options.clean and os.path.exists(apkcachefile):
 464         with open(apkcachefile, 'rb') as cf:
 465             apkcache = pickle.load(cf, encoding='utf-8')
 466         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
 467            or apkcache.get('allow_disabled_algorithms') != ada:
 468             apkcache = {}
 469     else:
 470         apkcache = {}
 471
 472     apkcache["METADATA_VERSION"] = METADATA_VERSION
 473     apkcache['allow_disabled_algorithms'] = ada
 474
 475     return apkcache
 476
 477
 478 def write_cache(apkcache):
 479     apkcachefile = get_cache_file()
 480     cache_path = os.path.dirname(apkcachefile)
 481     if not os.path.exists(cache_path):
 482         os.makedirs(cache_path)
 483     with open(apkcachefile, 'wb') as cf:
 484         pickle.dump(apkcache, cf)
 485
 486
 487 def get_icon_bytes(apkzip, iconsrc):
 488     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 489     try:
 490         return apkzip.read(iconsrc)
 491     except KeyError:
 492         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 493
 494
 495 def sha256sum(filename):
 496     '''Calculate the sha256 of the given file'''
 497     sha = hashlib.sha256()
 498     with open(filename, 'rb') as f:
 499         while True:
 500             t = f.read(16384)
 501             if len(t) == 0:
 502                 break
 503             sha.update(t)
 504     return sha.hexdigest()
 505
 506
 507 def has_known_vulnerability(filename):
 508     """checks for known vulnerabilities in the APK
 509
 510     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
 511     version.  Google also enforces this:
 512     https://support.google.com/faqs/answer/6376725?hl=en
 513
 514     Checks whether there are more than one classes.dex or AndroidManifest.xml
 515     files, which is invalid and an essential part of the "Master Key" attack.
 516     http://www.saurik.com/id/17
 517
 518     Janus is similar to Master Key but is perhaps easier to scan for.
 519     https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures
 520     """
 521
 522     found_vuln = False
 523
 524     # statically load this pattern
 525     if not hasattr(has_known_vulnerability, "pattern"):
 526         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 527
 528     with open(filename.encode(), 'rb') as fp:
 529         first4 = fp.read(4)
 530     if first4 != b'\x50\x4b\x03\x04':
 531         raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
 532                               .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
 533                               + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
 534
 535     files_in_apk = set()
 536     with zipfile.ZipFile(filename) as zf:
 537         for name in zf.namelist():
 538             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 539                 lib = zf.open(name)
 540                 while True:
 541                     chunk = lib.read(4096)
 542                     if chunk == b'':
 543                         break
 544                     m = has_known_vulnerability.pattern.search(chunk)
 545                     if m:
 546                         version = m.group(1).decode('ascii')
 547                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
 548                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
 549                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
 550                             logging.debug(_('"{path}" contains recent {name} ({version})')
 551                                           .format(path=filename, name=name, version=version))
 552                         else:
 553                             logging.warning(_('"{path}" contains outdated {name} ({version})')
 554                                             .format(path=filename, name=name, version=version))
 555                             found_vuln = True
 556                         break
 557             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
 558                 if name in files_in_apk:
 559                     logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
 560                                     .format(apkfilename=filename, name=name))
 561                     found_vuln = True
 562                 files_in_apk.add(name)
 563     return found_vuln
 564
 565
 566 def insert_obbs(repodir, apps, apks):
 567     """Scans the .obb files in a given repo directory and adds them to the
 568     relevant APK instances.  OBB files have versionCodes like APK
 569     files, and they are loosely associated.  If there is an OBB file
 570     present, then any APK with the same or higher versionCode will use
 571     that OBB file.  There are two OBB types: main and patch, each APK
 572     can only have only have one of each.
 573
 574     https://developer.android.com/google/play/expansion-files.html
 575
 576     :param repodir: repo directory to scan
 577     :param apps: list of current, valid apps
 578     :param apks: current information on all APKs
 579
 580     """
 581
 582     def obbWarnDelete(f, msg):
 583         logging.warning(msg + ' ' + f)
 584         if options.delete_unknown:
 585             logging.error(_("Deleting unknown file: {path}").format(path=f))
 586             os.remove(f)
 587
 588     obbs = []
 589     java_Integer_MIN_VALUE = -pow(2, 31)
 590     currentPackageNames = apps.keys()
 591     for f in glob.glob(os.path.join(repodir, '*.obb')):
 592         obbfile = os.path.basename(f)
 593         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 594         chunks = obbfile.split('.')
 595         if chunks[0] != 'main' and chunks[0] != 'patch':
 596             obbWarnDelete(f, _('OBB filename must start with "main." or "patch.":'))
 597             continue
 598         if not re.match(r'^-?[0-9]+$', chunks[1]):
 599             obbWarnDelete(f, _('The OBB version code must come after "{name}.":')
 600                           .format(name=chunks[0]))
 601             continue
 602         versionCode = int(chunks[1])
 603         packagename = ".".join(chunks[2:-1])
 604
 605         highestVersionCode = java_Integer_MIN_VALUE
 606         if packagename not in currentPackageNames:
 607             obbWarnDelete(f, _("OBB's packagename does not match a supported APK:"))
 608             continue
 609         for apk in apks:
 610             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 611                 highestVersionCode = apk['versionCode']
 612         if versionCode > highestVersionCode:
 613             obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
 614                           .format(integer=str(versionCode)))
 615             continue
 616         obbsha256 = sha256sum(f)
 617         obbs.append((packagename, versionCode, obbfile, obbsha256))
 618
 619     for apk in apks:
 620         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 621             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 622                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 623                     apk['obbMainFile'] = obbfile
 624                     apk['obbMainFileSha256'] = obbsha256
 625                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 626                     apk['obbPatchFile'] = obbfile
 627                     apk['obbPatchFileSha256'] = obbsha256
 628             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 629                 break
 630
 631
 632 def translate_per_build_anti_features(apps, apks):
 633     """Grab the anti-features list from the build metadata
 634
 635     For most Anti-Features, they are really most applicable per-APK,
 636     not for an app.  An app can fix a vulnerability, add/remove
 637     tracking, etc.  This reads the 'antifeatures' list from the Build
 638     entries in the fdroiddata metadata file, then transforms it into
 639     the 'antiFeatures' list of unique items for the index.
 640
 641     The field key is all lower case in the metadata file to match the
 642     rest of the Build fields.  It is 'antiFeatures' camel case in the
 643     implementation, index, and fdroidclient since it is translated
 644     from the build 'antifeatures' field, not directly included.
 645
 646     """
 647
 648     antiFeatures = dict()
 649     for packageName, app in apps.items():
 650         d = dict()
 651         for build in app['builds']:
 652             afl = build.get('antifeatures')
 653             if afl:
 654                 d[int(build.versionCode)] = afl
 655         if len(d) > 0:
 656             antiFeatures[packageName] = d
 657
 658     for apk in apks:
 659         d = antiFeatures.get(apk['packageName'])
 660         if d:
 661             afl = d.get(apk['versionCode'])
 662             if afl:
 663                 apk['antiFeatures'].update(afl)
 664
 665
 666 def _get_localized_dict(app, locale):
 667     '''get the dict to add localized store metadata to'''
 668     if 'localized' not in app:
 669         app['localized'] = collections.OrderedDict()
 670     if locale not in app['localized']:
 671         app['localized'][locale] = collections.OrderedDict()
 672     return app['localized'][locale]
 673
 674
 675 def _set_localized_text_entry(app, locale, key, f):
 676     limit = config['char_limits'][key]
 677     localized = _get_localized_dict(app, locale)
 678     with open(f) as fp:
 679         text = fp.read()[:limit]
 680         if len(text) > 0:
 681             localized[key] = text
 682
 683
 684 def _set_author_entry(app, key, f):
 685     limit = config['char_limits']['author']
 686     with open(f) as fp:
 687         text = fp.read()[:limit]
 688         if len(text) > 0:
 689             app[key] = text
 690
 691
 692 def _strip_and_copy_image(inpath, outpath):
 693     """Remove any metadata from image and copy it to new path
 694
 695     Sadly, image metadata like EXIF can be used to exploit devices.
 696     It is not used at all in the F-Droid ecosystem, so its much safer
 697     just to remove it entirely.
 698
 699     """
 700
 701     extension = common.get_extension(inpath)[1]
 702     if os.path.isdir(outpath):
 703         outpath = os.path.join(outpath, os.path.basename(inpath))
 704     if extension == 'png':
 705         with open(inpath, 'rb') as fp:
 706             in_image = Image.open(fp)
 707             in_image.save(outpath, "PNG", optimize=True,
 708                           pnginfo=BLANK_PNG_INFO, icc_profile=None)
 709     elif extension == 'jpg' or extension == 'jpeg':
 710         with open(inpath, 'rb') as fp:
 711             in_image = Image.open(fp)
 712             data = list(in_image.getdata())
 713             out_image = Image.new(in_image.mode, in_image.size)
 714         out_image.putdata(data)
 715         out_image.save(outpath, "JPEG", optimize=True)
 716     else:
 717         raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
 718                               .format(extension=extension))
 719
 720
 721 def copy_triple_t_store_metadata(apps):
 722     """Include store metadata from the app's source repo
 723
 724     The Triple-T Gradle Play Publisher is a plugin that has a standard
 725     file layout for all of the metadata and graphics that the Google
 726     Play Store accepts.  Since F-Droid has the git repo, it can just
 727     pluck those files directly.  This method reads any text files into
 728     the app dict, then copies any graphics into the fdroid repo
 729     directory structure.
 730
 731     This needs to be run before insert_localized_app_metadata() so that
 732     the graphics files that are copied into the fdroid repo get
 733     properly indexed.
 734
 735     https://github.com/Triple-T/gradle-play-publisher#upload-images
 736     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
 737
 738     """
 739
 740     if not os.path.isdir('build'):
 741         return  # nothing to do
 742
 743     for packageName, app in apps.items():
 744         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
 745             logging.debug('Triple-T Gradle Play Publisher: ' + d)
 746             for root, dirs, files in os.walk(d):
 747                 segments = root.split('/')
 748                 locale = segments[-2]
 749                 for f in files:
 750                     if f == 'fulldescription':
 751                         _set_localized_text_entry(app, locale, 'description',
 752                                                   os.path.join(root, f))
 753                         continue
 754                     elif f == 'shortdescription':
 755                         _set_localized_text_entry(app, locale, 'summary',
 756                                                   os.path.join(root, f))
 757                         continue
 758                     elif f == 'title':
 759                         _set_localized_text_entry(app, locale, 'name',
 760                                                   os.path.join(root, f))
 761                         continue
 762                     elif f == 'video':
 763                         _set_localized_text_entry(app, locale, 'video',
 764                                                   os.path.join(root, f))
 765                         continue
 766                     elif f == 'whatsnew':
 767                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
 768                                                   os.path.join(root, f))
 769                         continue
 770                     elif f == 'contactEmail':
 771                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
 772                         continue
 773                     elif f == 'contactPhone':
 774                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
 775                         continue
 776                     elif f == 'contactWebsite':
 777                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
 778                         continue
 779
 780                     base, extension = common.get_extension(f)
 781                     dirname = os.path.basename(root)
 782                     if extension in ALLOWED_EXTENSIONS \
 783                        and (dirname in GRAPHIC_NAMES or dirname in SCREENSHOT_DIRS):
 784                         if segments[-2] == 'listing':
 785                             locale = segments[-3]
 786                         else:
 787                             locale = segments[-2]
 788                         destdir = os.path.join('repo', packageName, locale, dirname)
 789                         os.makedirs(destdir, mode=0o755, exist_ok=True)
 790                         sourcefile = os.path.join(root, f)
 791                         destfile = os.path.join(destdir, os.path.basename(f))
 792                         logging.debug('copying ' + sourcefile + ' ' + destfile)
 793                         _strip_and_copy_image(sourcefile, destfile)
 794
 795
 796 def insert_localized_app_metadata(apps):
 797     """scans standard locations for graphics and localized text
 798
 799     Scans for localized description files, store graphics, and
 800     screenshot PNG files in statically defined screenshots directory
 801     and adds them to the app metadata.  The screenshots and graphic
 802     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 803     and must be in the following layout:
 804     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
 805
 806     repo/packageName/locale/featureGraphic.png
 807     repo/packageName/locale/phoneScreenshots/1.png
 808     repo/packageName/locale/phoneScreenshots/2.png
 809
 810     The changelog files must be text files named with the versionCode
 811     ending with ".txt" and must be in the following layout:
 812     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 813
 814     repo/packageName/locale/changelogs/12345.txt
 815
 816     This will scan the each app's source repo then the metadata/ dir
 817     for these standard locations of changelog files.  If it finds
 818     them, they will be added to the dict of all packages, with the
 819     versions in the metadata/ folder taking precendence over the what
 820     is in the app's source repo.
 821
 822     Where "packageName" is the app's packageName and "locale" is the locale
 823     of the graphics, e.g. what language they are in, using the IETF RFC5646
 824     format (en-US, fr-CA, es-MX, etc).
 825
 826     This will also scan the app's git for a fastlane folder, and the
 827     metadata/ folder and the apps' source repos for standard locations
 828     of graphic and screenshot files.  If it finds them, it will copy
 829     them into the repo.  The fastlane files follow this pattern:
 830     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 831
 832     """
 833
 834     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 835     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
 836     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
 837     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 838
 839     for srcd in sorted(sourcedirs):
 840         if not os.path.isdir(srcd):
 841             continue
 842         for root, dirs, files in os.walk(srcd):
 843             segments = root.split('/')
 844             packageName = segments[1]
 845             if packageName not in apps:
 846                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
 847                 continue
 848             locale = segments[-1]
 849             destdir = os.path.join('repo', packageName, locale)
 850
 851             # flavours specified in build receipt
 852             build_flavours = ""
 853             if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
 854                     and 'gradle' in apps[packageName].builds[-1]:
 855                 build_flavours = apps[packageName].builds[-1].gradle
 856
 857             if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
 858                 logging.debug("ignoring due to wrong flavour")
 859                 continue
 860
 861             for f in files:
 862                 if f in ('description.txt', 'full_description.txt'):
 863                     _set_localized_text_entry(apps[packageName], locale, 'description',
 864                                               os.path.join(root, f))
 865                     continue
 866                 elif f in ('summary.txt', 'short_description.txt'):
 867                     _set_localized_text_entry(apps[packageName], locale, 'summary',
 868                                               os.path.join(root, f))
 869                     continue
 870                 elif f in ('name.txt', 'title.txt'):
 871                     _set_localized_text_entry(apps[packageName], locale, 'name',
 872                                               os.path.join(root, f))
 873                     continue
 874                 elif f == 'video.txt':
 875                     _set_localized_text_entry(apps[packageName], locale, 'video',
 876                                               os.path.join(root, f))
 877                     continue
 878                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
 879                     locale = segments[-2]
 880                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
 881                                               os.path.join(root, f))
 882                     continue
 883
 884                 base, extension = common.get_extension(f)
 885                 if locale == 'images':
 886                     locale = segments[-2]
 887                     destdir = os.path.join('repo', packageName, locale)
 888                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
 889                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 890                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 891                     _strip_and_copy_image(os.path.join(root, f), destdir)
 892             for d in dirs:
 893                 if d in SCREENSHOT_DIRS:
 894                     if locale == 'images':
 895                         locale = segments[-2]
 896                         destdir = os.path.join('repo', packageName, locale)
 897                     for f in glob.glob(os.path.join(root, d, '*.*')):
 898                         _ignored, extension = common.get_extension(f)
 899                         if extension in ALLOWED_EXTENSIONS:
 900                             screenshotdestdir = os.path.join(destdir, d)
 901                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 902                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 903                             _strip_and_copy_image(f, screenshotdestdir)
 904
 905     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 906     for d in repofiles:
 907         if not os.path.isdir(d):
 908             continue
 909         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 910             if not os.path.isfile(f):
 911                 continue
 912             segments = f.split('/')
 913             packageName = segments[1]
 914             locale = segments[2]
 915             screenshotdir = segments[3]
 916             filename = os.path.basename(f)
 917             base, extension = common.get_extension(filename)
 918
 919             if packageName not in apps:
 920                 logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!')
 921                                 .format(path=filename, name=packageName))
 922                 continue
 923             graphics = _get_localized_dict(apps[packageName], locale)
 924
 925             if extension not in ALLOWED_EXTENSIONS:
 926                 logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f))
 927             elif base in GRAPHIC_NAMES:
 928                 # there can only be zero or one of these per locale
 929                 graphics[base] = filename
 930             elif screenshotdir in SCREENSHOT_DIRS:
 931                 # there can any number of these per locale
 932                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
 933                 if screenshotdir not in graphics:
 934                     graphics[screenshotdir] = []
 935                 graphics[screenshotdir].append(filename)
 936             else:
 937                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 938
 939
 940 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 941     """Scan a repo for all files with an extension except APK/OBB
 942
 943     :param apkcache: current cached info about all repo files
 944     :param repodir: repo directory to scan
 945     :param knownapks: list of all known files, as per metadata.read_metadata
 946     :param use_date_from_file: use date from file (instead of current date)
 947                                for newly added files
 948     """
 949
 950     cachechanged = False
 951     repo_files = []
 952     repodir = repodir.encode('utf-8')
 953     for name in os.listdir(repodir):
 954         file_extension = common.get_file_extension(name)
 955         if file_extension == 'apk' or file_extension == 'obb':
 956             continue
 957         filename = os.path.join(repodir, name)
 958         name_utf8 = name.decode('utf-8')
 959         if filename.endswith(b'_src.tar.gz'):
 960             logging.debug(_('skipping source tarball: {path}')
 961                           .format(path=filename.decode('utf-8')))
 962             continue
 963         if not common.is_repo_file(filename):
 964             continue
 965         stat = os.stat(filename)
 966         if stat.st_size == 0:
 967             raise FDroidException(_('{path} is zero size!')
 968                                   .format(path=filename))
 969
 970         shasum = sha256sum(filename)
 971         usecache = False
 972         if name in apkcache:
 973             repo_file = apkcache[name]
 974             # added time is cached as tuple but used here as datetime instance
 975             if 'added' in repo_file:
 976                 a = repo_file['added']
 977                 if isinstance(a, datetime):
 978                     repo_file['added'] = a
 979                 else:
 980                     repo_file['added'] = datetime(*a[:6])
 981             if repo_file.get('hash') == shasum:
 982                 logging.debug(_("Reading {apkfilename} from cache")
 983                               .format(apkfilename=name_utf8))
 984                 usecache = True
 985             else:
 986                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
 987                               .format(apkfilename=name_utf8))
 988
 989         if not usecache:
 990             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
 991             repo_file = collections.OrderedDict()
 992             repo_file['name'] = os.path.splitext(name_utf8)[0]
 993             # TODO rename apkname globally to something more generic
 994             repo_file['apkName'] = name_utf8
 995             repo_file['hash'] = shasum
 996             repo_file['hashType'] = 'sha256'
 997             repo_file['versionCode'] = 0
 998             repo_file['versionName'] = shasum
 999             # the static ID is the SHA256 unless it is set in the metadata
1000             repo_file['packageName'] = shasum
1001
1002             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
1003             if m:
1004                 repo_file['packageName'] = m.group(1)
1005                 repo_file['versionCode'] = int(m.group(2))
1006             srcfilename = name + b'_src.tar.gz'
1007             if os.path.exists(os.path.join(repodir, srcfilename)):
1008                 repo_file['srcname'] = srcfilename.decode('utf-8')
1009             repo_file['size'] = stat.st_size
1010
1011             apkcache[name] = repo_file
1012             cachechanged = True
1013
1014         if use_date_from_file:
1015             timestamp = stat.st_ctime
1016             default_date_param = time.gmtime(time.mktime(datetime.fromtimestamp(timestamp).timetuple()))
1017         else:
1018             default_date_param = None
1019
1020         # Record in knownapks, getting the added date at the same time..
1021         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
1022                                     default_date=default_date_param)
1023         if added:
1024             repo_file['added'] = added
1025
1026         repo_files.append(repo_file)
1027
1028     return repo_files, cachechanged
1029
1030
1031 def scan_apk(apk_file):
1032     """
1033     Scans an APK file and returns dictionary with metadata of the APK.
1034
1035     Attention: This does *not* verify that the APK signature is correct.
1036
1037     :param apk_file: The (ideally absolute) path to the APK file
1038     :raises BuildException
1039     :return A dict containing APK metadata
1040     """
1041     apk = {
1042         'hash': sha256sum(apk_file),
1043         'hashType': 'sha256',
1044         'uses-permission': [],
1045         'uses-permission-sdk-23': [],
1046         'features': [],
1047         'icons_src': {},
1048         'icons': {},
1049         'antiFeatures': set(),
1050     }
1051
1052     try:
1053         import androguard
1054         androguard  # silence pyflakes
1055         scan_apk_androguard(apk, apk_file)
1056     except ImportError:
1057         scan_apk_aapt(apk, apk_file)
1058
1059     # Get the signature, or rather the signing key fingerprints
1060     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
1061     apk['sig'] = getsig(apk_file)
1062     if not apk['sig']:
1063         raise BuildException("Failed to get apk signature")
1064     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
1065                                                                apk_file))
1066     if not apk.get('signer'):
1067         raise BuildException("Failed to get apk signing key fingerprint")
1068
1069     # Get size of the APK
1070     apk['size'] = os.path.getsize(apk_file)
1071
1072     if 'minSdkVersion' not in apk:
1073         logging.warning("No SDK version information found in {0}".format(apk_file))
1074         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
1075     if 'targetSdkVersion' not in apk:
1076         apk['targetSdkVersion'] = apk['minSdkVersion']
1077
1078     # Check for known vulnerabilities
1079     if has_known_vulnerability(apk_file):
1080         apk['antiFeatures'].add('KnownVuln')
1081
1082     return apk
1083
1084
1085 def scan_apk_aapt(apk, apkfile):
1086     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1087     if p.returncode != 0:
1088         if options.delete_unknown:
1089             if os.path.exists(apkfile):
1090                 logging.error(_("Failed to get apk information, deleting {path}").format(path=apkfile))
1091                 os.remove(apkfile)
1092             else:
1093                 logging.error("Could not find {0} to remove it".format(apkfile))
1094         else:
1095             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
1096         raise BuildException(_("Invalid APK"))
1097     for line in p.output.splitlines():
1098         if line.startswith("package:"):
1099             try:
1100                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1101                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1102                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1103             except Exception as e:
1104                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1105         elif line.startswith("application:"):
1106             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1107             # Keep path to non-dpi icon in case we need it
1108             match = re.match(APK_ICON_PAT_NODPI, line)
1109             if match:
1110                 apk['icons_src']['-1'] = match.group(1)
1111         elif line.startswith("launchable-activity:"):
1112             # Only use launchable-activity as fallback to application
1113             if not apk['name']:
1114                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1115             if '-1' not in apk['icons_src']:
1116                 match = re.match(APK_ICON_PAT_NODPI, line)
1117                 if match:
1118                     apk['icons_src']['-1'] = match.group(1)
1119         elif line.startswith("application-icon-"):
1120             match = re.match(APK_ICON_PAT, line)
1121             if match:
1122                 density = match.group(1)
1123                 path = match.group(2)
1124                 apk['icons_src'][density] = path
1125         elif line.startswith("sdkVersion:"):
1126             m = re.match(APK_SDK_VERSION_PAT, line)
1127             if m is None:
1128                 logging.error(line.replace('sdkVersion:', '')
1129                               + ' is not a valid minSdkVersion!')
1130             else:
1131                 apk['minSdkVersion'] = m.group(1)
1132         elif line.startswith("targetSdkVersion:"):
1133             m = re.match(APK_SDK_VERSION_PAT, line)
1134             if m is None:
1135                 logging.error(line.replace('targetSdkVersion:', '')
1136                               + ' is not a valid targetSdkVersion!')
1137             else:
1138                 apk['targetSdkVersion'] = m.group(1)
1139         elif line.startswith("maxSdkVersion:"):
1140             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1141         elif line.startswith("native-code:"):
1142             apk['nativecode'] = []
1143             for arch in line[13:].split(' '):
1144                 apk['nativecode'].append(arch[1:-1])
1145         elif line.startswith('uses-permission:'):
1146             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1147             if perm_match['maxSdkVersion']:
1148                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1149             permission = UsesPermission(
1150                 perm_match['name'],
1151                 perm_match['maxSdkVersion']
1152             )
1153
1154             apk['uses-permission'].append(permission)
1155         elif line.startswith('uses-permission-sdk-23:'):
1156             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1157             if perm_match['maxSdkVersion']:
1158                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1159             permission_sdk_23 = UsesPermissionSdk23(
1160                 perm_match['name'],
1161                 perm_match['maxSdkVersion']
1162             )
1163
1164             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1165
1166         elif line.startswith('uses-feature:'):
1167             feature = re.match(APK_FEATURE_PAT, line).group(1)
1168             # Filter out this, it's only added with the latest SDK tools and
1169             # causes problems for lots of apps.
1170             if feature != "android.hardware.screen.portrait" \
1171                     and feature != "android.hardware.screen.landscape":
1172                 if feature.startswith("android.feature."):
1173                     feature = feature[16:]
1174                 apk['features'].add(feature)
1175
1176
1177 def scan_apk_androguard(apk, apkfile):
1178     try:
1179         from androguard.core.bytecodes.apk import APK
1180         apkobject = APK(apkfile)
1181         if apkobject.is_valid_APK():
1182             arsc = apkobject.get_android_resources()
1183         else:
1184             if options.delete_unknown:
1185                 if os.path.exists(apkfile):
1186                     logging.error(_("Failed to get apk information, deleting {path}")
1187                                   .format(path=apkfile))
1188                     os.remove(apkfile)
1189                 else:
1190                     logging.error(_("Could not find {path} to remove it")
1191                                   .format(path=apkfile))
1192             else:
1193                 logging.error(_("Failed to get apk information, skipping {path}")
1194                               .format(path=apkfile))
1195             raise BuildException(_("Invalid APK"))
1196     except ImportError:
1197         raise FDroidException("androguard library is not installed and aapt not present")
1198     except FileNotFoundError:
1199         logging.error(_("Could not open apk file for analysis"))
1200         raise BuildException(_("Invalid APK"))
1201
1202     apk['packageName'] = apkobject.get_package()
1203     apk['versionCode'] = int(apkobject.get_androidversion_code())
1204     apk['versionName'] = apkobject.get_androidversion_name()
1205     if apk['versionName'][0] == "@":
1206         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1207         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1208         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1209     apk['name'] = apkobject.get_app_name()
1210
1211     if apkobject.get_max_sdk_version() is not None:
1212         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1213     if apkobject.get_min_sdk_version() is not None:
1214         apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1215     if apkobject.get_target_sdk_version() is not None:
1216         apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1217
1218     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1219     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1220
1221     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1222
1223     for file in apkobject.get_files():
1224         d_re = density_re.match(file)
1225         if d_re:
1226             folder = d_re.group(1).split('-')
1227             if len(folder) > 1:
1228                 resolution = folder[1]
1229             else:
1230                 resolution = 'mdpi'
1231             density = screen_resolutions[resolution]
1232             apk['icons_src'][density] = d_re.group(0)
1233
1234     if apk['icons_src'].get('-1') is None:
1235         apk['icons_src']['-1'] = apk['icons_src']['160']
1236
1237     arch_re = re.compile("^lib/(.*)/.*$")
1238     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1239     if len(arch) >= 1:
1240         apk['nativecode'] = []
1241         apk['nativecode'].extend(sorted(list(arch)))
1242
1243     xml = apkobject.get_android_manifest_xml()
1244
1245     for item in xml.findall('uses-permission'):
1246         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1247         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1248         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1249         permission = UsesPermission(
1250             name,
1251             maxSdkVersion
1252         )
1253         apk['uses-permission'].append(permission)
1254
1255     for item in xml.findall('uses-permission-sdk-23'):
1256         name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1257         maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
1258         maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
1259         permission_sdk_23 = UsesPermissionSdk23(
1260             name,
1261             maxSdkVersion
1262         )
1263         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1264
1265     for item in xml.findall('uses-feature'):
1266         feature = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
1267         if feature != "android.hardware.screen.portrait" \
1268                 and feature != "android.hardware.screen.landscape":
1269             if feature.startswith("android.feature."):
1270                 feature = feature[16:]
1271         apk['features'].append(feature)
1272
1273
1274 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1275                 allow_disabled_algorithms=False, archive_bad_sig=False):
1276     """Processes the apk with the given filename in the given repo directory.
1277
1278     This also extracts the icons.
1279
1280     :param apkcache: current apk cache information
1281     :param apkfilename: the filename of the apk to scan
1282     :param repodir: repo directory to scan
1283     :param knownapks: known apks info
1284     :param use_date_from_apk: use date from APK (instead of current date)
1285                               for newly added APKs
1286     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1287                                       disabled algorithms in the signature (e.g. MD5)
1288     :param archive_bad_sig: move APKs with a bad signature to the archive
1289     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1290      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1291     """
1292
1293     apk = {}
1294     apkfile = os.path.join(repodir, apkfilename)
1295
1296     cachechanged = False
1297     usecache = False
1298     if apkfilename in apkcache:
1299         apk = apkcache[apkfilename]
1300         if apk.get('hash') == sha256sum(apkfile):
1301             logging.debug(_("Reading {apkfilename} from cache")
1302                           .format(apkfilename=apkfilename))
1303             usecache = True
1304         else:
1305             logging.debug(_("Ignoring stale cache data for {apkfilename}")
1306                           .format(apkfilename=apkfilename))
1307
1308     if not usecache:
1309         logging.debug(_("Processing {apkfilename}").format(apkfilename=apkfilename))
1310
1311         try:
1312             apk = scan_apk(apkfile)
1313         except BuildException:
1314             logging.warning(_("Skipping '{apkfilename}' with invalid signature!")
1315                             .format(apkfilename=apkfilename))
1316             return True, None, False
1317
1318         # Check for debuggable apks...
1319         if common.isApkAndDebuggable(apkfile):
1320             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1321
1322         if options.rename_apks:
1323             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1324             std_short_name = os.path.join(repodir, n)
1325             if apkfile != std_short_name:
1326                 if os.path.exists(std_short_name):
1327                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1328                     if apkfile != std_long_name:
1329                         if os.path.exists(std_long_name):
1330                             dupdir = os.path.join('duplicates', repodir)
1331                             if not os.path.isdir(dupdir):
1332                                 os.makedirs(dupdir, exist_ok=True)
1333                             dupfile = os.path.join('duplicates', std_long_name)
1334                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1335                             os.rename(apkfile, dupfile)
1336                             return True, None, False
1337                         else:
1338                             os.rename(apkfile, std_long_name)
1339                     apkfile = std_long_name
1340                 else:
1341                     os.rename(apkfile, std_short_name)
1342                     apkfile = std_short_name
1343                 apkfilename = apkfile[len(repodir) + 1:]
1344
1345         apk['apkName'] = apkfilename
1346         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1347         if os.path.exists(os.path.join(repodir, srcfilename)):
1348             apk['srcname'] = srcfilename
1349
1350         # verify the jar signature is correct, allow deprecated
1351         # algorithms only if the APK is in the archive.
1352         skipapk = False
1353         if not common.verify_apk_signature(apkfile):
1354             if repodir == 'archive' or allow_disabled_algorithms:
1355                 if common.verify_old_apk_signature(apkfile):
1356                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1357                 else:
1358                     skipapk = True
1359             else:
1360                 skipapk = True
1361
1362         if skipapk:
1363             if archive_bad_sig:
1364                 logging.warning(_('Archiving {apkfilename} with invalid signature!')
1365                                 .format(apkfilename=apkfilename))
1366                 move_apk_between_sections(repodir, 'archive', apk)
1367             else:
1368                 logging.warning(_('Skipping {apkfilename} with invalid signature!')
1369                                 .format(apkfilename=apkfilename))
1370             return True, None, False
1371
1372         apkzip = zipfile.ZipFile(apkfile, 'r')
1373
1374         manifest = apkzip.getinfo('AndroidManifest.xml')
1375         # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
1376         if (1980, 0, 0) != manifest.date_time[0:3]:
1377             try:
1378                 common.check_system_clock(datetime(*manifest.date_time), apkfilename)
1379             except ValueError as e:
1380                 logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
1381                                 .format(apkfilename=apkfile) + str(e))
1382
1383         # extract icons from APK zip file
1384         iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
1385         try:
1386             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1387         finally:
1388             apkzip.close()  # ensure that APK zip file gets closed
1389
1390         # resize existing icons for densities missing in the APK
1391         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1392
1393         if use_date_from_apk and manifest.date_time[1] != 0:
1394             default_date_param = datetime(*manifest.date_time)
1395         else:
1396             default_date_param = None
1397
1398         # Record in known apks, getting the added date at the same time..
1399         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1400                                     default_date=default_date_param)
1401         if added:
1402             apk['added'] = added
1403
1404         apkcache[apkfilename] = apk
1405         cachechanged = True
1406
1407     return False, apk, cachechanged
1408
1409
1410 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1411     """Processes the apks in the given repo directory.
1412
1413     This also extracts the icons.
1414
1415     :param apkcache: current apk cache information
1416     :param repodir: repo directory to scan
1417     :param knownapks: known apks info
1418     :param use_date_from_apk: use date from APK (instead of current date)
1419                               for newly added APKs
1420     :returns: (apks, cachechanged) where apks is a list of apk information,
1421               and cachechanged is True if the apkcache got changed.
1422     """
1423
1424     cachechanged = False
1425
1426     for icon_dir in get_all_icon_dirs(repodir):
1427         if os.path.exists(icon_dir):
1428             if options.clean:
1429                 shutil.rmtree(icon_dir)
1430                 os.makedirs(icon_dir)
1431         else:
1432             os.makedirs(icon_dir)
1433
1434     apks = []
1435     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1436         apkfilename = apkfile[len(repodir) + 1:]
1437         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1438         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1439                                              use_date_from_apk, ada, True)
1440         if skip:
1441             continue
1442         apks.append(apk)
1443         cachechanged = cachechanged or cachethis
1444
1445     return apks, cachechanged
1446
1447
1448 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1449     """Extracts PNG icons from an APK with the supported pixel densities
1450
1451     Extracts icons from the given APK zip in various densities, saves
1452     them into given repo directory and stores their names in the APK
1453     metadata dictionary.  If the icon is an XML icon, then this tries
1454     to find PNG icon that can replace it.
1455
1456     :param icon_filename: A string representing the icon's file name
1457     :param apk: A populated dictionary containing APK metadata.
1458                 Needs to have 'icons_src' key
1459     :param apkzip: An opened zipfile.ZipFile of the APK file
1460     :param repo_dir: The directory of the APK's repository
1461     :return: A list of icon densities that are missing
1462
1463     """
1464     res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
1465     pngs = dict()
1466     for f in apkzip.namelist():
1467         m = res_name_re.match(f)
1468         if m and m.group(4) == 'png':
1469             density = screen_resolutions[m.group(2)]
1470             pngs[m.group(3) + '/' + density] = m.group(0)
1471
1472     icon_type = None
1473     empty_densities = []
1474     for density in screen_densities:
1475         if density not in apk['icons_src']:
1476             empty_densities.append(density)
1477             continue
1478         icon_src = apk['icons_src'][density]
1479         icon_dir = get_icon_dir(repo_dir, density)
1480         icon_type = '.png'
1481
1482         # Extract the icon files per density
1483         if icon_src.endswith('.xml'):
1484             m = res_name_re.match(icon_src)
1485             if m:
1486                 name = pngs.get(m.group(3) + '/' + str(density))
1487                 if name:
1488                     icon_src = name
1489             if icon_src.endswith('.xml'):
1490                 empty_densities.append(density)
1491                 icon_type = '.xml'
1492         icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
1493
1494         try:
1495             with open(icon_dest, 'wb') as f:
1496                 f.write(get_icon_bytes(apkzip, icon_src))
1497             apk['icons'][density] = icon_filename + icon_type
1498         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1499             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1500             del apk['icons_src'][density]
1501             empty_densities.append(density)
1502
1503     # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
1504     if '-1' in apk['icons_src']:
1505         icon_src = apk['icons_src']['-1']
1506         icon_type = icon_src[-4:]
1507         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
1508         with open(icon_path, 'wb') as f:
1509             f.write(get_icon_bytes(apkzip, icon_src))
1510         if icon_type == '.png':
1511             im = None
1512             try:
1513                 im = Image.open(icon_path)
1514                 dpi = px_to_dpi(im.size[0])
1515                 for density in screen_densities:
1516                     if density in apk['icons']:
1517                         break
1518                     if density == screen_densities[-1] or dpi >= int(density):
1519                         apk['icons'][density] = icon_filename
1520                         shutil.move(icon_path,
1521                                     os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1522                         empty_densities.remove(density)
1523                         break
1524             except Exception as e:
1525                 logging.warning(_("Failed reading {path}: {error}")
1526                                 .format(path=icon_path, error=e))
1527             finally:
1528                 if im and hasattr(im, 'close'):
1529                     im.close()
1530
1531     if apk['icons']:
1532         apk['icon'] = icon_filename + icon_type
1533
1534     return empty_densities
1535
1536
1537 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1538     """
1539     Resize existing PNG icons for densities missing in the APK to ensure all densities are available
1540
1541     :param empty_densities: A list of icon densities that are missing
1542     :param icon_filename: A string representing the icon's file name
1543     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1544     :param repo_dir: The directory of the APK's repository
1545
1546     """
1547     icon_filename += '.png'
1548     # First try resizing down to not lose quality
1549     last_density = None
1550     for density in screen_densities:
1551         if density == '65534':  # not possible to generate 'anydpi' from other densities
1552             continue
1553         if density not in empty_densities:
1554             last_density = density
1555             continue
1556         if last_density is None:
1557             continue
1558         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1559
1560         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1561         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1562         fp = None
1563         try:
1564             fp = open(last_icon_path, 'rb')
1565             im = Image.open(fp)
1566
1567             size = dpi_to_px(density)
1568
1569             im.thumbnail((size, size), Image.ANTIALIAS)
1570             im.save(icon_path, "PNG", optimize=True,
1571                     pnginfo=BLANK_PNG_INFO, icc_profile=None)
1572             empty_densities.remove(density)
1573         except Exception as e:
1574             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1575         finally:
1576             if fp:
1577                 fp.close()
1578
1579     # Then just copy from the highest resolution available
1580     last_density = None
1581     for density in reversed(screen_densities):
1582         if density not in empty_densities:
1583             last_density = density
1584             continue
1585
1586         if last_density is None:
1587             continue
1588
1589         shutil.copyfile(
1590             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1591             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1592         )
1593         empty_densities.remove(density)
1594
1595     for density in screen_densities:
1596         icon_dir = get_icon_dir(repo_dir, density)
1597         icon_dest = os.path.join(icon_dir, icon_filename)
1598         resize_icon(icon_dest, density)
1599
1600     # Copy from icons-mdpi to icons since mdpi is the baseline density
1601     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1602     if os.path.isfile(baseline):
1603         apk['icons']['0'] = icon_filename
1604         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1605
1606
1607 def apply_info_from_latest_apk(apps, apks):
1608     """
1609     Some information from the apks needs to be applied up to the application level.
1610     When doing this, we use the info from the most recent version's apk.
1611     We deal with figuring out when the app was added and last updated at the same time.
1612     """
1613     for appid, app in apps.items():
1614         bestver = UNSET_VERSION_CODE
1615         for apk in apks:
1616             if apk['packageName'] == appid:
1617                 if apk['versionCode'] > bestver:
1618                     bestver = apk['versionCode']
1619                     bestapk = apk
1620
1621                 if 'added' in apk:
1622                     if not app.added or apk['added'] < app.added:
1623                         app.added = apk['added']
1624                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1625                         app.lastUpdated = apk['added']
1626
1627         if not app.added:
1628             logging.debug("Don't know when " + appid + " was added")
1629         if not app.lastUpdated:
1630             logging.debug("Don't know when " + appid + " was last updated")
1631
1632         if bestver == UNSET_VERSION_CODE:
1633
1634             if app.Name is None:
1635                 app.Name = app.AutoName or appid
1636             app.icon = None
1637             logging.debug("Application " + appid + " has no packages")
1638         else:
1639             if app.Name is None:
1640                 app.Name = bestapk['name']
1641             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1642             if app.CurrentVersionCode is None:
1643                 app.CurrentVersionCode = str(bestver)
1644
1645
1646 def make_categories_txt(repodir, categories):
1647     '''Write a category list in the repo to allow quick access'''
1648     catdata = ''
1649     for cat in sorted(categories):
1650         catdata += cat + '\n'
1651     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1652         f.write(catdata)
1653
1654
1655 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1656
1657     def filter_apk_list_sorted(apk_list):
1658         res = []
1659         for apk in apk_list:
1660             if apk['packageName'] == appid:
1661                 res.append(apk)
1662
1663         # Sort the apk list by version code. First is highest/newest.
1664         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1665
1666     for appid, app in apps.items():
1667
1668         if app.ArchivePolicy:
1669             keepversions = int(app.ArchivePolicy[:-9])
1670         else:
1671             keepversions = defaultkeepversions
1672
1673         logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}")
1674                       .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks)))
1675
1676         current_app_apks = filter_apk_list_sorted(apks)
1677         if len(current_app_apks) > keepversions:
1678             # Move back the ones we don't want.
1679             for apk in current_app_apks[keepversions:]:
1680                 move_apk_between_sections(repodir, archivedir, apk)
1681                 archapks.append(apk)
1682                 apks.remove(apk)
1683
1684         current_app_archapks = filter_apk_list_sorted(archapks)
1685         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1686             kept = 0
1687             # Move forward the ones we want again, except DisableAlgorithm
1688             for apk in current_app_archapks:
1689                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1690                     move_apk_between_sections(archivedir, repodir, apk)
1691                     archapks.remove(apk)
1692                     apks.append(apk)
1693                     kept += 1
1694                 if kept == keepversions:
1695                     break
1696
1697
1698 def move_apk_between_sections(from_dir, to_dir, apk):
1699     """move an APK from repo to archive or vice versa"""
1700
1701     def _move_file(from_dir, to_dir, filename, ignore_missing):
1702         from_path = os.path.join(from_dir, filename)
1703         if ignore_missing and not os.path.exists(from_path):
1704             return
1705         to_path = os.path.join(to_dir, filename)
1706         if not os.path.exists(to_dir):
1707             os.mkdir(to_dir)
1708         shutil.move(from_path, to_path)
1709
1710     if from_dir == to_dir:
1711         return
1712
1713     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1714     _move_file(from_dir, to_dir, apk['apkName'], False)
1715     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1716     for density in all_screen_densities:
1717         from_icon_dir = get_icon_dir(from_dir, density)
1718         to_icon_dir = get_icon_dir(to_dir, density)
1719         if density not in apk.get('icons', []):
1720             continue
1721         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1722     if 'srcname' in apk:
1723         _move_file(from_dir, to_dir, apk['srcname'], False)
1724
1725
1726 def add_apks_to_per_app_repos(repodir, apks):
1727     apks_per_app = dict()
1728     for apk in apks:
1729         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1730         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1731         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1732         apks_per_app[apk['packageName']] = apk
1733
1734         if not os.path.exists(apk['per_app_icons']):
1735             logging.info(_('Adding new repo for only {name}').format(name=apk['packageName']))
1736             os.makedirs(apk['per_app_icons'])
1737
1738         apkpath = os.path.join(repodir, apk['apkName'])
1739         shutil.copy(apkpath, apk['per_app_repo'])
1740         apksigpath = apkpath + '.sig'
1741         if os.path.exists(apksigpath):
1742             shutil.copy(apksigpath, apk['per_app_repo'])
1743         apkascpath = apkpath + '.asc'
1744         if os.path.exists(apkascpath):
1745             shutil.copy(apkascpath, apk['per_app_repo'])
1746
1747
1748 def create_metadata_from_template(apk):
1749     '''create a new metadata file using internal or external template
1750
1751     Generate warnings for apk's with no metadata (or create skeleton
1752     metadata files, if requested on the command line).  Though the
1753     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1754     since those impose things on the metadata file made from the
1755     template: field sort order, empty field value, formatting, etc.
1756     '''
1757
1758     import yaml
1759     if os.path.exists('template.yml'):
1760         with open('template.yml') as f:
1761             metatxt = f.read()
1762         if 'name' in apk and apk['name'] != '':
1763             metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',
1764                              r'\1 ' + apk['name'],
1765                              metatxt,
1766                              flags=re.IGNORECASE | re.MULTILINE)
1767         else:
1768             logging.warning(_('{appid} does not have a name! Using package name instead.')
1769                             .format(appid=apk['packageName']))
1770             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1771                              r'\1 ' + apk['packageName'],
1772                              metatxt,
1773                              flags=re.IGNORECASE | re.MULTILINE)
1774         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1775             f.write(metatxt)
1776     else:
1777         app = dict()
1778         app['Categories'] = [os.path.basename(os.getcwd())]
1779         # include some blanks as part of the template
1780         app['AuthorName'] = ''
1781         app['Summary'] = ''
1782         app['WebSite'] = ''
1783         app['IssueTracker'] = ''
1784         app['SourceCode'] = ''
1785         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1786         if 'name' in apk and apk['name'] != '':
1787             app['Name'] = apk['name']
1788         else:
1789             logging.warning(_('{appid} does not have a name! Using package name instead.')
1790                             .format(appid=apk['packageName']))
1791             app['Name'] = apk['packageName']
1792         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1793             yaml.dump(app, f, default_flow_style=False)
1794     logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
1795
1796
1797 config = None
1798 options = None
1799 start_timestamp = time.gmtime()
1800
1801
1802 def main():
1803
1804     global config, options
1805
1806     # Parse command line...
1807     parser = ArgumentParser()
1808     common.setup_global_opts(parser)
1809     parser.add_argument("--create-key", action="store_true", default=False,
1810                         help=_("Add a repo signing key to an unsigned repo"))
1811     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1812                         help=_("Add skeleton metadata files for APKs that are missing them"))
1813     parser.add_argument("--delete-unknown", action="store_true", default=False,
1814                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1815     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1816                         help=_("Report on build data status"))
1817     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1818                         help=_("Interactively ask about things that need updating."))
1819     parser.add_argument("-I", "--icons", action="store_true", default=False,
1820                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1821     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1822                         help=_("Specify editor to use in interactive mode. Default " +
1823                                "is {path}").format(path='/etc/alternatives/editor'))
1824     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1825                         help=_("Update the wiki"))
1826     parser.add_argument("--pretty", action="store_true", default=False,
1827                         help=_("Produce human-readable XML/JSON for index files"))
1828     parser.add_argument("--clean", action="store_true", default=False,
1829                         help=_("Clean update - don't uses caches, reprocess all APKs"))
1830     parser.add_argument("--nosign", action="store_true", default=False,
1831                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1832     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1833                         help=_("Use date from APK instead of current time for newly added APKs"))
1834     parser.add_argument("--rename-apks", action="store_true", default=False,
1835                         help=_("Rename APK files that do not match package.name_123.apk"))
1836     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1837                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1838     metadata.add_metadata_arguments(parser)
1839     options = parser.parse_args()
1840     metadata.warnings_action = options.W
1841
1842     config = common.read_config(options)
1843
1844     if not ('jarsigner' in config and 'keytool' in config):
1845         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))
1846
1847     repodirs = ['repo']
1848     if config['archive_older'] != 0:
1849         repodirs.append('archive')
1850         if not os.path.exists('archive'):
1851             os.mkdir('archive')
1852
1853     if options.icons:
1854         resize_all_icons(repodirs)
1855         sys.exit(0)
1856
1857     if options.rename_apks:
1858         options.clean = True
1859
1860     # check that icons exist now, rather than fail at the end of `fdroid update`
1861     for k in ['repo_icon', 'archive_icon']:
1862         if k in config:
1863             if not os.path.exists(config[k]):
1864                 logging.critical(_('{name} "{path}" does not exist! Correct it in config.py.')
1865                                  .format(name=k, path=config[k]))
1866                 sys.exit(1)
1867
1868     # if the user asks to create a keystore, do it now, reusing whatever it can
1869     if options.create_key:
1870         if os.path.exists(config['keystore']):
1871             logging.critical(_("Cowardily refusing to overwrite existing signing key setup!"))
1872             logging.critical("\t'" + config['keystore'] + "'")
1873             sys.exit(1)
1874
1875         if 'repo_keyalias' not in config:
1876             config['repo_keyalias'] = socket.getfqdn()
1877             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1878         if 'keydname' not in config:
1879             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1880             common.write_to_config(config, 'keydname', config['keydname'])
1881         if 'keystore' not in config:
1882             config['keystore'] = common.default_config['keystore']
1883             common.write_to_config(config, 'keystore', config['keystore'])
1884
1885         password = common.genpassword()
1886         if 'keystorepass' not in config:
1887             config['keystorepass'] = password
1888             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1889         if 'keypass' not in config:
1890             config['keypass'] = password
1891             common.write_to_config(config, 'keypass', config['keypass'])
1892         common.genkeystore(config)
1893
1894     # Get all apps...
1895     apps = metadata.read_metadata()
1896
1897     # Generate a list of categories...
1898     categories = set()
1899     for app in apps.values():
1900         categories.update(app.Categories)
1901
1902     # Read known apks data (will be updated and written back when we've finished)
1903     knownapks = common.KnownApks()
1904
1905     # Get APK cache
1906     apkcache = get_cache()
1907
1908     # Delete builds for disabled apps
1909     delete_disabled_builds(apps, apkcache, repodirs)
1910
1911     # Scan all apks in the main repo
1912     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1913
1914     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1915                                            options.use_date_from_apk)
1916     cachechanged = cachechanged or fcachechanged
1917     apks += files
1918     for apk in apks:
1919         if apk['packageName'] not in apps:
1920             if options.create_metadata:
1921                 create_metadata_from_template(apk)
1922                 apps = metadata.read_metadata()
1923             else:
1924                 msg = _("{apkfilename} ({appid}) has no metadata!") \
1925                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
1926                 if options.delete_unknown:
1927                     logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
1928                                  .format(apkfilename=apk['apkName']))
1929                     rmf = os.path.join(repodirs[0], apk['apkName'])
1930                     if not os.path.exists(rmf):
1931                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
1932                     else:
1933                         os.remove(rmf)
1934                 else:
1935                     logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
1936
1937     copy_triple_t_store_metadata(apps)
1938     insert_obbs(repodirs[0], apps, apks)
1939     insert_localized_app_metadata(apps)
1940     translate_per_build_anti_features(apps, apks)
1941
1942     # Scan the archive repo for apks as well
1943     if len(repodirs) > 1:
1944         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1945         if cc:
1946             cachechanged = True
1947     else:
1948         archapks = []
1949
1950     # Apply information from latest apks to the application and update dates
1951     apply_info_from_latest_apk(apps, apks + archapks)
1952
1953     # Sort the app list by name, then the web site doesn't have to by default.
1954     # (we had to wait until we'd scanned the apks to do this, because mostly the
1955     # name comes from there!)
1956     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1957
1958     # APKs are placed into multiple repos based on the app package, providing
1959     # per-app subscription feeds for nightly builds and things like it
1960     if config['per_app_repos']:
1961         add_apks_to_per_app_repos(repodirs[0], apks)
1962         for appid, app in apps.items():
1963             repodir = os.path.join(appid, 'fdroid', 'repo')
1964             appdict = dict()
1965             appdict[appid] = app
1966             if os.path.isdir(repodir):
1967                 index.make(appdict, [appid], apks, repodir, False)
1968             else:
1969                 logging.info(_('Skipping index generation for {appid}').format(appid=appid))
1970         return
1971
1972     if len(repodirs) > 1:
1973         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1974
1975     # Make the index for the main repo...
1976     index.make(apps, sortedids, apks, repodirs[0], False)
1977     make_categories_txt(repodirs[0], categories)
1978
1979     # If there's an archive repo,  make the index for it. We already scanned it
1980     # earlier on.
1981     if len(repodirs) > 1:
1982         index.make(apps, sortedids, archapks, repodirs[1], True)
1983
1984     git_remote = config.get('binary_transparency_remote')
1985     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1986         from . import btlog
1987         btlog.make_binary_transparency_log(repodirs)
1988
1989     if config['update_stats']:
1990         # Update known apks info...
1991         knownapks.writeifchanged()
1992
1993         # Generate latest apps data for widget
1994         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1995             data = ''
1996             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1997                 for line in f:
1998                     appid = line.rstrip()
1999                     data += appid + "\t"
2000                     app = apps[appid]
2001                     data += app.Name + "\t"
2002                     if app.icon is not None:
2003                         data += app.icon + "\t"
2004                     data += app.License + "\n"
2005             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
2006                 f.write(data)
2007
2008     if cachechanged:
2009         write_cache(apkcache)
2010
2011     # Update the wiki...
2012     if options.wiki:
2013         update_wiki(apps, sortedids, apks + archapks)
2014
2015     logging.info(_("Finished"))
2016
2017
2018 if __name__ == "__main__":
2019     main()