X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=fdroidserver%2Fscanner.py;h=272bebd224cbec354c63cc71060b17a05072395a;hb=44d17663fdd85fdda3717b4bf35cecdd151809a8;hp=4ac6276ddaa3abe3d858d00244e7a3036239e809;hpb=bf8518ee8ff120f219273e8e2d309cd8de2da9ab;p=fdroidserver.git diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 4ac6276d..272bebd2 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- +#!/usr/bin/env python3 # # scanner.py - part of the FDroid server tools # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com @@ -23,16 +22,23 @@ import traceback from argparse import ArgumentParser import logging -import common -import metadata -from common import BuildException, VCSException +from . import _ +from . import common +from . import metadata +from .exception import BuildException, VCSException config = None options = None def get_gradle_compile_commands(build): - compileCommands = ['compile', 'releaseCompile'] + compileCommands = ['compile', 'releaseCompile' + 'provided', 'releaseProvided', + 'apk', 'releaseApk', + 'implementation', 'releaseImplementation', + 'api', 'releaseApi', + 'compileOnly', 'releaseCompileOnly', + 'runtimeOnly', 'releaseRuntimeOnly'] if build.gradle and build.gradle != ['yes']: compileCommands += [flavor + 'Compile' for flavor in build.gradle] compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle] @@ -40,9 +46,10 @@ def get_gradle_compile_commands(build): return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands] -# Scan the source code in the given directory (and all subdirectories) -# and return the number of fatal problems encountered -def scan_source(build_dir, root_dir, build): +def scan_source(build_dir, build=metadata.Build()): + """Scan the source code in the given directory (and all subdirectories) + and return the number of fatal problems encountered + """ count = 0 @@ -64,12 +71,22 @@ def scan_source(build_dir, root_dir, build): r'crashlytics', r'ouya.*sdk', r'libspen23', + r'firebase', ] } + whitelisted = [ + 'firebase-jobdispatcher', # https://github.com/firebase/firebase-jobdispatcher-android/blob/master/LICENSE + 'com.firebaseui', # https://github.com/firebase/FirebaseUI-Android/blob/master/LICENSE + 'geofire-android' # https://github.com/firebase/geofire-java/blob/master/LICENSE + ] + + def is_whitelisted(s): + return any(wl in s for wl in whitelisted) + def suspects_found(s): - for n, r in usual_suspects.iteritems(): - if r.match(s): + for n, r in usual_suspects.items(): + if r.match(s) and not is_whitelisted(s): yield n gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?') @@ -79,9 +96,14 @@ def scan_source(build_dir, root_dir, build): 'jcenter.bintray.com', # jcenter() 'jitpack.io', 'repo.maven.apache.org/maven2', + 'oss.jfrog.org/artifactory/oss-snapshot-local', 'oss.sonatype.org/content/repositories/snapshots', 'oss.sonatype.org/content/repositories/releases', 'oss.sonatype.org/content/groups/public', + 'clojars.org/repo', # Clojure free software libs + 's3.amazonaws.com/repo.commonsware.com', # CommonsWare + 'plugins.gradle.org/m2', # Gradle plugin repo + 'maven.google.com', # Google Maven Repo, https://developer.android.com/studio/build/dependencies.html#google-maven ] ] @@ -91,42 +113,42 @@ def scan_source(build_dir, root_dir, build): scanignore_worked = set() scandelete_worked = set() - def toignore(fd): - for k, paths in scanignore.iteritems(): + def toignore(path_in_build_dir): + for k, paths in scanignore.items(): for p in paths: - if fd.startswith(p): + if path_in_build_dir.startswith(p): scanignore_worked.add(k) return True return False - def todelete(fd): - for k, paths in scandelete.iteritems(): + def todelete(path_in_build_dir): + for k, paths in scandelete.items(): for p in paths: - if fd.startswith(p): + if path_in_build_dir.startswith(p): scandelete_worked.add(k) return True return False - def ignoreproblem(what, fd, fp): - logging.info('Ignoring %s at %s' % (what, fd)) + def ignoreproblem(what, path_in_build_dir): + logging.info('Ignoring %s at %s' % (what, path_in_build_dir)) return 0 - def removeproblem(what, fd, fp): - logging.info('Removing %s at %s' % (what, fd)) - os.remove(fp) + def removeproblem(what, path_in_build_dir, filepath): + logging.info('Removing %s at %s' % (what, path_in_build_dir)) + os.remove(filepath) return 0 - def warnproblem(what, fd): - if toignore(fd): + def warnproblem(what, path_in_build_dir): + if toignore(path_in_build_dir): return - logging.warn('Found %s at %s' % (what, fd)) - - def handleproblem(what, fd, fp): - if toignore(fd): - return ignoreproblem(what, fd, fp) - if todelete(fd): - return removeproblem(what, fd, fp) - logging.error('Found %s at %s' % (what, fd)) + logging.warn('Found %s at %s' % (what, path_in_build_dir)) + + def handleproblem(what, path_in_build_dir, filepath): + if toignore(path_in_build_dir): + return ignoreproblem(what, path_in_build_dir) + if todelete(path_in_build_dir): + return removeproblem(what, path_in_build_dir, filepath) + logging.error('Found %s at %s' % (what, path_in_build_dir)) return 1 def is_executable(path): @@ -159,72 +181,79 @@ def scan_source(build_dir, root_dir, build): return any(command.match(line) for command in gradle_compile_commands) # Iterate through all files in the source code - for r, d, f in os.walk(build_dir, topdown=True): + for root, dirs, files in os.walk(build_dir, topdown=True): # It's topdown, so checking the basename is enough for ignoredir in ('.hg', '.git', '.svn', '.bzr'): - if ignoredir in d: - d.remove(ignoredir) + if ignoredir in dirs: + dirs.remove(ignoredir) - for curfile in f: + for curfile in files: if curfile in ['.DS_Store']: continue # Path (relative) to the file - fp = os.path.join(r, curfile) + filepath = os.path.join(root, curfile) - if os.path.islink(fp): + if os.path.islink(filepath): continue - fd = fp[len(build_dir) + 1:] - _, ext = common.get_extension(fd) + path_in_build_dir = os.path.relpath(filepath, build_dir) + _ignored, ext = common.get_extension(path_in_build_dir) if ext == 'so': - count += handleproblem('shared library', fd, fp) + count += handleproblem('shared library', path_in_build_dir, filepath) elif ext == 'a': - count += handleproblem('static library', fd, fp) + count += handleproblem('static library', path_in_build_dir, filepath) elif ext == 'class': - count += handleproblem('Java compiled class', fd, fp) + count += handleproblem('Java compiled class', path_in_build_dir, filepath) elif ext == 'apk': - removeproblem('APK file', fd, fp) + removeproblem('APK file', path_in_build_dir, filepath) elif ext == 'jar': for name in suspects_found(curfile): - count += handleproblem('usual supect \'%s\'' % name, fd, fp) - warnproblem('JAR file', fd) + count += handleproblem('usual suspect \'%s\'' % name, path_in_build_dir, filepath) + if curfile == 'gradle-wrapper.jar': + removeproblem('gradle-wrapper.jar', path_in_build_dir, filepath) + else: + warnproblem('JAR file', path_in_build_dir) + + elif ext == 'aar': + warnproblem('AAR file', path_in_build_dir) elif ext == 'java': - if not os.path.isfile(fp): + if not os.path.isfile(filepath): continue - for line in file(fp): - if 'DexClassLoader' in line: - count += handleproblem('DexClassLoader', fd, fp) - break + with open(filepath, 'r', encoding='utf8', errors='replace') as f: + for line in f: + if 'DexClassLoader' in line: + count += handleproblem('DexClassLoader', path_in_build_dir, filepath) + break elif ext == 'gradle': - if not os.path.isfile(fp): + if not os.path.isfile(filepath): continue - with open(fp, 'r') as f: + with open(filepath, 'r', encoding='utf8', errors='replace') as f: lines = f.readlines() for i, line in enumerate(lines): if is_used_by_gradle(line): for name in suspects_found(line): - count += handleproblem('usual supect \'%s\' at line %d' % (name, i+1), fd, fp) + count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath) noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)] joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines)) for m in gradle_mavenrepo.finditer(joined): url = m.group(2) if not any(r.match(url) for r in allowed_repos): - count += handleproblem('unknown maven repo \'%s\'' % url, fd, fp) + count += handleproblem('unknown maven repo \'%s\'' % url, path_in_build_dir, filepath) elif ext in ['', 'bin', 'out', 'exe']: - if is_binary(fp): - count += handleproblem('binary', fd, fp) + if is_binary(filepath): + count += handleproblem('binary', path_in_build_dir, filepath) - elif is_executable(fp): - if is_binary(fp) and not safe_path(fd): - warnproblem('possible binary', fd) + elif is_executable(filepath): + if is_binary(filepath) and not safe_path(path_in_build_dir): + warnproblem('possible binary', path_in_build_dir) for p in scanignore: if p not in scanignore_worked: @@ -236,14 +265,6 @@ def scan_source(build_dir, root_dir, build): logging.error('Unused scandelete path: %s' % p) count += 1 - # Presence of a jni directory without buildjni=yes might - # indicate a problem (if it's not a problem, explicitly use - # buildjni=no to bypass this check) - if (os.path.exists(os.path.join(root_dir, 'jni')) and - not build.buildjni): - logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.') - count += 1 - return count @@ -254,8 +275,10 @@ def main(): # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") + parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]")) + metadata.add_metadata_arguments(parser) options = parser.parse_args() + metadata.warnings_action = options.W config = common.read_config(options) @@ -272,24 +295,30 @@ def main(): srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') - for appid, app in apps.iteritems(): + for appid, app in apps.items(): if app.Disabled: - logging.info("Skipping %s: disabled" % appid) + logging.info(_("Skipping {appid}: disabled").format(appid=appid)) continue - if not app.builds: - logging.info("Skipping %s: no builds specified" % appid) - continue - - logging.info("Processing " + appid) try: - if app.RepoType == 'srclib': build_dir = os.path.join('build', 'srclib', app.Repo) else: build_dir = os.path.join('build', appid) + if app.builds: + logging.info(_("Processing {appid}").format(appid=appid)) + else: + logging.info(_("{appid}: no builds specified, running on current source state") + .format(appid=appid)) + count = scan_source(build_dir) + if count > 0: + logging.warn(_('Scanner found {count} problems in {appid}:') + .format(count=count, appid=appid)) + probcount += count + continue + # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app.RepoType, app.Repo, build_dir) @@ -297,21 +326,20 @@ def main(): if build.disable: logging.info("...skipping version %s - %s" % ( - build.version, build.get('disable', build.commit[1:]))) - else: - logging.info("...scanning version " + build.version) + build.versionName, build.get('disable', build.commit[1:]))) + continue - # Prepare the source code... - root_dir, _ = common.prepare_source(vcs, app, build, - build_dir, srclib_dir, - extlib_dir, False) + logging.info("...scanning version " + build.versionName) + # Prepare the source code... + common.prepare_source(vcs, app, build, + build_dir, srclib_dir, + extlib_dir, False) - # Do the scan... - count = scan_source(build_dir, root_dir, build) - if count > 0: - logging.warn('Scanner found %d problems in %s (%s)' % ( - count, appid, build.vercode)) - probcount += count + count = scan_source(build_dir, build) + if count > 0: + logging.warn(_('Scanner found {count} problems in {appid}:{versionCode}:') + .format(count=count, appid=appid, versionCode=build.versionCode)) + probcount += count except BuildException as be: logging.warn("Could not scan app %s due to BuildException: %s" % ( @@ -325,8 +353,9 @@ def main(): appid, traceback.format_exc())) probcount += 1 - logging.info("Finished:") - print "%d problems found" % probcount + logging.info(_("Finished")) + print(_("%d problems found") % probcount) + if __name__ == "__main__": main()