options = None
-def init_mime_type():
- '''
- There are two incompatible versions of the 'magic' module, one
- that comes as part of libmagic, which is what Debian includes as
- python-magic, then another called python-magic that is a separate
- project that wraps libmagic. The second is 'magic' on pypi, so
- both need to be supported. Then on platforms where libmagic is
- not easily included, e.g. OSX and Windows, fallback to the
- built-in 'mimetypes' module so this will work without
- libmagic. Hence this function with the following hacks:
- '''
-
- init_path = ''
- method = ''
- ms = None
-
- def mime_from_file(path):
- try:
- return magic.from_file(path, mime=True)
- except UnicodeError:
- return None
-
- def mime_file(path):
- try:
- return ms.file(path)
- except UnicodeError:
- return None
-
- def mime_guess_type(path):
- return mimetypes.guess_type(path, strict=False)
+def get_gradle_compile_commands(build):
+ compileCommands = ['compile', 'releaseCompile']
+ if build.gradle and build.gradle != ['yes']:
+ compileCommands += [flavor + 'Compile' for flavor in build.gradle]
+ compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
- try:
- import magic
- try:
- ms = magic.open(magic.MIME_TYPE)
- ms.load()
- magic.from_file(init_path, mime=True)
- method = 'from_file'
- except AttributeError:
- ms.file(init_path)
- method = 'file'
- except ImportError:
- import mimetypes
- mimetypes.init()
- method = 'guess_type'
-
- logging.info("Using magic method " + method)
- if method == 'from_file':
- return mime_from_file
- if method == 'file':
- return mime_file
- if method == 'guess_type':
- return mime_guess_type
-
- logging.critical("unknown magic method!")
+ return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
# Scan the source code in the given directory (and all subdirectories)
# and return the number of fatal problems encountered
-def scan_source(build_dir, root_dir, thisbuild):
+def scan_source(build_dir, root_dir, build):
count = 0
# Common known non-free blobs (always lower case):
- usual_suspects = [
- re.compile(r'.*flurryagent', re.IGNORECASE),
- re.compile(r'.*paypal.*mpl', re.IGNORECASE),
- re.compile(r'.*google.*analytics', re.IGNORECASE),
- re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
- re.compile(r'.*google.*ad.*view', re.IGNORECASE),
- re.compile(r'.*google.*admob', re.IGNORECASE),
- re.compile(r'.*google.*play.*services', re.IGNORECASE),
- re.compile(r'.*crittercism', re.IGNORECASE),
- re.compile(r'.*heyzap', re.IGNORECASE),
- re.compile(r'.*jpct.*ae', re.IGNORECASE),
- re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
- re.compile(r'.*bugsense', re.IGNORECASE),
- re.compile(r'.*crashlytics', re.IGNORECASE),
- re.compile(r'.*ouya.*sdk', re.IGNORECASE),
- re.compile(r'.*libspen23', re.IGNORECASE),
+ usual_suspects = {
+ exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
+ r'flurryagent',
+ r'paypal.*mpl',
+ r'google.*analytics',
+ r'admob.*sdk.*android',
+ r'google.*ad.*view',
+ r'google.*admob',
+ r'google.*play.*services',
+ r'crittercism',
+ r'heyzap',
+ r'jpct.*ae',
+ r'youtube.*android.*player.*api',
+ r'bugsense',
+ r'crashlytics',
+ r'ouya.*sdk',
+ r'libspen23',
+ ]
+ }
+
+ def suspects_found(s):
+ for n, r in usual_suspects.iteritems():
+ if r.match(s):
+ yield n
+
+ gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?')
+
+ allowed_repos = [re.compile(r'^https?://' + re.escape(repo) + r'/*') for repo in [
+ 'repo1.maven.org/maven2', # mavenCentral()
+ 'jcenter.bintray.com', # jcenter()
+ 'jitpack.io',
+ 'repo.maven.apache.org/maven2',
+ 'oss.sonatype.org/content/repositories/snapshots',
+ 'oss.sonatype.org/content/repositories/releases',
+ 'oss.sonatype.org/content/groups/public',
+ 'clojars.org/repo', # Clojure free software libs
+ ]
]
- scanignore = common.getpaths(build_dir, thisbuild, 'scanignore')
- scandelete = common.getpaths(build_dir, thisbuild, 'scandelete')
+ scanignore = common.getpaths_map(build_dir, build.scanignore)
+ scandelete = common.getpaths_map(build_dir, build.scandelete)
scanignore_worked = set()
scandelete_worked = set()
def toignore(fd):
- for p in scanignore:
- if fd.startswith(p):
- scanignore_worked.add(p)
- return True
+ for k, paths in scanignore.iteritems():
+ for p in paths:
+ if fd.startswith(p):
+ scanignore_worked.add(k)
+ return True
return False
def todelete(fd):
- for p in scandelete:
- if fd.startswith(p):
- scandelete_worked.add(p)
- return True
+ for k, paths in scandelete.iteritems():
+ for p in paths:
+ if fd.startswith(p):
+ scandelete_worked.add(k)
+ return True
return False
def ignoreproblem(what, fd, fp):
return 0
def warnproblem(what, fd):
+ if toignore(fd):
+ return
logging.warn('Found %s at %s' % (what, fd))
def handleproblem(what, fd, fp):
logging.error('Found %s at %s' % (what, fd))
return 1
- get_mime_type = init_mime_type()
+ def is_executable(path):
+ return os.path.exists(path) and os.access(path, os.X_OK)
+
+ textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
+
+ def is_binary(path):
+ d = None
+ with open(path, 'rb') as f:
+ d = f.read(1024)
+ return bool(d.translate(None, textchars))
+
+ # False positives patterns for files that are binary and executable.
+ safe_paths = [re.compile(r) for r in [
+ r".*/drawable[^/]*/.*\.png$", # png drawables
+ r".*/mipmap[^/]*/.*\.png$", # png mipmaps
+ ]
+ ]
+
+ def safe_path(path):
+ for sp in safe_paths:
+ if sp.match(path):
+ return True
+ return False
+
+ gradle_compile_commands = get_gradle_compile_commands(build)
+
+ def is_used_by_gradle(line):
+ return any(command.match(line) for command in gradle_compile_commands)
# Iterate through all files in the source code
for r, d, f in os.walk(build_dir, topdown=True):
for curfile in f:
+ if curfile in ['.DS_Store']:
+ continue
+
# Path (relative) to the file
fp = os.path.join(r, curfile)
- fd = fp[len(build_dir) + 1:]
- mime = get_mime_type(fp)
+ if os.path.islink(fp):
+ continue
- if mime == 'application/x-sharedlib':
- count += handleproblem('shared library', fd, fp)
+ fd = fp[len(build_dir) + 1:]
+ _, ext = common.get_extension(fd)
- elif mime == 'application/x-archive':
+ if ext == 'so':
+ count += handleproblem('shared library', fd, fp)
+ elif ext == 'a':
count += handleproblem('static library', fd, fp)
-
- elif mime == 'application/x-executable' or mime == 'application/x-mach-binary':
- count += handleproblem('binary executable', fd, fp)
-
- elif mime == 'application/x-java-applet':
+ elif ext == 'class':
count += handleproblem('Java compiled class', fd, fp)
+ elif ext == 'apk':
+ removeproblem('APK file', fd, fp)
- elif mime in (
- 'application/jar',
- 'application/zip',
- 'application/java-archive',
- 'application/octet-stream',
- 'binary', ):
+ elif ext == 'jar':
+ for name in suspects_found(curfile):
+ count += handleproblem('usual supect \'%s\'' % name, fd, fp)
+ warnproblem('JAR file', fd)
- if common.has_extension(fp, 'apk'):
- removeproblem('APK file', fd, fp)
-
- elif common.has_extension(fp, 'jar'):
-
- if any(suspect.match(curfile) for suspect in usual_suspects):
- count += handleproblem('usual supect', fd, fp)
- else:
- warnproblem('JAR file', fd)
-
- elif common.has_extension(fp, 'zip'):
- warnproblem('ZIP file', fd)
-
- else:
- warnproblem('unknown compressed or binary file', fd)
-
- elif common.has_extension(fp, 'java'):
+ elif ext == 'java':
if not os.path.isfile(fp):
continue
for line in file(fp):
count += handleproblem('DexClassLoader', fd, fp)
break
- elif common.has_extension(fp, 'gradle'):
+ elif ext == 'gradle':
if not os.path.isfile(fp):
continue
- for i, line in enumerate(file(fp)):
- i = i + 1
- if any(suspect.match(line) for suspect in usual_suspects):
- count += handleproblem('usual suspect at line %d' % i, fd, fp)
- break
+ with open(fp, 'r') as f:
+ lines = f.readlines()
+ for i, line in enumerate(lines):
+ if is_used_by_gradle(line):
+ for name in suspects_found(line):
+ count += handleproblem('usual supect \'%s\' at line %d' % (name, i + 1), fd, fp)
+ noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
+ joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
+ for m in gradle_mavenrepo.finditer(joined):
+ url = m.group(2)
+ if not any(r.match(url) for r in allowed_repos):
+ count += handleproblem('unknown maven repo \'%s\'' % url, fd, fp)
+
+ elif ext in ['', 'bin', 'out', 'exe']:
+ if is_binary(fp):
+ count += handleproblem('binary', fd, fp)
+
+ elif is_executable(fp):
+ if is_binary(fp) and not safe_path(fd):
+ warnproblem('possible binary', fd)
for p in scanignore:
if p not in scanignore_worked:
logging.error('Unused scandelete path: %s' % p)
count += 1
- # Presence of a jni directory without buildjni=yes might
- # indicate a problem (if it's not a problem, explicitly use
- # buildjni=no to bypass this check)
- if (os.path.exists(os.path.join(root_dir, 'jni')) and
- not thisbuild['buildjni']):
- logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
- count += 1
-
return count
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
+ common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
- parser.add_argument("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
- parser.add_argument("-q", "--quiet", action="store_true", default=False,
- help="Restrict output to warnings and errors")
options = parser.parse_args()
config = common.read_config(options)
for appid, app in apps.iteritems():
- if app['Disabled']:
+ if app.Disabled:
logging.info("Skipping %s: disabled" % appid)
continue
- if not app['builds']:
+ if not app.builds:
logging.info("Skipping %s: no builds specified" % appid)
continue
try:
- build_dir = 'build/' + appid
+ if app.RepoType == 'srclib':
+ build_dir = os.path.join('build', 'srclib', app.Repo)
+ else:
+ build_dir = os.path.join('build', appid)
# Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+ vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
- for thisbuild in app['builds']:
+ for build in app.builds:
- if thisbuild['disable']:
+ if build.disable:
logging.info("...skipping version %s - %s" % (
- thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:])))
+ build.version, build.get('disable', build.commit[1:])))
else:
- logging.info("...scanning version " + thisbuild['version'])
+ logging.info("...scanning version " + build.version)
# Prepare the source code...
- root_dir, _ = common.prepare_source(vcs, app, thisbuild,
+ root_dir, _ = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, False)
# Do the scan...
- count = scan_source(build_dir, root_dir, thisbuild)
+ count = scan_source(build_dir, root_dir, build)
if count > 0:
logging.warn('Scanner found %d problems in %s (%s)' % (
- count, appid, thisbuild['vercode']))
+ count, appid, build.vercode))
probcount += count
except BuildException as be:
probcount += 1
logging.info("Finished:")
- print "%d app(s) with problems" % probcount
+ print("%d problems found" % probcount)
if __name__ == "__main__":
main()