3 # scanner.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
22 from argparse import ArgumentParser
27 from . import metadata
28 from .exception import BuildException, VCSException
34 def get_gradle_compile_commands(build):
35 compileCommands = ['compile', 'releaseCompile'
36 'provided', 'releaseProvided',
38 'implementation', 'releaseImplementation',
40 'compileOnly', 'releaseCompileOnly',
41 'runtimeOnly', 'releaseRuntimeOnly']
42 if build.gradle and build.gradle != ['yes']:
43 compileCommands += [flavor + 'Compile' for flavor in build.gradle]
44 compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
46 return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
49 def scan_source(build_dir, build=metadata.Build()):
50 """Scan the source code in the given directory (and all subdirectories)
51 and return the number of fatal problems encountered
56 # Common known non-free blobs (always lower case):
58 exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
62 r'admob.*sdk.*android',
65 r'google.*play.*services',
69 r'youtube.*android.*player.*api',
79 'firebase-jobdispatcher', # https://github.com/firebase/firebase-jobdispatcher-android/blob/master/LICENSE
80 'com.firebaseui', # https://github.com/firebase/FirebaseUI-Android/blob/master/LICENSE
81 'geofire-android' # https://github.com/firebase/geofire-java/blob/master/LICENSE
84 def is_whitelisted(s):
85 return any(wl in s for wl in whitelisted)
87 def suspects_found(s):
88 for n, r in usual_suspects.items():
89 if r.match(s) and not is_whitelisted(s):
92 gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?')
94 allowed_repos = [re.compile(r'^https?://' + re.escape(repo) + r'/*') for repo in [
95 'repo1.maven.org/maven2', # mavenCentral()
96 'jcenter.bintray.com', # jcenter()
98 'repo.maven.apache.org/maven2',
99 'oss.jfrog.org/artifactory/oss-snapshot-local',
100 'oss.sonatype.org/content/repositories/snapshots',
101 'oss.sonatype.org/content/repositories/releases',
102 'oss.sonatype.org/content/groups/public',
103 'clojars.org/repo', # Clojure free software libs
104 's3.amazonaws.com/repo.commonsware.com', # CommonsWare
105 'plugins.gradle.org/m2', # Gradle plugin repo
106 'maven.google.com', # Google Maven Repo, https://developer.android.com/studio/build/dependencies.html#google-maven
110 scanignore = common.getpaths_map(build_dir, build.scanignore)
111 scandelete = common.getpaths_map(build_dir, build.scandelete)
113 scanignore_worked = set()
114 scandelete_worked = set()
116 def toignore(path_in_build_dir):
117 for k, paths in scanignore.items():
119 if path_in_build_dir.startswith(p):
120 scanignore_worked.add(k)
124 def todelete(path_in_build_dir):
125 for k, paths in scandelete.items():
127 if path_in_build_dir.startswith(p):
128 scandelete_worked.add(k)
132 def ignoreproblem(what, path_in_build_dir):
133 logging.info('Ignoring %s at %s' % (what, path_in_build_dir))
136 def removeproblem(what, path_in_build_dir, filepath):
137 logging.info('Removing %s at %s' % (what, path_in_build_dir))
141 def warnproblem(what, path_in_build_dir):
142 if toignore(path_in_build_dir):
144 logging.warn('Found %s at %s' % (what, path_in_build_dir))
146 def handleproblem(what, path_in_build_dir, filepath):
147 if toignore(path_in_build_dir):
148 return ignoreproblem(what, path_in_build_dir)
149 if todelete(path_in_build_dir):
150 return removeproblem(what, path_in_build_dir, filepath)
151 logging.error('Found %s at %s' % (what, path_in_build_dir))
154 def is_executable(path):
155 return os.path.exists(path) and os.access(path, os.X_OK)
157 textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
161 with open(path, 'rb') as f:
163 return bool(d.translate(None, textchars))
165 # False positives patterns for files that are binary and executable.
166 safe_paths = [re.compile(r) for r in [
167 r".*/drawable[^/]*/.*\.png$", # png drawables
168 r".*/mipmap[^/]*/.*\.png$", # png mipmaps
173 for sp in safe_paths:
178 gradle_compile_commands = get_gradle_compile_commands(build)
180 def is_used_by_gradle(line):
181 return any(command.match(line) for command in gradle_compile_commands)
183 # Iterate through all files in the source code
184 for root, dirs, files in os.walk(build_dir, topdown=True):
186 # It's topdown, so checking the basename is enough
187 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
188 if ignoredir in dirs:
189 dirs.remove(ignoredir)
191 for curfile in files:
193 if curfile in ['.DS_Store']:
196 # Path (relative) to the file
197 filepath = os.path.join(root, curfile)
199 if os.path.islink(filepath):
202 path_in_build_dir = os.path.relpath(filepath, build_dir)
203 _ignored, ext = common.get_extension(path_in_build_dir)
206 count += handleproblem('shared library', path_in_build_dir, filepath)
208 count += handleproblem('static library', path_in_build_dir, filepath)
210 count += handleproblem('Java compiled class', path_in_build_dir, filepath)
212 removeproblem('APK file', path_in_build_dir, filepath)
215 for name in suspects_found(curfile):
216 count += handleproblem('usual suspect \'%s\'' % name, path_in_build_dir, filepath)
217 if curfile == 'gradle-wrapper.jar':
218 removeproblem('gradle-wrapper.jar', path_in_build_dir, filepath)
220 warnproblem('JAR file', path_in_build_dir)
223 warnproblem('AAR file', path_in_build_dir)
226 if not os.path.isfile(filepath):
228 with open(filepath, 'r', encoding='utf8', errors='replace') as f:
230 if 'DexClassLoader' in line:
231 count += handleproblem('DexClassLoader', path_in_build_dir, filepath)
234 elif ext == 'gradle':
235 if not os.path.isfile(filepath):
237 with open(filepath, 'r', encoding='utf8', errors='replace') as f:
238 lines = f.readlines()
239 for i, line in enumerate(lines):
240 if is_used_by_gradle(line):
241 for name in suspects_found(line):
242 count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath)
243 noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
244 joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
245 for m in gradle_mavenrepo.finditer(joined):
247 if not any(r.match(url) for r in allowed_repos):
248 count += handleproblem('unknown maven repo \'%s\'' % url, path_in_build_dir, filepath)
250 elif ext in ['', 'bin', 'out', 'exe']:
251 if is_binary(filepath):
252 count += handleproblem('binary', path_in_build_dir, filepath)
254 elif is_executable(filepath):
255 if is_binary(filepath) and not safe_path(path_in_build_dir):
256 warnproblem('possible binary', path_in_build_dir)
259 if p not in scanignore_worked:
260 logging.error('Unused scanignore path: %s' % p)
264 if p not in scandelete_worked:
265 logging.error('Unused scandelete path: %s' % p)
273 global config, options
275 # Parse command line...
276 parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
277 common.setup_global_opts(parser)
278 parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
279 metadata.add_metadata_arguments(parser)
280 options = parser.parse_args()
281 metadata.warnings_action = options.W
283 config = common.read_config(options)
285 # Read all app and srclib metadata
286 allapps = metadata.read_metadata()
287 apps = common.read_app_args(options.appid, allapps, True)
292 if not os.path.isdir(build_dir):
293 logging.info("Creating build directory")
294 os.makedirs(build_dir)
295 srclib_dir = os.path.join(build_dir, 'srclib')
296 extlib_dir = os.path.join(build_dir, 'extlib')
298 for appid, app in apps.items():
301 logging.info(_("Skipping {appid}: disabled").format(appid=appid))
305 if app.RepoType == 'srclib':
306 build_dir = os.path.join('build', 'srclib', app.Repo)
308 build_dir = os.path.join('build', appid)
311 logging.info(_("Processing {appid}").format(appid=appid))
313 logging.info(_("{appid}: no builds specified, running on current source state")
314 .format(appid=appid))
315 count = scan_source(build_dir)
317 logging.warn(_('Scanner found {count} problems in {appid}:')
318 .format(count=count, appid=appid))
322 # Set up vcs interface and make sure we have the latest code...
323 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
325 for build in app.builds:
328 logging.info("...skipping version %s - %s" % (
329 build.versionName, build.get('disable', build.commit[1:])))
332 logging.info("...scanning version " + build.versionName)
333 # Prepare the source code...
334 common.prepare_source(vcs, app, build,
335 build_dir, srclib_dir,
338 count = scan_source(build_dir, build)
340 logging.warn(_('Scanner found {count} problems in {appid}:{versionCode}:')
341 .format(count=count, appid=appid, versionCode=build.versionCode))
344 except BuildException as be:
345 logging.warn("Could not scan app %s due to BuildException: %s" % (
348 except VCSException as vcse:
349 logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
352 logging.warn("Could not scan app %s due to unknown error: %s" % (
353 appid, traceback.format_exc()))
356 logging.info(_("Finished"))
357 print(_("%d problems found") % probcount)
360 if __name__ == "__main__":