3 # scanner.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
22 from argparse import ArgumentParser
26 from . import metadata
27 from .exception import BuildException, VCSException
33 def get_gradle_compile_commands(build):
34 compileCommands = ['compile', 'releaseCompile']
35 if build.gradle and build.gradle != ['yes']:
36 compileCommands += [flavor + 'Compile' for flavor in build.gradle]
37 compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
39 return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
42 def scan_source(build_dir, build):
43 """Scan the source code in the given directory (and all subdirectories)
44 and return the number of fatal problems encountered
49 # Common known non-free blobs (always lower case):
51 exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
55 r'admob.*sdk.*android',
58 r'google.*play.*services',
62 r'youtube.*android.*player.*api',
71 def suspects_found(s):
72 for n, r in usual_suspects.items():
76 gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?')
78 allowed_repos = [re.compile(r'^https?://' + re.escape(repo) + r'/*') for repo in [
79 'repo1.maven.org/maven2', # mavenCentral()
80 'jcenter.bintray.com', # jcenter()
82 'repo.maven.apache.org/maven2',
83 'oss.jfrog.org/artifactory/oss-snapshot-local',
84 'oss.sonatype.org/content/repositories/snapshots',
85 'oss.sonatype.org/content/repositories/releases',
86 'oss.sonatype.org/content/groups/public',
87 'clojars.org/repo', # Clojure free software libs
88 's3.amazonaws.com/repo.commonsware.com', # CommonsWare
89 'plugins.gradle.org/m2', # Gradle plugin repo
90 'maven.google.com', # Google Maven Repo, https://developer.android.com/studio/build/dependencies.html#google-maven
94 scanignore = common.getpaths_map(build_dir, build.scanignore)
95 scandelete = common.getpaths_map(build_dir, build.scandelete)
97 scanignore_worked = set()
98 scandelete_worked = set()
101 for k, paths in scanignore.items():
104 scanignore_worked.add(k)
109 for k, paths in scandelete.items():
112 scandelete_worked.add(k)
116 def ignoreproblem(what, fd):
117 logging.info('Ignoring %s at %s' % (what, fd))
120 def removeproblem(what, fd, fp):
121 logging.info('Removing %s at %s' % (what, fd))
125 def warnproblem(what, fd):
128 logging.warn('Found %s at %s' % (what, fd))
130 def handleproblem(what, fd, fp):
132 return ignoreproblem(what, fd)
134 return removeproblem(what, fd, fp)
135 logging.error('Found %s at %s' % (what, fd))
138 def is_executable(path):
139 return os.path.exists(path) and os.access(path, os.X_OK)
141 textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
145 with open(path, 'rb') as f:
147 return bool(d.translate(None, textchars))
149 # False positives patterns for files that are binary and executable.
150 safe_paths = [re.compile(r) for r in [
151 r".*/drawable[^/]*/.*\.png$", # png drawables
152 r".*/mipmap[^/]*/.*\.png$", # png mipmaps
157 for sp in safe_paths:
162 gradle_compile_commands = get_gradle_compile_commands(build)
164 def is_used_by_gradle(line):
165 return any(command.match(line) for command in gradle_compile_commands)
167 # Iterate through all files in the source code
168 for r, d, f in os.walk(build_dir, topdown=True):
170 # It's topdown, so checking the basename is enough
171 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
177 if curfile in ['.DS_Store']:
180 # Path (relative) to the file
181 fp = os.path.join(r, curfile)
183 if os.path.islink(fp):
186 fd = fp[len(build_dir) + 1:]
187 _, ext = common.get_extension(fd)
190 count += handleproblem('shared library', fd, fp)
192 count += handleproblem('static library', fd, fp)
194 count += handleproblem('Java compiled class', fd, fp)
196 removeproblem('APK file', fd, fp)
199 for name in suspects_found(curfile):
200 count += handleproblem('usual supect \'%s\'' % name, fd, fp)
201 warnproblem('JAR file', fd)
204 if not os.path.isfile(fp):
206 with open(fp, 'r', encoding='utf8', errors='replace') as f:
208 if 'DexClassLoader' in line:
209 count += handleproblem('DexClassLoader', fd, fp)
212 elif ext == 'gradle':
213 if not os.path.isfile(fp):
215 with open(fp, 'r', encoding='utf8', errors='replace') as f:
216 lines = f.readlines()
217 for i, line in enumerate(lines):
218 if is_used_by_gradle(line):
219 for name in suspects_found(line):
220 count += handleproblem('usual supect \'%s\' at line %d' % (name, i + 1), fd, fp)
221 noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
222 joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
223 for m in gradle_mavenrepo.finditer(joined):
225 if not any(r.match(url) for r in allowed_repos):
226 count += handleproblem('unknown maven repo \'%s\'' % url, fd, fp)
228 elif ext in ['', 'bin', 'out', 'exe']:
230 count += handleproblem('binary', fd, fp)
232 elif is_executable(fp):
233 if is_binary(fp) and not safe_path(fd):
234 warnproblem('possible binary', fd)
237 if p not in scanignore_worked:
238 logging.error('Unused scanignore path: %s' % p)
242 if p not in scandelete_worked:
243 logging.error('Unused scandelete path: %s' % p)
251 global config, options
253 # Parse command line...
254 parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
255 common.setup_global_opts(parser)
256 parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
257 metadata.add_metadata_arguments(parser)
258 options = parser.parse_args()
259 metadata.warnings_action = options.W
261 config = common.read_config(options)
263 # Read all app and srclib metadata
264 allapps = metadata.read_metadata()
265 apps = common.read_app_args(options.appid, allapps, True)
270 if not os.path.isdir(build_dir):
271 logging.info("Creating build directory")
272 os.makedirs(build_dir)
273 srclib_dir = os.path.join(build_dir, 'srclib')
274 extlib_dir = os.path.join(build_dir, 'extlib')
276 for appid, app in apps.items():
279 logging.info("Skipping %s: disabled" % appid)
282 logging.info("Skipping %s: no builds specified" % appid)
285 logging.info("Processing " + appid)
289 if app.RepoType == 'srclib':
290 build_dir = os.path.join('build', 'srclib', app.Repo)
292 build_dir = os.path.join('build', appid)
294 # Set up vcs interface and make sure we have the latest code...
295 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
297 for build in app.builds:
300 logging.info("...skipping version %s - %s" % (
301 build.versionName, build.get('disable', build.commit[1:])))
303 logging.info("...scanning version " + build.versionName)
305 # Prepare the source code...
306 common.prepare_source(vcs, app, build,
307 build_dir, srclib_dir,
311 count = scan_source(build_dir, build)
313 logging.warn('Scanner found %d problems in %s (%s)' % (
314 count, appid, build.versionCode))
317 except BuildException as be:
318 logging.warn("Could not scan app %s due to BuildException: %s" % (
321 except VCSException as vcse:
322 logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
325 logging.warn("Could not scan app %s due to unknown error: %s" % (
326 appid, traceback.format_exc()))
329 logging.info("Finished:")
330 print("%d problems found" % probcount)
333 if __name__ == "__main__":