2 # -*- coding: utf-8 -*-
4 # scanner.py - part of the FDroid server tools
5 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 from argparse import ArgumentParser
28 from common import BuildException, VCSException
34 def get_gradle_compile_commands(thisbuild):
35 compileCommands = ['compile', 'releaseCompile']
36 if thisbuild['gradle'] and thisbuild['gradle'] != ['yes']:
37 compileCommands += [flavor + 'Compile' for flavor in thisbuild['gradle']]
38 compileCommands += [flavor + 'ReleaseCompile' for flavor in thisbuild['gradle']]
40 return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
43 # Scan the source code in the given directory (and all subdirectories)
44 # and return the number of fatal problems encountered
45 def scan_source(build_dir, root_dir, thisbuild):
49 # Common known non-free blobs (always lower case):
51 exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
55 r'admob.*sdk.*android',
58 r'google.*play.*services',
62 r'youtube.*android.*player.*api',
70 def suspects_found(s):
71 for n, r in usual_suspects.iteritems():
75 gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?')
77 allowed_repos = [re.compile(r'^https?://' + re.escape(repo) + r'/*') for repo in [
78 'repo1.maven.org/maven2', # mavenCentral()
79 'jcenter.bintray.com', # jcenter()
81 'repo.maven.apache.org/maven2',
82 'oss.sonatype.org/content/repositories/snapshots',
83 'oss.sonatype.org/content/repositories/releases',
84 'oss.sonatype.org/content/groups/public',
88 scanignore = common.getpaths_map(build_dir, thisbuild['scanignore'])
89 scandelete = common.getpaths_map(build_dir, thisbuild['scandelete'])
91 scanignore_worked = set()
92 scandelete_worked = set()
95 for k, paths in scanignore.iteritems():
98 scanignore_worked.add(k)
103 for k, paths in scandelete.iteritems():
106 scandelete_worked.add(k)
110 def ignoreproblem(what, fd, fp):
111 logging.info('Ignoring %s at %s' % (what, fd))
114 def removeproblem(what, fd, fp):
115 logging.info('Removing %s at %s' % (what, fd))
119 def warnproblem(what, fd):
122 logging.warn('Found %s at %s' % (what, fd))
124 def handleproblem(what, fd, fp):
126 return ignoreproblem(what, fd, fp)
128 return removeproblem(what, fd, fp)
129 logging.error('Found %s at %s' % (what, fd))
132 def is_executable(path):
133 return os.path.exists(path) and os.access(path, os.X_OK)
135 textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
139 with open(path, 'rb') as f:
141 return bool(d.translate(None, textchars))
143 # False positives patterns for files that are binary and executable.
144 safe_paths = [re.compile(r) for r in [
145 r".*/drawable[^/]*/.*\.png$", # png drawables
146 r".*/mipmap[^/]*/.*\.png$", # png mipmaps
151 for sp in safe_paths:
156 gradle_compile_commands = get_gradle_compile_commands(thisbuild)
158 def is_used_by_gradle(line):
159 return any(command.match(line) for command in gradle_compile_commands)
161 # Iterate through all files in the source code
162 for r, d, f in os.walk(build_dir, topdown=True):
164 # It's topdown, so checking the basename is enough
165 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
171 if curfile in ['.DS_Store']:
174 # Path (relative) to the file
175 fp = os.path.join(r, curfile)
177 if os.path.islink(fp):
180 fd = fp[len(build_dir) + 1:]
181 _, ext = common.get_extension(fd)
184 count += handleproblem('shared library', fd, fp)
186 count += handleproblem('static library', fd, fp)
188 count += handleproblem('Java compiled class', fd, fp)
190 removeproblem('APK file', fd, fp)
193 for name in suspects_found(curfile):
194 count += handleproblem('usual supect \'%s\'' % name, fd, fp)
195 warnproblem('JAR file', fd)
198 if not os.path.isfile(fp):
200 for line in file(fp):
201 if 'DexClassLoader' in line:
202 count += handleproblem('DexClassLoader', fd, fp)
205 elif ext == 'gradle':
206 if not os.path.isfile(fp):
208 with open(fp, 'r') as f:
209 lines = f.readlines()
210 for i, line in enumerate(lines):
211 if is_used_by_gradle(line):
212 for name in suspects_found(line):
213 count += handleproblem('usual supect \'%s\' at line %d' % (name, i+1), fd, fp)
214 noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
215 joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
216 for m in gradle_mavenrepo.finditer(joined):
218 if not any(r.match(url) for r in allowed_repos):
219 count += handleproblem('unknown maven repo \'%s\'' % url, fd, fp)
221 elif ext in ['', 'bin', 'out', 'exe']:
223 count += handleproblem('binary', fd, fp)
225 elif is_executable(fp):
226 if is_binary(fp) and not safe_path(fd):
227 warnproblem('possible binary', fd)
230 if p not in scanignore_worked:
231 logging.error('Unused scanignore path: %s' % p)
235 if p not in scandelete_worked:
236 logging.error('Unused scandelete path: %s' % p)
239 # Presence of a jni directory without buildjni=yes might
240 # indicate a problem (if it's not a problem, explicitly use
241 # buildjni=no to bypass this check)
242 if (os.path.exists(os.path.join(root_dir, 'jni')) and
243 not thisbuild['buildjni']):
244 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
252 global config, options
254 # Parse command line...
255 parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
256 common.setup_global_opts(parser)
257 parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
258 options = parser.parse_args()
260 config = common.read_config(options)
262 # Read all app and srclib metadata
263 allapps = metadata.read_metadata()
264 apps = common.read_app_args(options.appid, allapps, True)
269 if not os.path.isdir(build_dir):
270 logging.info("Creating build directory")
271 os.makedirs(build_dir)
272 srclib_dir = os.path.join(build_dir, 'srclib')
273 extlib_dir = os.path.join(build_dir, 'extlib')
275 for appid, app in apps.iteritems():
278 logging.info("Skipping %s: disabled" % appid)
280 if not app['builds']:
281 logging.info("Skipping %s: no builds specified" % appid)
284 logging.info("Processing " + appid)
288 if app['Repo Type'] == 'srclib':
289 build_dir = os.path.join('build', 'srclib', app['Repo'])
291 build_dir = os.path.join('build', appid)
293 # Set up vcs interface and make sure we have the latest code...
294 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
296 for thisbuild in app['builds']:
298 if thisbuild['disable']:
299 logging.info("...skipping version %s - %s" % (
300 thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:])))
302 logging.info("...scanning version " + thisbuild['version'])
304 # Prepare the source code...
305 root_dir, _ = common.prepare_source(vcs, app, thisbuild,
306 build_dir, srclib_dir,
310 count = scan_source(build_dir, root_dir, thisbuild)
312 logging.warn('Scanner found %d problems in %s (%s)' % (
313 count, appid, thisbuild['vercode']))
316 except BuildException as be:
317 logging.warn("Could not scan app %s due to BuildException: %s" % (
320 except VCSException as vcse:
321 logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
324 logging.warn("Could not scan app %s due to unknown error: %s" % (
325 appid, traceback.format_exc()))
328 logging.info("Finished:")
329 print "%d problems found" % probcount
331 if __name__ == "__main__":