chiark / gitweb /
~ianmdlvl / fdroidserver.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
scanner: support Gradle plugin 3.0 syntax for dependencies
[fdroidserver.git] / fdroidserver / scanner.py
diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py
index f768e6788a98d6f9a1cdc5d89f512df88d3a3ef4..272bebd224cbec354c63cc71060b17a05072395a 100644 (file)
--- a/fdroidserver/scanner.py
+++ b/fdroidserver/scanner.py
@@ -22,6 +22,7 @@ import traceback
 from argparse import ArgumentParser
 import logging
 
 from argparse import ArgumentParser
 import logging
 
+from . import _
 from . import common
 from . import metadata
 from .exception import BuildException, VCSException
 from . import common
 from . import metadata
 from .exception import BuildException, VCSException
@@ -31,7 +32,13 @@ options = None
 
 
 def get_gradle_compile_commands(build):
 
 
 def get_gradle_compile_commands(build):
-    compileCommands = ['compile', 'releaseCompile']
+    compileCommands = ['compile',          'releaseCompile'
+                       'provided',         'releaseProvided',
+                       'apk',              'releaseApk',
+                       'implementation',   'releaseImplementation',
+                       'api',              'releaseApi',
+                       'compileOnly',      'releaseCompileOnly',
+                       'runtimeOnly',      'releaseRuntimeOnly']
     if build.gradle and build.gradle != ['yes']:
         compileCommands += [flavor + 'Compile' for flavor in build.gradle]
         compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
     if build.gradle and build.gradle != ['yes']:
         compileCommands += [flavor + 'Compile' for flavor in build.gradle]
         compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
@@ -39,7 +46,7 @@ def get_gradle_compile_commands(build):
     return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
 
 
     return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
 
 
-def scan_source(build_dir, build):
+def scan_source(build_dir, build=metadata.Build()):
     """Scan the source code in the given directory (and all subdirectories)
     and return the number of fatal problems encountered
     """
     """Scan the source code in the given directory (and all subdirectories)
     and return the number of fatal problems encountered
     """
@@ -68,9 +75,18 @@ def scan_source(build_dir, build):
         ]
     }
 
         ]
     }
 
+    whitelisted = [
+        'firebase-jobdispatcher',  # https://github.com/firebase/firebase-jobdispatcher-android/blob/master/LICENSE
+        'com.firebaseui',          # https://github.com/firebase/FirebaseUI-Android/blob/master/LICENSE
+        'geofire-android'          # https://github.com/firebase/geofire-java/blob/master/LICENSE
+    ]
+
+    def is_whitelisted(s):
+        return any(wl in s for wl in whitelisted)
+
     def suspects_found(s):
         for n, r in usual_suspects.items():
     def suspects_found(s):
         for n, r in usual_suspects.items():
-            if r.match(s):
+            if r.match(s) and not is_whitelisted(s):
                 yield n
 
     gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?')
                 yield n
 
     gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?')
@@ -165,26 +181,26 @@ def scan_source(build_dir, build):
         return any(command.match(line) for command in gradle_compile_commands)
 
     # Iterate through all files in the source code
         return any(command.match(line) for command in gradle_compile_commands)
 
     # Iterate through all files in the source code
-    for dirpath, dirnames, filenames in os.walk(build_dir, topdown=True):
+    for root, dirs, files in os.walk(build_dir, topdown=True):
 
         # It's topdown, so checking the basename is enough
         for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
 
         # It's topdown, so checking the basename is enough
         for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
-            if ignoredir in dirnames:
-                dirnames.remove(ignoredir)
+            if ignoredir in dirs:
+                dirs.remove(ignoredir)
 
 
-        for curfile in filenames:
+        for curfile in files:
 
             if curfile in ['.DS_Store']:
                 continue
 
             # Path (relative) to the file
 
             if curfile in ['.DS_Store']:
                 continue
 
             # Path (relative) to the file
-            filepath = os.path.join(dirpath, curfile)
+            filepath = os.path.join(root, curfile)
 
             if os.path.islink(filepath):
                 continue
 
             path_in_build_dir = os.path.relpath(filepath, build_dir)
 
             if os.path.islink(filepath):
                 continue
 
             path_in_build_dir = os.path.relpath(filepath, build_dir)
-            _, ext = common.get_extension(path_in_build_dir)
+            _ignored, ext = common.get_extension(path_in_build_dir)
 
             if ext == 'so':
                 count += handleproblem('shared library', path_in_build_dir, filepath)
 
             if ext == 'so':
                 count += handleproblem('shared library', path_in_build_dir, filepath)
@@ -197,8 +213,11 @@ def scan_source(build_dir, build):
 
             elif ext == 'jar':
                 for name in suspects_found(curfile):
 
             elif ext == 'jar':
                 for name in suspects_found(curfile):
-                    count += handleproblem('usual supect \'%s\'' % name, path_in_build_dir, filepath)
-                warnproblem('JAR file', path_in_build_dir)
+                    count += handleproblem('usual suspect \'%s\'' % name, path_in_build_dir, filepath)
+                if curfile == 'gradle-wrapper.jar':
+                    removeproblem('gradle-wrapper.jar', path_in_build_dir, filepath)
+                else:
+                    warnproblem('JAR file', path_in_build_dir)
 
             elif ext == 'aar':
                 warnproblem('AAR file', path_in_build_dir)
 
             elif ext == 'aar':
                 warnproblem('AAR file', path_in_build_dir)
@@ -220,7 +239,7 @@ def scan_source(build_dir, build):
                 for i, line in enumerate(lines):
                     if is_used_by_gradle(line):
                         for name in suspects_found(line):
                 for i, line in enumerate(lines):
                     if is_used_by_gradle(line):
                         for name in suspects_found(line):
-                            count += handleproblem('usual supect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath)
+                            count += handleproblem('usual suspect \'%s\' at line %d' % (name, i + 1), path_in_build_dir, filepath)
                 noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
                 joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
                 for m in gradle_mavenrepo.finditer(joined):
                 noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
                 joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
                 for m in gradle_mavenrepo.finditer(joined):
@@ -256,7 +275,7 @@ def main():
     # Parse command line...
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
     # Parse command line...
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
-    parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
+    parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W
@@ -279,21 +298,27 @@ def main():
     for appid, app in apps.items():
 
         if app.Disabled:
     for appid, app in apps.items():
 
         if app.Disabled:
-            logging.info("Skipping %s: disabled" % appid)
-            continue
-        if not app.builds:
-            logging.info("Skipping %s: no builds specified" % appid)
+            logging.info(_("Skipping {appid}: disabled").format(appid=appid))
             continue
 
             continue
 
-        logging.info("Processing " + appid)
-
         try:
         try:
-
             if app.RepoType == 'srclib':
                 build_dir = os.path.join('build', 'srclib', app.Repo)
             else:
                 build_dir = os.path.join('build', appid)
 
             if app.RepoType == 'srclib':
                 build_dir = os.path.join('build', 'srclib', app.Repo)
             else:
                 build_dir = os.path.join('build', appid)
 
+            if app.builds:
+                logging.info(_("Processing {appid}").format(appid=appid))
+            else:
+                logging.info(_("{appid}: no builds specified, running on current source state")
+                             .format(appid=appid))
+                count = scan_source(build_dir)
+                if count > 0:
+                    logging.warn(_('Scanner found {count} problems in {appid}:')
+                                 .format(count=count, appid=appid))
+                    probcount += count
+                continue
+
             # Set up vcs interface and make sure we have the latest code...
             vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
 
             # Set up vcs interface and make sure we have the latest code...
             vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
 
@@ -302,20 +327,19 @@ def main():
                 if build.disable:
                     logging.info("...skipping version %s - %s" % (
                         build.versionName, build.get('disable', build.commit[1:])))
                 if build.disable:
                     logging.info("...skipping version %s - %s" % (
                         build.versionName, build.get('disable', build.commit[1:])))
-                else:
-                    logging.info("...scanning version " + build.versionName)
+                    continue
 
 
-                    # Prepare the source code...
-                    common.prepare_source(vcs, app, build,
-                                          build_dir, srclib_dir,
-                                          extlib_dir, False)
+                logging.info("...scanning version " + build.versionName)
+                # Prepare the source code...
+                common.prepare_source(vcs, app, build,
+                                      build_dir, srclib_dir,
+                                      extlib_dir, False)
 
 
-                    # Do the scan...
-                    count = scan_source(build_dir, build)
-                    if count > 0:
-                        logging.warn('Scanner found %d problems in %s (%s)' % (
-                            count, appid, build.versionCode))
-                        probcount += count
+                count = scan_source(build_dir, build)
+                if count > 0:
+                    logging.warn(_('Scanner found {count} problems in {appid}:{versionCode}:')
+                                 .format(count=count, appid=appid, versionCode=build.versionCode))
+                    probcount += count
 
         except BuildException as be:
             logging.warn("Could not scan app %s due to BuildException: %s" % (
 
         except BuildException as be:
             logging.warn("Could not scan app %s due to BuildException: %s" % (
@@ -329,8 +353,8 @@ def main():
                 appid, traceback.format_exc()))
             probcount += 1
 
                 appid, traceback.format_exc()))
             probcount += 1
 
-    logging.info("Finished:")
-    print("%d problems found" % probcount)
+    logging.info(_("Finished"))
+    print(_("%d problems found") % probcount)
 
 
 if __name__ == "__main__":
 
 
 if __name__ == "__main__":
Unnamed repository; edit this file 'description' to name the repository.