Refactor post scan plugins so they can have command line options

Signed-off-by: Haiko Schol <ext-haiko.schol@here.com>

setup.py

@@ -222,8 +222,8 @@ def read(*names, **kwargs):
         # becomes the ScanCode CLI boolean flag used to enable a
         # given post_scan plugin
         'scancode_post_scan': [
-            'only-findings = scancode.plugin_only_findings:process_only_findings',
-            'mark-source = scancode.plugin_mark_source:process_mark_source',
+            'only-findings = scancode.plugin_only_findings:OnlyFindings',
+            'mark-source = scancode.plugin_mark_source:MarkSource',
         ],
 
         # scancode_pre_scan is an entry point to define pre_scan plugins.

src/plugincode/post_scan.py

@@ -38,14 +38,30 @@
 
 
 @post_scan_spec
-def post_scan_handler(active_scans, results):
+class PostScanPlugin(object):
     """
-    Process the scanned files and yield the modified results.
-    Parameters:
-     - `active_scans`: a list of scanners names requested in the current run.
-     - `results`: an iterable of scan results for each file or directory.
+    A post-scan plugin layout class to be extended by the post_scan plugins.
     """
-    pass
+
+    def __init__(self, option, user_input):
+        self.option = option
+        self.user_input = user_input
+
+    def process_results(self, results, active_scans):
+        """
+        Process the scan results.
+        results - an iterable of resources
+        active_scans - iterable of scanners that were used to obtain the results (e.g. "copyrights", "licenses")
+        """
+        return results
+
+    @staticmethod
+    def get_click_options():
+        """
+        Return an iterable of `click.Option` objects to be
+        used for calling the plugin.
+        """
+        return ()
 
 
 post_scan_plugins = PluginManager('post_scan')
@@ -57,6 +73,9 @@ def initialize():
     NOTE: this defines the entry points for use in setup.py
     """
     post_scan_plugins.load_setuptools_entrypoints('scancode_post_scan')
+    for name, plugin in get_post_scan_plugins().items():
+        if not issubclass(plugin, PostScanPlugin):
+            raise Exception('Invalid post-scan plugin "%(name)s": does not extend "plugincode.post_scan.PostScanPlugin".' % locals())
 
 
 def get_post_scan_plugins():

src/scancode/cli.py

@@ -499,14 +499,15 @@ def scancode(ctx,
 
         has_requested_post_scan_plugins = False
 
-        for option, post_scan_handler in plugincode.post_scan.get_post_scan_plugins().items():
-            is_requested = kwargs[option.replace('-', '_')]
-            if is_requested:
-                options['--' + option] = True
-                if not quiet:
-                    echo_stderr('Running post-scan plugin: %(option)s...' % locals(), fg='green')
-                results = post_scan_handler(active_scans, results)
-                has_requested_post_scan_plugins = True
+        for name, plugin in plugincode.post_scan.get_post_scan_plugins().items():
+            for option in plugin.get_click_options():
+                user_input = kwargs[option.name]
+                if user_input:
+                    options['--' + name] = user_input
+                    if not quiet:
+                        echo_stderr('Running post-scan plugin: %(option)s...' % locals(), fg='green')
+                    results = plugin(option.name, user_input).process_results(results, active_scans)
+                    has_requested_post_scan_plugins = True
 
         if has_requested_post_scan_plugins:
             # FIXME: computing len needs a list and therefore needs loading it all ahead of time

src/scancode/plugin_mark_source.py

@@ -28,43 +28,51 @@
 
 from os import path
 
+from click import Option
+
+from plugincode.post_scan import PostScanPlugin
 from plugincode.post_scan import post_scan_impl
 
 
 @post_scan_impl
-def process_mark_source(active_scans, results):
+class MarkSource(PostScanPlugin):
     """
     Set the "is_source" flag to true for directories that contain
     over 90% of source files as direct children.
     Has no effect unless the --info scan is requested.
     """
 
-    # FIXME: this is forcing all the scan results to be loaded in memory
-    # and defeats lazy loading from cache
-    results = list(results)
+    def process_results(self, results, _):
+        # FIXME: this is forcing all the scan results to be loaded in memory
+        # and defeats lazy loading from cache
+        results = list(results)
+
+        # FIXME: we should test for active scans instead, but "info" may not
+        # be present for now. check if the first item has a file info.
+        has_file_info = 'type' in results[0]
 
-    # FIXME: we should test for active scans instead, but "info" may not
-    # be present for now. check if the first item has a file info.
-    has_file_info = 'type' in results[0]
+        if not has_file_info:
+            # just yield results untouched
+            for scanned_file in results:
+                yield scanned_file
+            return
 
-    if not has_file_info:
-        # just yield results untouched
+        # FIXME: this is an nested loop, looping twice on results
+        # TODO: this may not recusrively roll up the is_source flag, as we
+        # may not iterate bottom up.
         for scanned_file in results:
+            if scanned_file['type'] == 'directory' and scanned_file['files_count'] > 0:
+                source_files_count = 0
+                for scanned_file2 in results:
+                    if path.dirname(scanned_file2['path']) == scanned_file['path']:
+                        if scanned_file2['is_source']:
+                            source_files_count += 1
+                mark_source(source_files_count, scanned_file)
             yield scanned_file
-        return
 
-    # FIXME: this is an nested loop, looping twice on results
-    # TODO: this may not recusrively roll up the is_source flag, as we
-    # may not iterate bottom up.
-    for scanned_file in results:
-        if scanned_file['type'] == 'directory' and scanned_file['files_count'] > 0:
-            source_files_count = 0
-            for scanned_file2 in results:
-                if path.dirname(scanned_file2['path']) == scanned_file['path']:
-                    if scanned_file2['is_source']:
-                        source_files_count += 1
-            mark_source(source_files_count, scanned_file)
-        yield scanned_file
+    @staticmethod
+    def get_click_options():
+        return [Option(('--mark-source',), is_flag=True)]
 
 
 def mark_source(source_files_count, scanned_file):

src/scancode/plugin_only_findings.py

@@ -25,25 +25,33 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+from click import Option
+
+from plugincode.post_scan import PostScanPlugin
 from plugincode.post_scan import post_scan_impl
 
 
 @post_scan_impl
-def process_only_findings(active_scans, results):
+class OnlyFindings(PostScanPlugin):
     """
     Only return files or directories with findings for the requested
     scans. Files and directories without findings are omitted (not
     considering basic file information as findings).
     """
 
-    # FIXME: this is forcing all the scan results to be loaded in memory
-    # and defeats lazy loading from cache. Only a different caching
-    # (e.g. DB) could work here.
-    # FIXME: We should instead use a generator or use a filter function
-    # that pass to the scan results loader iterator
-    for scanned_file in results:
-        if has_findings(active_scans, scanned_file):
-            yield scanned_file
+    def process_results(self, results, active_scans):
+        # FIXME: this is forcing all the scan results to be loaded in memory
+        # and defeats lazy loading from cache. Only a different caching
+        # (e.g. DB) could work here.
+        # FIXME: We should instead use a generator or use a filter function
+        # that pass to the scan results loader iterator
+        for scanned_file in results:
+            if has_findings(active_scans, scanned_file):
+                yield scanned_file
+
+    @staticmethod
+    def get_click_options():
+        return [Option(('--only-findings',), is_flag=True)]
 
 
 def has_findings(active_scans, scanned_file):