[WIP-feat] Extend Cppcheck functionality

This patch is a work in progress, there are debug prints all over the
place inside Codechecker, treat it accordingly. Will be amended, and
split up, when I am back from vacation.

Cppcheck now works in a raw directory instead of directly into the
workspace folder.
Exponential explosion of reports in plist files are now fixed.
Checker disable now works on native levels.
The `--disable <cppcheck_check>` now properly translates to `--suppress
<cppcheck_check> in the cppcheck invocation.
The current run configuration of cppcheck is `--enable=all`.

Added two new input paramaters:
With `cppcheck-addons` extra cppcheck checkers can be specified.
With `cppcheck-libraries` the cppcheck library definitions can be added.

Cppcheck reports have to be "fixed" for Codechecker to be able to
properly interpret them. The actual error message must be added to any
multistep report as a last bug path event.

Cppcheck checkers are prefixed with the `cppcheck-` prefix.

analyzer/codechecker_analyzer/analysis_manager.py

@@ -764,6 +764,11 @@ def signal_handler(signum, frame):
     if not os.path.exists(reproducer_dir) and generate_reproducer:
         os.makedirs(reproducer_dir)
 
+    # Cppcheck raw output directory
+    cppcheck_dir = os.path.join(output_path, "cppcheck")
+    if not os.path.exists(cppcheck_dir):
+        os.makedirs(cppcheck_dir)
+
     # Collect what other TUs were involved during CTU analysis.
     ctu_connections_dir = os.path.join(output_path, "ctu_connections")
     if not os.path.exists(ctu_connections_dir):

analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py

@@ -6,9 +6,11 @@
 """
 """
 from distutils.version import StrictVersion
+from pathlib import Path
 import os
 import re
 import shlex
+import shutil
 import subprocess
 import xml.etree.ElementTree as ET
 
@@ -22,6 +24,8 @@
 from .config_handler import CppcheckConfigHandler
 from .result_handler import CppcheckResultHandler
 
+from ..config_handler import CheckerState
+
 LOG = get_logger('analyzer.cppcheck')
 
 
@@ -36,6 +40,8 @@ def parse_checkers(cppcheck_output):
     errors = root.find('errors')
     for error in errors.findall('error'):
         name = error.attrib.get('id')
+        if name:
+            name = "cppcheck-" + name
         msg = error.attrib.get('msg')
         # TODO: Check severity handling in cppcheck
         # severity = error.attrib.get('severity')
@@ -88,19 +94,22 @@ def construct_analyzer_cmd(self, result_handler):
             # Enable or disable checkers.
             enabled_severity_levels = set()
             suppressed_checkers = set()
+
             for checker_name, value in config.checks().items():
-                if not value[0]:
+                if value[0] == CheckerState.disabled:
                     suppressed_checkers.add(checker_name)
-                # TODO: Check severity handling in cppcheck
-                # elif value.severity and value.severity != 'error':
-                #    enabled_severity_levels.add(value.severity)
+
+            # Cppcheck runs with all checkers enabled for the time being
+            # the unneded checkers are passed as suppressed checkers
+            analyzer_cmd.append('--enable=all')
 
             if enabled_severity_levels:
                 analyzer_cmd.append('--enable=' +
                                     ','.join(enabled_severity_levels))
 
             for checker_name in suppressed_checkers:
-                analyzer_cmd.append('--suppress=' + checker_name)
+                # TODO python3.9 removeprefix method is better than lstrip
+                analyzer_cmd.append('--suppress=' + checker_name.lstrip("cppcheck-"))
 
             # Add extra arguments.
             analyzer_cmd.extend(config.analyzer_extra_arguments)
@@ -110,13 +119,36 @@ def construct_analyzer_cmd(self, result_handler):
                 if analyzer_option.startswith("-I") or \
                     analyzer_option.startswith("-D"):
                     analyzer_cmd.extend([analyzer_option])
+                elif analyzer_option.startswith("-std"):
+                    standard = analyzer_option.split("=")[-1] \
+                        .lower().replace("gnu", "c")
+                    analyzer_cmd.extend(["--std=" + standard])
+
+            # TODO no platform translation eg. no cross platform analysis
+            analyzer_cmd.extend(["--platform=native"])
+
+            if 'cppcheck-addons' in config.analyzer_config:
+                for addon in config.analyzer_config["cppcheck-addons"]:
+                    analyzer_cmd.extend(["--addon=" + str(Path(addon).absolute())])
+                #addons = " ".join(config.analyzer_config["cppcheck-addons"])
+                #analyzer_cmd.extend(["--addon=" + addons])
+
+            if 'cppcheck-libraries' in config.analyzer_config:
+                for lib in config.analyzer_config["cppcheck-libraries"]:
+                    analyzer_cmd.extend(["--library=" + str(Path(lib).absolute())])
 
             # Cppcheck does not handle compiler includes well
             #for include in self.buildaction.compiler_includes:
-            #    print(include)
             #    analyzer_cmd.extend(['-I',  include])
 
-            analyzer_cmd.append('--plist-output=' + result_handler.workspace)
+            # TODO Suggest a better place for this
+            # cppcheck wont create the output folders for itself
+            output_dir = Path(result_handler.workspace, "cppcheck",
+                    result_handler.buildaction_hash)
+            output_dir.mkdir(exist_ok=True)
+
+            analyzer_cmd.append('--plist-output=' + str(output_dir))
+
             analyzer_cmd.append(self.source_file)
 
             return analyzer_cmd
@@ -158,16 +190,19 @@ def get_checker_config(cls, cfg_handler, environ):
 
     def post_analyze(self, result_handler):
         """
-        Renames the generated plist file with a unique name.
+        Copies the generated plist file with a unique name,
+
         """
         file_name = os.path.splitext(os.path.basename(self.source_file))[0]
-        output_file = os.path.join(result_handler.workspace,
+        cppcheck_out = os.path.join(result_handler.workspace, "cppcheck",
+                                    result_handler.buildaction_hash,
                                    file_name + '.plist')
-        if os.path.exists(output_file):
-            output = os.path.join(result_handler.workspace,
+        if os.path.exists(cppcheck_out):
+            codechecker_out = os.path.join(result_handler.workspace,
                                   result_handler.analyzer_result_file)
 
-            os.rename(output_file, output)
+            shutil.copy2(cppcheck_out, codechecker_out)
+            Path(cppcheck_out).rename(cppcheck_out + ".bak")
 
     @classmethod
     def resolve_missing_binary(cls, configured_binary, env):
@@ -243,6 +278,14 @@ def construct_config_handler(cls, args, context):
         handler.analyzer_binary = context.analyzer_binaries.get(
             cls.ANALYZER_NAME)
 
+        analyzer_config = {}
+
+        if "cppcheck_addons" in args:
+            analyzer_config['cppcheck-addons'] = args.cppcheck_addons
+        if "cppcheck_libraries" in args:
+            analyzer_config["cppcheck-libraries"] = args.cppcheck_libraries
+        handler.analyzer_config = analyzer_config
+
         check_env = extend(context.path_env_extra,
                            context.ld_lib_path_extra)
 

analyzer/codechecker_analyzer/analyzers/cppcheck/config_handler.py

@@ -7,10 +7,22 @@
 Config handler for Cppcheck analyzer.
 """
 from .. import config_handler
-
+from ..config_handler import CheckerState
 
 class CppcheckConfigHandler(config_handler.AnalyzerConfigHandler):
     """
     Configuration handler for Cppcheck analyzer.
     """
-    pass
+    def initialize_checkers(self, analyzer_context, checkers, cmdline_enable=..., enable_all=False):
+        """
+        Set all the default checkers to disabled. This will ensure that
+        --enable=all will not run with all the possible checkers
+        """
+        super().initialize_checkers(analyzer_context, checkers, cmdline_enable, enable_all)
+
+        # Set all the default checkers to disabled. This will ensure that
+        # --enable=all will not run with all the possible checkers
+        for checker_name, data in self.checks().items():
+            if data[0] == CheckerState.default:
+                self.set_checker_enabled(checker_name, enabled = False)
+        return

analyzer/codechecker_analyzer/analyzers/cppcheck/result_handler.py

@@ -7,6 +7,7 @@
 Result handler for Cppcheck.
 """
 from typing import Optional
+from pprint import pprint
 
 from codechecker_report_converter.report.parser.base import AnalyzerInfo
 from codechecker_report_converter.analyzers.cppcheck.analyzer_result import \
@@ -17,6 +18,8 @@
 from codechecker_common.logger import get_logger
 from codechecker_common.skiplist_handler import SkipListHandlers
 
+from .config_handler import CppcheckConfigHandler
+
 from ..result_handler_base import ResultHandler
 
 LOG = get_logger('analyzer.cppcheck')
@@ -39,8 +42,11 @@ def postprocess_result(self, skip_handlers: Optional[SkipListHandlers]):
         """
         LOG.debug_analyzer(self.analyzer_stdout)
 
-        reports = AnalyzerResult().get_reports(self.workspace)
+        reports = AnalyzerResult().get_reports(self.analyzer_result_file)
         reports = [r for r in reports if not r.skip(skip_handlers)]
+        for report in reports:
+            if not report.checker_name.startswith("cppcheck-"):
+                report.checker_name = "cppcheck-" + report.checker_name
 
         hash_type = HashType.PATH_SENSITIVE
         if self.report_hash_type == 'context-free-v2':
@@ -54,3 +60,4 @@ def postprocess_result(self, skip_handlers: Optional[SkipListHandlers]):
         report_file.create(
             self.analyzer_result_file, reports, self.checker_labels,
             self.analyzer_info)
+

analyzer/codechecker_analyzer/analyzers/result_handler_base.py

@@ -59,6 +59,7 @@ def __init__(self, action, workspace, report_hash_type=None):
         self.skiplist_handler = None
         self.analyzed_source_file = None
         self.analyzer_returncode = 1
+        self.buildaction_hash = ''
         self.__buildaction = action
 
         self.__result_file = None
@@ -133,9 +134,12 @@ def analyzer_action_str(self):
 
         build_info = source_file + '_' + ' '.join(args)
 
+        self.buildaction_hash = \
+            hashlib.md5(build_info.encode(errors='ignore')).hexdigest()
+
         return analyzed_file_name + '_' + \
             str(self.buildaction.analyzer_type) + '_' + \
-            hashlib.md5(build_info.encode(errors='ignore')).hexdigest()
+            self.buildaction_hash
 
     @property
     def analyzer_result_file(self):

analyzer/codechecker_analyzer/cmd/analyze.py

@@ -406,6 +406,20 @@ def add_arguments_to_parser(parser):
                                     "'CodeChecker analyzers --dump-config "
                                     "clang-tidy' command.")
 
+    analyzer_opts.add_argument('--cppcheck-addons',
+                               dest='cppcheck_addons',
+                               required=False,
+                               nargs='*',
+                               default=argparse.SUPPRESS,
+                               help="A list of cppcheck addon files.")
+
+    analyzer_opts.add_argument('--cppcheck-libraries',
+                               dest='cppcheck_libraries',
+                               required=False,
+                               nargs='*',
+                               default=argparse.SUPPRESS,
+                               help="A list of cppcheck library definiton files.")
+
     analyzer_opts.add_argument('--analyzer-config',
                                dest='analyzer_config',
                                nargs='*',

analyzer/codechecker_analyzer/cmd/check.py

@@ -354,6 +354,20 @@ def add_arguments_to_parser(parser):
                                     "'CodeChecker analyzers --dump-config "
                                     "clang-tidy' command.")
 
+    analyzer_opts.add_argument('--cppcheck-addons',
+                               dest='cppcheck_addons',
+                               required=False,
+                               nargs='*',
+                               default=argparse.SUPPRESS,
+                               help="A list of cppcheck addon files.")
+
+    analyzer_opts.add_argument('--cppcheck-libraries',
+                               dest='cppcheck_libraries',
+                               required=False,
+                               nargs='*',
+                               default=argparse.SUPPRESS,
+                               help="A list of cppcheck library definiton files.")
+
     analyzer_opts.add_argument('--analyzer-config',
                                dest='analyzer_config',
                                nargs='*',
@@ -820,6 +834,8 @@ def __update_if_key_exists(source, target, key):
                           'clangsa_args_cfg_file',
                           'tidy_args_cfg_file',
                           'tidy_config',
+                          'cppcheck_addons',
+                          'cppcheck_libraries',
                           'analyzer_config',
                           'checker_config',
                           'capture_analysis_output',

codechecker_common/cli.py

@@ -93,8 +93,9 @@ def main():
     """
     CodeChecker main command line.
     """
-    os.environ['CC_LIB_DIR'] = os.path.dirname(os.path.dirname(
-        os.path.realpath(__file__)))
+    if not os.environ.get('CC_LIB_DIR'):
+        os.environ['CC_LIB_DIR'] = \
+                os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
     data_files_dir_path = get_data_files_dir_path()
     os.environ['CC_DATA_FILES_DIR'] = data_files_dir_path