From ea1d3c0c85aafc4b36f79e36fd4f3a6982e20ede Mon Sep 17 00:00:00 2001 From: vodorok Date: Mon, 14 Oct 2024 14:46:41 +0200 Subject: [PATCH] Add single plist option This patch adds the logic of report output merging into one singluar `.plist` file. Now analyze and check can be invoked with the `--plist-file-name` option, which if provided will merge all new `.plist` result files into the specified file name under the specified output report folder. --- .../codechecker_analyzer/analysis_manager.py | 32 ++++++++++++++++--- analyzer/codechecker_analyzer/analyzer.py | 1 + analyzer/codechecker_analyzer/cmd/analyze.py | 9 ++++++ analyzer/codechecker_analyzer/cmd/check.py | 9 ++++++ .../report/parser/plist.py | 25 +++++++++------ 5 files changed, 62 insertions(+), 14 deletions(-) diff --git a/analyzer/codechecker_analyzer/analysis_manager.py b/analyzer/codechecker_analyzer/analysis_manager.py index 6b22ca4231..fed7965f21 100644 --- a/analyzer/codechecker_analyzer/analysis_manager.py +++ b/analyzer/codechecker_analyzer/analysis_manager.py @@ -17,11 +17,14 @@ import traceback import zipfile +from pathlib import Path from threading import Timer import multiprocess import psutil +import plistlib + from codechecker_common.logger import get_logger from codechecker_common.review_status_handler import ReviewStatusHandler @@ -57,8 +60,29 @@ def print_analyzer_statistic_summary(metadata_analyzers, status, msg=None): LOG.info(" %s: %s", analyzer_type, res) -def worker_result_handler(results, metadata_tool, output_path): - """ Print the analysis summary. """ +def worker_result_handler(results, metadata_tool, output_path, plist_file_name): + """ + Handle analysis results after all the analyzer threads returned. It may + merge all the plist output files into one, and print the analysis summary. + """ + LOG.info("Merging plist files into %s", plist_file_name) + if plist_file_name: + plist_data = [] + single_plist = Path(output_path, plist_file_name) + for _, _, _, _, original_plist, _ in results: + original_plist = Path(original_plist) + if os.path.exists(original_plist): + with open(original_plist, 'rb') as plist: + LOG.debug(f"Merging original plist {original_plist}") + plist_data.append(plistlib.load(plist)) + + with open(single_plist, 'wb') as plist: + LOG.debug(f"Dumping merged plist file into {single_plist}") + plistlib.dump(plist_data, plist) + + LOG.debug(f"Removing original plist file {original_plist}") + original_plist.unlink() + skipped_num = 0 reanalyzed_num = 0 metadata_analyzers = metadata_tool['analyzers'] @@ -719,7 +743,7 @@ def skip_cpp(compile_actions, skip_handlers): def start_workers(actions_map, actions, analyzer_config_map, - jobs, output_path, skip_handlers, + jobs, output_path, plist_file_name, skip_handlers, rs_handler: ReviewStatusHandler, metadata_tool, quiet_analyze, capture_analysis_output, generate_reproducer, timeout, ctu_reanalyze_on_failure, statistics_data, manager, @@ -813,7 +837,7 @@ def signal_handler(signum, _): analyzed_actions, 1, callback=lambda results: worker_result_handler( - results, metadata_tool, output_path) + results, metadata_tool, output_path, plist_file_name) ).get(timeout) pool.close() diff --git a/analyzer/codechecker_analyzer/analyzer.py b/analyzer/codechecker_analyzer/analyzer.py index 9b83037315..666e7e0947 100644 --- a/analyzer/codechecker_analyzer/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzer.py @@ -332,6 +332,7 @@ def perform_analysis(args, skip_handlers, rs_handler: ReviewStatusHandler, analysis_manager.start_workers(actions_map, actions, config_map, args.jobs, args.output_path, + args.plist_file_name, skip_handlers, rs_handler, metadata_tool, diff --git a/analyzer/codechecker_analyzer/cmd/analyze.py b/analyzer/codechecker_analyzer/cmd/analyze.py index 4ebf0b9ee9..04ff26403c 100644 --- a/analyzer/codechecker_analyzer/cmd/analyze.py +++ b/analyzer/codechecker_analyzer/cmd/analyze.py @@ -212,6 +212,15 @@ def add_arguments_to_parser(parser): default=argparse.SUPPRESS, help="Store the analysis output in the given folder.") + parser.add_argument('--plist-file-name', + type=str, + dest="plist_file_name", + required=False, + help="If given, all the `.plist` files containing " + "the analyzer result files will be merged " + "into a single `.plist` file in the report " + "output folder given by `-o/--output`.") + parser.add_argument('--compiler-info-file', dest="compiler_info_file", required=False, diff --git a/analyzer/codechecker_analyzer/cmd/check.py b/analyzer/codechecker_analyzer/cmd/check.py index e654755f8f..c2381c4bbc 100644 --- a/analyzer/codechecker_analyzer/cmd/check.py +++ b/analyzer/codechecker_analyzer/cmd/check.py @@ -111,6 +111,15 @@ def add_arguments_to_parser(parser): "If it is not given then the results go into a " "temporary directory which will be removed after " "the analysis.") + + parser.add_argument('--plist-file-name', + type=str, + dest="plist_file_name", + required=False, + help="If given, all the `.plist` files containing " + "the analyzer result files will be merged " + "into a single `.plist` file in the report " + "output folder given by `-o/--output`.") parser.add_argument('-t', '--type', '--output-format', dest="output_format", diff --git a/tools/report-converter/codechecker_report_converter/report/parser/plist.py b/tools/report-converter/codechecker_report_converter/report/parser/plist.py index 3f79878f06..471b079ec6 100644 --- a/tools/report-converter/codechecker_report_converter/report/parser/plist.py +++ b/tools/report-converter/codechecker_report_converter/report/parser/plist.py @@ -206,20 +206,25 @@ def get_reports( if not plist: return reports - metadata = plist.get('metadata') + if not isinstance(plist, list): + plist=[plist] - files = get_file_index_map( - plist, source_dir_path, self._file_cache) + for sub_plist in plist: - for diag in plist.get('diagnostics', []): - report = self.__create_report( - analyzer_result_file_path, diag, files, metadata) + metadata = sub_plist.get('metadata') - if report.report_hash is None: - report.report_hash = get_report_hash( - report, HashType.PATH_SENSITIVE) + files = get_file_index_map( + sub_plist, source_dir_path, self._file_cache) + + for diag in sub_plist.get('diagnostics', []): + report = self.__create_report( + analyzer_result_file_path, diag, files, metadata) + + if report.report_hash is None: + report.report_hash = get_report_hash( + report, HashType.PATH_SENSITIVE) - reports.append(report) + reports.append(report) except KeyError as ex: LOG.warning("Failed to get file path id! Found files: %s. " "KeyError: %s", files, ex)