prepare_report.py: Print some statistics about contracts and errors

Author: cameel

scripts/bytecodecompare/prepare_report.py

@@ -83,6 +83,33 @@ def format_summary(self, verbose: bool) -> str:
             return '.'
 
 
+@dataclass
+class Statistics:
+    file_count: int = 0
+    contract_count: int = 0
+    error_count: int = 0
+    missing_bytecode_count: int = 0
+    missing_metadata_count: int = 0
+
+    def aggregate(self, report: FileReport):
+        contract_reports = report.contract_reports if report.contract_reports is not None else []
+
+        self.file_count += 1
+        self.contract_count += len(contract_reports)
+        self.error_count += (1 if report.contract_reports is None else 0)
+        self.missing_bytecode_count += sum(1 for c in contract_reports if c.bytecode is None)
+        self.missing_metadata_count += sum(1 for c in contract_reports if c.metadata is None)
+
+    def __str__(self) -> str:
+        return "test cases: {}, contracts: {}, errors: {}, missing bytecode: {}, missing metadata: {}".format(
+            self.file_count,
+            str(self.contract_count) + ('+' if self.error_count > 0 else ''),
+            self.error_count,
+            self.missing_bytecode_count,
+            self.missing_metadata_count,
+        )
+
+
 def load_source(path: Union[Path, str], smt_use: SMTUse) -> str:
     # NOTE: newline='' disables newline conversion.
     # We want the file exactly as is because changing even a single byte in the source affects metadata.
@@ -292,7 +319,7 @@ def run_compiler(  # pylint: disable=too-many-arguments
         return parse_cli_output(Path(source_file_name), process.stdout)
 
 
-def generate_report(  # pylint: disable=too-many-arguments
+def generate_report(  # pylint: disable=too-many-arguments,too-many-locals
     source_file_names: List[str],
     compiler_path: Path,
     interface: CompilerInterface,
@@ -302,42 +329,49 @@ def generate_report(  # pylint: disable=too-many-arguments
     verbose: bool,
     exit_on_error: bool,
 ):
+    statistics = Statistics()
     metadata_option_supported = detect_metadata_cli_option_support(compiler_path)
 
-    with open(report_file_path, mode='w', encoding='utf8', newline='\n') as report_file:
-        for optimize in [False, True]:
-            with TemporaryDirectory(prefix='prepare_report-') as tmp_dir:
-                for source_file_name in sorted(source_file_names):
-                    try:
-                        report = run_compiler(
-                            compiler_path,
-                            Path(source_file_name),
-                            optimize,
-                            force_no_optimize_yul,
-                            interface,
-                            smt_use,
-                            metadata_option_supported,
-                            Path(tmp_dir),
-                            exit_on_error,
-                        )
-                        print(report.format_summary(verbose), end=('\n' if verbose else ''), flush=True)
-                        report_file.write(report.format_report())
-                    except subprocess.CalledProcessError as exception:
-                        print(
-                            f"\n\nInterrupted by an exception while processing file "
-                            f"'{source_file_name}' with optimize={optimize}\n\n"
-                            f"COMPILER STDOUT:\n{exception.stdout}\n"
-                            f"COMPILER STDERR:\n{exception.stderr}\n",
-                            file=sys.stderr
-                        )
-                        raise
-                    except:
-                        print(
-                            f"\n\nInterrupted by an exception while processing file "
-                            f"'{source_file_name}' with optimize={optimize}\n",
-                            file=sys.stderr
-                        )
-                        raise
+    try:
+        with open(report_file_path, mode='w', encoding='utf8', newline='\n') as report_file:
+            for optimize in [False, True]:
+                with TemporaryDirectory(prefix='prepare_report-') as tmp_dir:
+                    for source_file_name in sorted(source_file_names):
+                        try:
+                            report = run_compiler(
+                                compiler_path,
+                                Path(source_file_name),
+                                optimize,
+                                force_no_optimize_yul,
+                                interface,
+                                smt_use,
+                                metadata_option_supported,
+                                Path(tmp_dir),
+                                exit_on_error,
+                            )
+
+                            statistics.aggregate(report)
+                            print(report.format_summary(verbose), end=('\n' if verbose else ''), flush=True)
+
+                            report_file.write(report.format_report())
+                        except subprocess.CalledProcessError as exception:
+                            print(
+                                f"\n\nInterrupted by an exception while processing file "
+                                f"'{source_file_name}' with optimize={optimize}\n\n"
+                                f"COMPILER STDOUT:\n{exception.stdout}\n"
+                                f"COMPILER STDERR:\n{exception.stderr}\n",
+                                file=sys.stderr
+                            )
+                            raise
+                        except:
+                            print(
+                                f"\n\nInterrupted by an exception while processing file "
+                                f"'{source_file_name}' with optimize={optimize}\n",
+                                file=sys.stderr
+                            )
+                            raise
+    finally:
+        print('\n', statistics, '\n', sep='')
 
 
 def commandline_parser() -> ArgumentParser:

test/scripts/test_bytecodecompare_prepare_report.py

@@ -9,7 +9,7 @@
 
 # NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports
 # pragma pylint: disable=import-error
-from bytecodecompare.prepare_report import CompilerInterface, FileReport, ContractReport, SMTUse
+from bytecodecompare.prepare_report import CompilerInterface, FileReport, ContractReport, SMTUse, Statistics
 from bytecodecompare.prepare_report import load_source, parse_cli_output, parse_standard_json_output, prepare_compiler_input
 # pragma pylint: enable=import-error
 
@@ -99,6 +99,58 @@ def test_format_report_should_not_print_anything_if_contract_report_list_is_empt
 
         self.assertEqual(report.format_report(), '')
 
+class TestPrepareReport_Statistics(unittest.TestCase):
+    def test_initialization(self):
+        self.assertEqual(Statistics(), Statistics(0, 0, 0, 0, 0))
+
+    def test_aggregate_bytecode_and_metadata_present(self):
+        statistics = Statistics()
+        statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[ContractReport('C', 'c.sol', 'B', 'M')]))
+        self.assertEqual(statistics, Statistics(1, 1, 0, 0, 0))
+
+    def test_aggregate_bytecode_missing(self):
+        statistics = Statistics()
+        statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[ContractReport('C', 'c.sol', None, 'M')]))
+        self.assertEqual(statistics, Statistics(1, 1, 0, 1, 0))
+
+    def test_aggregate_metadata_missing(self):
+        statistics = Statistics()
+        statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[ContractReport('C', 'c.sol', 'B', None)]))
+        self.assertEqual(statistics, Statistics(1, 1, 0, 0, 1))
+
+    def test_aggregate_no_contract_reports(self):
+        statistics = Statistics()
+        statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[]))
+        self.assertEqual(statistics, Statistics(1, 0, 0, 0, 0))
+
+    def test_aggregate_missing_contract_report_list(self):
+        statistics = Statistics()
+        statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=None))
+        self.assertEqual(statistics, Statistics(1, 0, 1, 0, 0))
+
+    def test_aggregate_multiple_contract_reports(self):
+        statistics = Statistics()
+        statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[
+            ContractReport('C', 'c.sol', 'B', 'M'),
+            ContractReport('C', 'c.sol', None, 'M'),
+            ContractReport('C', 'c.sol', 'B', None),
+            ContractReport('C', 'c.sol', None, None),
+        ]))
+        self.assertEqual(statistics, Statistics(1, 4, 0, 2, 2))
+
+    def test_str(self):
+        statistics = Statistics()
+        statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[
+            ContractReport('C', 'c.sol', 'B', 'M'),
+            ContractReport('C', 'c.sol', None, 'M'),
+            ContractReport('C', 'c.sol', 'B', None),
+            ContractReport('C', 'c.sol', None, None),
+        ]))
+        statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=None))
+
+        self.assertEqual(statistics, Statistics(2, 4, 1, 2, 2))
+        self.assertEqual(str(statistics), "test cases: 2, contracts: 4+, errors: 1, missing bytecode: 2, missing metadata: 2")
+
 
 class TestLoadSource(PrepareReportTestBase):
     def test_load_source_should_strip_smt_pragmas_if_requested(self):