Bytecode comparison via CLI interface

.circleci/config.yml

@@ -925,13 +925,19 @@ jobs:
       - checkout
       - attach_workspace:
           at: build
-      - run: scripts/bytecodecompare/storebytecode.sh && cp -v report.txt bytecode-report-ubuntu.txt
+      - run: mkdir test-cases/
+      - run: cd test-cases && ../scripts/isolate_tests.py ../test/
+      - run: cd test-cases && ../scripts/bytecodecompare/prepare_report.py ../build/solc/solc --interface standard-json && mv -v report.txt ../bytecode-report-ubuntu-json.txt
+      - run: cd test-cases && ../scripts/bytecodecompare/prepare_report.py ../build/solc/solc --interface cli && mv -v report.txt ../bytecode-report-ubuntu-cli.txt
       - store_artifacts:
-          path: report.txt
+          path: bytecode-report-ubuntu-json.txt
+      - store_artifacts:
+          path: bytecode-report-ubuntu-cli.txt
       - persist_to_workspace:
           root: .
           paths:
-            - bytecode-report-ubuntu.txt
+            - bytecode-report-ubuntu-json.txt
+            - bytecode-report-ubuntu-cli.txt
 
   b_bytecode_osx:
     macos:
@@ -942,13 +948,19 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - run: scripts/bytecodecompare/storebytecode.sh && cp -v report.txt bytecode-report-osx.txt
+      - run: mkdir test-cases/
+      - run: cd test-cases && ../scripts/isolate_tests.py ../test/
+      - run: cd test-cases && ../scripts/bytecodecompare/prepare_report.py ../build/solc/solc --interface standard-json && mv -v report.txt ../bytecode-report-osx-json.txt
+      - run: cd test-cases && ../scripts/bytecodecompare/prepare_report.py ../build/solc/solc --interface cli && mv -v report.txt ../bytecode-report-osx-cli.txt
+      - store_artifacts:
+          path: bytecode-report-osx-json.txt
       - store_artifacts:
-          path: report.txt
+          path: bytecode-report-osx-cli.txt
       - persist_to_workspace:
           root: .
           paths:
-            - bytecode-report-osx.txt
+            - bytecode-report-osx-json.txt
+            - bytecode-report-osx-cli.txt
 
   b_bytecode_win:
     executor:
@@ -961,15 +973,19 @@ jobs:
       - checkout
       - attach_workspace:
           at: build
-      - run: python scripts\isolate_tests.py test\
-      - run: python scripts\bytecodecompare\prepare_report.py build\solc\Release\solc.exe
-      - run: cp report.txt bytecode-report-windows.txt
+      - run: mkdir test-cases\
+      - run: cd test-cases\ && python ..\scripts\isolate_tests.py ..\test\
+      - run: cd test-cases\ && python ..\scripts\bytecodecompare\prepare_report.py ..\build\solc\Release\solc.exe --interface standard-json && move report.txt ..\bytecode-report-windows-json.txt
+      - run: cd test-cases\ && python ..\scripts\bytecodecompare\prepare_report.py ..\build\solc\Release\solc.exe --interface cli && move report.txt ..\bytecode-report-windows-cli.txt
+      - store_artifacts:
+          path: bytecode-report-windows-json.txt
       - store_artifacts:
-          path: report.txt
+          path: bytecode-report-windows-cli.txt
       - persist_to_workspace:
           root: .
           paths:
-            - bytecode-report-windows.txt
+            - bytecode-report-windows-json.txt
+            - bytecode-report-windows-cli.txt
 
   b_bytecode_ems:
     docker:
@@ -980,9 +996,9 @@ jobs:
       - checkout
       - attach_workspace:
           at: emscripten_build/libsolc
-      - run: scripts/bytecodecompare/storebytecode.sh && cp -v report.txt bytecode-report-emscripten.txt
+      - run: scripts/bytecodecompare/storebytecode.sh && mv -v report.txt bytecode-report-emscripten.txt
       - store_artifacts:
-          path: report.txt
+          path: bytecode-report-emscripten.txt
       - persist_to_workspace:
           root: .
           paths:
@@ -994,7 +1010,15 @@ jobs:
     steps:
       - attach_workspace:
           at: .
-      - run: diff --report-identical-files --from-file bytecode-report-emscripten.txt bytecode-report-ubuntu.txt bytecode-report-osx.txt bytecode-report-windows.txt
+      - run: |
+          diff --report-identical-files --from-file \
+            bytecode-report-emscripten.txt \
+            bytecode-report-ubuntu-json.txt \
+            bytecode-report-ubuntu-cli.txt \
+            bytecode-report-osx-json.txt \
+            bytecode-report-osx-cli.txt \
+            bytecode-report-windows-json.txt \
+            bytecode-report-windows-cli.txt
 
 workflows:
   version: 2

scripts/bytecodecompare/prepare_report.js

@@ -24,10 +24,25 @@ for (const optimize of [false, true])
 
             const result = JSON.parse(compiler.compile(JSON.stringify(input)))
 
+            let internalCompilerError = false
+            if ('errors' in result)
+            {
+                for (const error of result['errors'])
+                    // JSON interface still returns contract metadata in case of an internal compiler error while
+                    // CLI interface does not. To make reports comparable we must force this case to be detected as
+                    // an error in both cases.
+                    if (['UnimplementedFeatureError', 'CompilerError', 'CodeGenerationError'].includes(error['type']))
+                    {
+                        internalCompilerError = true
+                        break
+                    }
+            }
+
             if (
                 !('contracts' in result) ||
                 Object.keys(result['contracts']).length === 0 ||
-                Object.keys(result['contracts']).every(file => Object.keys(result['contracts'][file]).length === 0)
+                Object.keys(result['contracts']).every(file => Object.keys(result['contracts'][file]).length === 0) ||
+                internalCompilerError
             )
                 // NOTE: do not exit here because this may be run on source which cannot be compiled
                 console.log(filename + ': <ERROR>')

scripts/bytecodecompare/prepare_report.py

@@ -3,13 +3,26 @@
 import sys
 import subprocess
 import json
+import re
 from argparse import ArgumentParser
 from dataclasses import dataclass
+from enum import Enum
 from glob import glob
 from pathlib import Path
+from tempfile import TemporaryDirectory
 from typing import List, Optional, Tuple, Union
 
 
+CONTRACT_SEPARATOR_PATTERN = re.compile(r'^======= (?P<file_name>.+):(?P<contract_name>[^:]+) =======$', re.MULTILINE)
+BYTECODE_REGEX = re.compile(r'^Binary:\n(?P<bytecode>.*)$', re.MULTILINE)
+METADATA_REGEX = re.compile(r'^Metadata:\n(?P<metadata>\{.*\})$', re.MULTILINE)
+
+
+class CompilerInterface(Enum):
+    CLI = 'cli'
+    STANDARD_JSON = 'standard-json'
+
+
 @dataclass(frozen=True)
 class ContractReport:
     contract_name: str
@@ -53,10 +66,19 @@ def load_source(path: Union[Path, str]) -> str:
 def parse_standard_json_output(source_file_name: Path, standard_json_output: str) -> FileReport:
     decoded_json_output = json.loads(standard_json_output.strip())
 
+    # JSON interface still returns contract metadata in case of an internal compiler error while
+    # CLI interface does not. To make reports comparable we must force this case to be detected as
+    # an error in both cases.
+    internal_compiler_error = any(
+        error['type'] in ['UnimplementedFeatureError', 'CompilerError', 'CodeGenerationError']
+        for error in decoded_json_output.get('errors', {})
+    )
+
     if (
         'contracts' not in decoded_json_output or
         len(decoded_json_output['contracts']) == 0 or
-        all(len(file_results) == 0 for file_name, file_results in decoded_json_output['contracts'].items())
+        all(len(file_results) == 0 for file_name, file_results in decoded_json_output['contracts'].items()) or
+        internal_compiler_error
     ):
         return FileReport(file_name=source_file_name, contract_reports=None)
 
@@ -74,62 +96,145 @@ def parse_standard_json_output(source_file_name: Path, standard_json_output: str
     return file_report
 
 
-def prepare_compiler_input(compiler_path: Path, source_file_name: Path, optimize: bool) -> Tuple[List[str], str]:
-    json_input: dict = {
-        'language': 'Solidity',
-        'sources': {
-            str(source_file_name): {'content': load_source(source_file_name)}
-        },
-        'settings': {
-            'optimizer': {'enabled': optimize},
-            'outputSelection': {'*': {'*': ['evm.bytecode.object', 'metadata']}},
-            'modelChecker': {'engine': 'none'},
-        }
-    }
+def parse_cli_output(source_file_name: Path, cli_output: str) -> FileReport:
+    # re.split() returns a list containing the text between pattern occurrences but also inserts the
+    # content of matched groups in between. It also never omits the empty elements so the number of
+    # list items is predictable (3 per match + the text before the first match)
+    output_segments = re.split(CONTRACT_SEPARATOR_PATTERN, cli_output)
+    assert len(output_segments) % 3 == 1
 
-    command_line = [str(compiler_path), '--standard-json']
-    compiler_input = json.dumps(json_input)
+    if len(output_segments) == 1:
+        return FileReport(file_name=source_file_name, contract_reports=None)
 
-    return (command_line, compiler_input)
+    file_report = FileReport(file_name=source_file_name, contract_reports=[])
+    for file_name, contract_name, contract_output in zip(output_segments[1::3], output_segments[2::3], output_segments[3::3]):
+        bytecode_match = re.search(BYTECODE_REGEX, contract_output)
+        metadata_match = re.search(METADATA_REGEX, contract_output)
+
+        assert file_report.contract_reports is not None
+        file_report.contract_reports.append(ContractReport(
+            contract_name=contract_name,
+            file_name=Path(file_name),
+            bytecode=bytecode_match['bytecode'] if bytecode_match is not None else None,
+            metadata=metadata_match['metadata'] if metadata_match is not None else None,
+        ))
 
+    return file_report
 
-def run_compiler(compiler_path: Path, source_file_name: Path, optimize: bool) -> FileReport:
-    (command_line, compiler_input) = prepare_compiler_input(compiler_path, Path(Path(source_file_name).name), optimize)
 
-    process = subprocess.run(
-        command_line,
-        input=compiler_input,
-        encoding='utf8',
-        capture_output=True,
-        check=False,
-    )
+def prepare_compiler_input(
+    compiler_path: Path,
+    source_file_name: Path,
+    optimize: bool,
+    interface: CompilerInterface
+) -> Tuple[List[str], str]:
+
+    if interface == CompilerInterface.STANDARD_JSON:
+        json_input: dict = {
+            'language': 'Solidity',
+            'sources': {
+                str(source_file_name): {'content': load_source(source_file_name)}
+            },
+            'settings': {
+                'optimizer': {'enabled': optimize},
+                'outputSelection': {'*': {'*': ['evm.bytecode.object', 'metadata']}},
+                'modelChecker': {'engine': 'none'},
+            }
+        }
+
+        command_line = [str(compiler_path), '--standard-json']
+        compiler_input = json.dumps(json_input)
+    else:
+        assert interface == CompilerInterface.CLI
 
-    return parse_standard_json_output(Path(source_file_name), process.stdout)
+        compiler_options = [str(source_file_name), '--bin', '--metadata', '--model-checker-engine', 'none']
+        if optimize:
+            compiler_options.append('--optimize')
 
+        command_line = [str(compiler_path)] + compiler_options
+        compiler_input = load_source(source_file_name)
 
-def generate_report(source_file_names: List[str], compiler_path: Path):
+    return (command_line, compiler_input)
+
+
+def run_compiler(
+    compiler_path: Path,
+    source_file_name: Path,
+    optimize: bool,
+    interface: CompilerInterface,
+    tmp_dir: Path,
+) -> FileReport:
+
+    if interface == CompilerInterface.STANDARD_JSON:
+        (command_line, compiler_input) = prepare_compiler_input(
+            compiler_path,
+            Path(source_file_name.name),
+            optimize,
+            interface
+        )
+
+        process = subprocess.run(
+            command_line,
+            input=compiler_input,
+            encoding='utf8',
+            capture_output=True,
+            check=False,
+        )
+
+        return parse_standard_json_output(Path(source_file_name), process.stdout)
+    else:
+        assert interface == CompilerInterface.CLI
+        assert tmp_dir is not None
+
+        (command_line, compiler_input) = prepare_compiler_input(
+            compiler_path.absolute(),
+            Path(source_file_name.name),
+            optimize,
+            interface
+        )
+
+        # Create a copy that we can use directly with the CLI interface
+        modified_source_path = tmp_dir / source_file_name.name
+        # NOTE: newline='' disables newline conversion.
+        # We want the file exactly as is because changing even a single byte in the source affects metadata.
+        with open(modified_source_path, 'w', encoding='utf8', newline='') as modified_source_file:
+            modified_source_file.write(compiler_input)
+
+        process = subprocess.run(
+            command_line,
+            cwd=tmp_dir,
+            encoding='utf8',
+            capture_output=True,
+            check=False,
+        )
+
+        return parse_cli_output(Path(source_file_name), process.stdout)
+
+
+def generate_report(source_file_names: List[str], compiler_path: Path, interface: CompilerInterface):
     with open('report.txt', mode='w', encoding='utf8', newline='\n') as report_file:
         for optimize in [False, True]:
-            for source_file_name in sorted(source_file_names):
-                try:
-                    report = run_compiler(Path(compiler_path), Path(source_file_name), optimize)
-                    report_file.write(report.format_report())
-                except subprocess.CalledProcessError as exception:
-                    print(
-                        f"\n\nInterrupted by an exception while processing file "
-                        f"'{source_file_name}' with optimize={optimize}\n\n"
-                        f"COMPILER STDOUT:\n{exception.stdout}\n"
-                        f"COMPILER STDERR:\n{exception.stderr}\n",
-                        file=sys.stderr
-                    )
-                    raise
-                except:
-                    print(
-                        f"\n\nInterrupted by an exception while processing file "
-                        f"'{source_file_name}' with optimize={optimize}\n",
-                        file=sys.stderr
-                    )
-                    raise
+            with TemporaryDirectory(prefix='prepare_report-') as tmp_dir:
+                for source_file_name in sorted(source_file_names):
+                    try:
+                        report = run_compiler(compiler_path, Path(source_file_name), optimize, interface, Path(tmp_dir))
+                        report_file.write(report.format_report())
+                    except subprocess.CalledProcessError as exception:
+                        print(
+                            f"\n\nInterrupted by an exception while processing file "
+                            f"'{source_file_name}' with optimize={optimize}\n\n"
+                            f"COMPILER STDOUT:\n{exception.stdout}\n"
+                            f"COMPILER STDERR:\n{exception.stderr}\n",
+                            file=sys.stderr
+                        )
+                        raise
+                    except:
+                        print(
+                            f"\n\nInterrupted by an exception while processing file "
+                            f"'{source_file_name}' with optimize={optimize}\n",
+                            file=sys.stderr
+                        )
+                        raise
 
 
 def commandline_parser() -> ArgumentParser:
@@ -140,6 +245,13 @@ def commandline_parser() -> ArgumentParser:
 
     parser = ArgumentParser(description=script_description)
     parser.add_argument(dest='compiler_path', help="Solidity compiler executable")
+    parser.add_argument(
+        '--interface',
+        dest='interface',
+        default=CompilerInterface.STANDARD_JSON.value,
+        choices=[c.value for c in CompilerInterface],
+        help="Compiler interface to use."
+    )
     return parser;
 
 
@@ -148,4 +260,5 @@ def commandline_parser() -> ArgumentParser:
     generate_report(
         glob("*.sol"),
         Path(options.compiler_path),
+        CompilerInterface(options.interface),
     )

test/scripts/fixtures/code_generation_error_cli_output.txt

@@ -0,0 +1,8 @@
+Warning: SPDX license identifier not provided in source file. Before publishing, consider adding a comment containing "SPDX-License-Identifier: <SPDX-License>" to each source file. Use "SPDX-License-Identifier: UNLICENSED" for non-open-source code. Please see https://spdx.org for more information.
+--> test_1c3426238b8296745d8d8bd0ff995ab65a51992b568dc7c5ce73c3f59b107825_no_assignments_sol.sol
+
+Warning: Source file does not specify required compiler version! Consider adding "pragma solidity ^0.8.0;"
+--> test_1c3426238b8296745d8d8bd0ff995ab65a51992b568dc7c5ce73c3f59b107825_no_assignments_sol.sol
+
+Error: Some immutables were read from but never assigned, possibly because of optimization.
+