From 289aa7ed2389b74c778e41442392d670ca313c5e Mon Sep 17 00:00:00 2001 From: antazoey Date: Mon, 18 Dec 2023 19:15:18 -0600 Subject: [PATCH] feat: upgrade to ape 0.7 (#122) --- .mdformat.toml | 1 - .pre-commit-config.yaml | 4 +- README.md | 6 +- ape_solidity/_utils.py | 49 ++---- ape_solidity/compiler.py | 232 +++++++++++++++++++---------- pyproject.toml | 3 + setup.py | 18 +-- tests/ape-config.yaml | 3 + tests/conftest.py | 26 +++- tests/contracts/RandomVyperFile.vy | 2 +- tests/contracts/SpacesInPragma.sol | 2 +- tests/test_compiler.py | 198 +++++++++++++----------- 12 files changed, 320 insertions(+), 224 deletions(-) delete mode 100644 .mdformat.toml diff --git a/.mdformat.toml b/.mdformat.toml deleted file mode 100644 index 01b2fb0..0000000 --- a/.mdformat.toml +++ /dev/null @@ -1 +0,0 @@ -number = true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96157e2..517ed44 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 23.11.0 + rev: 23.12.0 hooks: - id: black name: black @@ -21,7 +21,7 @@ repos: - id: flake8 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.0 + rev: v1.7.1 hooks: - id: mypy additional_dependencies: [types-requests, types-setuptools, pydantic, types-pkg-resources] diff --git a/README.md b/README.md index 7ab59bf..4dfa726 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,8 @@ Use the `add_library()` method from the `ape-solidity` compiler class to add the A typical flow is: 1. Deploy the library. -2. Call `add_library()` using the Solidity compiler plugin, which will also re-compile contracts that need the library. -3. Deploy and use contracts that require the library. +1. Call `add_library()` using the Solidity compiler plugin, which will also re-compile contracts that need the library. +1. Deploy and use contracts that require the library. For example: @@ -111,7 +111,7 @@ def contract(accounts, project, compilers): # Deploy the library. account = accounts[0] library = project.Set.deploy(sender=account) - + # Add the library to Solidity (re-compiles contracts that use the library). compilers.solidity.add_library(library) diff --git a/ape_solidity/_utils.py b/ape_solidity/_utils.py index 237ec71..ba2e5d5 100644 --- a/ape_solidity/_utils.py +++ b/ape_solidity/_utils.py @@ -5,13 +5,13 @@ from pathlib import Path from typing import Dict, List, Optional, Sequence, Set, Union -from ape._pydantic_compat import BaseModel, validator from ape.exceptions import CompilerError -from ape.logging import logger +from ape.utils import pragma_str_to_specifier_set from packaging.specifiers import SpecifierSet from packaging.version import InvalidVersion from packaging.version import Version from packaging.version import Version as _Version +from pydantic import BaseModel, field_validator from solcx.install import get_executable from solcx.wrapper import get_solc_version as get_solc_version_from_binary @@ -36,7 +36,8 @@ class ImportRemapping(BaseModel): entry: str packages_cache: Path - @validator("entry") + @field_validator("entry", mode="before") + @classmethod def validate_entry(cls, value): if len((value or "").split("=")) != 2: raise IncorrectMappingFormatError() @@ -93,6 +94,8 @@ def package_id(self) -> Path: class ImportRemappingBuilder: def __init__(self, contracts_cache: Path): + # import_map maps import keys like `@openzeppelin/contracts` + # to str paths in the contracts' .cache folder. self.import_map: Dict[str, str] = {} self.dependencies_added: Set[Path] = set() self.contracts_cache = contracts_cache @@ -145,7 +148,7 @@ def get_pragma_spec_from_path(source_file_path: Union[Path, str]) -> Optional[Sp source_file_path (Union[Path, str]): Solidity source file path. Returns: - ``packaging.specifiers.SpecifierSet`` + ``Optional[packaging.specifiers.SpecifierSet]`` """ path = Path(source_file_path) if not path.is_file(): @@ -163,41 +166,7 @@ def get_pragma_spec_from_str(source_str: str) -> Optional[SpecifierSet]: ): return None # Try compiling with latest - # The following logic handles the case where the user puts a space - # between the operator and the version number in the pragma string, - # such as `solidity >= 0.4.19 < 0.7.0`. - pragma_parts = pragma_match.groups()[0].split() - - def _to_spec(item: str) -> str: - item = item.replace("^", "~=") - if item and item[0].isnumeric(): - return f"=={item}" - elif item and len(item) >= 2 and item[0] == "=" and item[1] != "=": - return f"={item}" - - return item - - pragma_parts_fixed = [] - builder = "" - for sub_part in pragma_parts: - if not any(c.isnumeric() for c in sub_part): - # Handle pragma with spaces between constraint and values - # like `>= 0.6.0`. - builder += sub_part - continue - elif builder: - spec = _to_spec(f"{builder}{sub_part}") - builder = "" - else: - spec = _to_spec(sub_part) - - pragma_parts_fixed.append(spec) - - try: - return SpecifierSet(",".join(pragma_parts_fixed)) - except ValueError as err: - logger.error(str(err)) - return None + return pragma_str_to_specifier_set(pragma_match.groups()[0]) def load_dict(data: Union[str, dict]) -> Dict: @@ -215,7 +184,7 @@ def add_commit_hash(version: Union[str, Version]) -> Version: return get_solc_version_from_binary(solc, with_commit_hash=True) -def verify_contract_filepaths(contract_filepaths: List[Path]) -> Set[Path]: +def verify_contract_filepaths(contract_filepaths: Sequence[Path]) -> Set[Path]: invalid_files = [p.name for p in contract_filepaths if p.suffix != Extension.SOL.value] if not invalid_files: return set(contract_filepaths) diff --git a/ape_solidity/compiler.py b/ape_solidity/compiler.py index 99e6c02..ba62a95 100644 --- a/ape_solidity/compiler.py +++ b/ape_solidity/compiler.py @@ -1,7 +1,7 @@ import os import re from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union, cast +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Type, Union, cast from ape.api import CompilerAPI, PluginConfig from ape.contracts import ContractInstance @@ -9,10 +9,11 @@ from ape.logging import logger from ape.types import AddressType, ContractType from ape.utils import cached_property, get_relative_path +from eth_pydantic_types import HexBytes from eth_utils import add_0x_prefix, is_0x_prefixed -from ethpm_types import ASTNode, HexBytes, PackageManifest +from ethpm_types import ASTNode, PackageManifest from ethpm_types.ast import ASTClassification -from ethpm_types.source import Content +from ethpm_types.source import Compiler, Content from packaging.specifiers import SpecifierSet from packaging.version import Version from pkg_resources import get_distribution @@ -56,10 +57,9 @@ IMPORTS_PATTERN = re.compile( r"import\s+((.*?)(?=;)|[\s\S]*?from\s+(.*?)(?=;));\s", flags=re.MULTILINE ) - LICENSES_PATTERN = re.compile(r"(// SPDX-License-Identifier:\s*([^\n]*)\s)") - VERSION_PRAGMA_PATTERN = re.compile(r"pragma solidity[^;]*;") +DEFAULT_OPTIMIZATION_RUNS = 200 class SolidityConfig(PluginConfig): @@ -69,7 +69,7 @@ class SolidityConfig(PluginConfig): optimize: bool = True version: Optional[str] = None evm_version: Optional[str] = None - via_ir: bool = False + via_ir: Optional[bool] = None class SolidityCompiler(CompilerAPI): @@ -194,7 +194,7 @@ def add_library(self, *contracts: ContractInstance): self._contracts_needing_libraries = set() - def get_versions(self, all_paths: List[Path]) -> Set[str]: + def get_versions(self, all_paths: Sequence[Path]) -> Set[str]: versions = set() for path in all_paths: # Make sure we have the compiler available to compile this @@ -206,21 +206,30 @@ def get_versions(self, all_paths: List[Path]) -> Set[str]: def get_import_remapping(self, base_path: Optional[Path] = None) -> Dict[str, str]: """ - Specify the remapping using a ``=`` separated str - e.g. ``'@import_name=path/to/dependency'``. + Config remappings like ``'@import_name=path/to/dependency'`` parsed here + as ``{'@import_name': 'path/to/dependency'}``. + + Returns: + Dict[str, str]: Where the key is the import name, e.g. ``"@openzeppelin/contracts"` + and the value is a stringified relative path (source ID) of the cached contract, + e.g. `".cache/OpenZeppelin/v4.4.2". """ base_path = base_path or self.project_manager.contracts_folder - remappings = self.settings.import_remapping - if not remappings: + if not (remappings := self.settings.import_remapping): return {} elif not isinstance(remappings, (list, tuple)) or not isinstance(remappings[0], str): raise IncorrectMappingFormatError() + # We use these helpers to transform the values configured + # to values matching files in the `contracts/.cache` folder. contracts_cache = base_path / ".cache" builder = ImportRemappingBuilder(contracts_cache) + packages_cache = self.config_manager.packages_folder - # Convert to tuple for hashing, check if there's been a change + # Here we hash and validate if there were changes to remappings. + # If there were, we continue, else return the cached value for + # performance reasons. remappings_tuple = tuple(remappings) if ( self._import_remapping_hash @@ -229,11 +238,11 @@ def get_import_remapping(self, base_path: Optional[Path] = None) -> Dict[str, st ): return self._cached_import_map - packages_cache = self.config_manager.packages_folder - - # Download dependencies for first time. - # This only happens if calling this method before compiling in ape core. - dependencies = self.project_manager.dependencies + # NOTE: Dependencies are only extracted if calling for the first. + # Likely, this was done already before this point, unless + # calling python methods manually. However, dependencies MUST be + # fully loaded to properly evaluate remapping paths. + dependencies = self.project_manager.load_dependencies() for item in remappings: remapping_obj = ImportRemapping(entry=item, packages_cache=packages_cache) @@ -242,9 +251,17 @@ def get_import_remapping(self, base_path: Optional[Path] = None) -> Dict[str, st # Handle missing version ID if len(package_id.parts) == 1: - if package_id.name in dependencies and len(dependencies[package_id.name]) == 1: - version_id = next(iter(dependencies[package_id.name])) - package_id = package_id / version_id + if package_id.name not in dependencies or len(dependencies[package_id.name]) == 0: + logger.warning(f"Missing dependency '{package_id.name}'.") + continue + + elif len(dependencies[package_id.name]) != 1: + logger.warning("version ID missing and unable to evaluate version.") + continue + + version_id = next(iter(dependencies[package_id.name])) + logger.debug(f"for {package_id.name} version ID missing, using {version_id}") + package_id = package_id / version_id data_folder_cache = packages_cache / package_id @@ -256,7 +273,7 @@ def get_import_remapping(self, base_path: Optional[Path] = None) -> Dict[str, st logger.debug(f"Unable to find dependency '{package_id}'.") else: - manifest = PackageManifest.parse_file(cached_manifest_file) + manifest = PackageManifest.model_validate_json(cached_manifest_file.read_text()) self._add_dependencies(manifest, sub_contracts_cache, builder) # Update cache and hash @@ -353,7 +370,7 @@ def _add_dependencies( dependency_path = dependency_root_path / version / f"{dependency_name}.json" if dependency_path.is_file(): - sub_manifest = PackageManifest.parse_file(dependency_path) + sub_manifest = PackageManifest.model_validate_json(dependency_path.read_text()) dep_id = Path(dependency_name) / version if dep_id not in builder.dependencies_added: builder.dependencies_added.add(dep_id) @@ -364,7 +381,7 @@ def _add_dependencies( ) def get_compiler_settings( - self, contract_filepaths: List[Path], base_path: Optional[Path] = None + self, contract_filepaths: Sequence[Path], base_path: Optional[Path] = None ) -> Dict[Version, Dict]: base_path = base_path or self.config_manager.contracts_folder files_by_solc_version = self.get_version_map(contract_filepaths, base_path=base_path) @@ -375,50 +392,60 @@ def get_compiler_settings( settings: Dict = {} for solc_version, sources in files_by_solc_version.items(): version_settings: Dict[str, Union[Any, List[Any]]] = { - "optimizer": {"enabled": self.settings.optimize, "runs": 200}, + "optimizer": {"enabled": self.settings.optimize, "runs": DEFAULT_OPTIMIZATION_RUNS}, "outputSelection": { str(get_relative_path(p, base_path)): {"*": OUTPUT_SELECTION, "": ["ast"]} for p in sources }, } - remappings_used = set() - if import_remappings: - # Filter out unused import remapping - resolved_remapped_sources = set( - [ - x - for ls in self.get_imports(list(sources), base_path=base_path).values() - for x in ls - if x.startswith(".cache") - ] - ) - for source in resolved_remapped_sources: - parent_key = os.path.sep.join(source.split(os.path.sep)[:3]) - for k, v in [(k, v) for k, v in import_remappings.items() if parent_key in v]: - remappings_used.add(f"{k}={v}") + if remappings_used := self._get_used_remappings( + sources, remappings=import_remappings, base_path=base_path + ): + remappings_str = [f"{k}={v}" for k, v in remappings_used.items()] - if remappings_used: # Standard JSON input requires remappings to be sorted. - version_settings["remappings"] = sorted(list(remappings_used)) + version_settings["remappings"] = sorted(remappings_str) - evm_version = self.settings.evm_version - if evm_version: + if evm_version := self.settings.evm_version: version_settings["evmVersion"] = evm_version - if solc_version >= Version("0.7.5"): + if solc_version >= Version("0.7.5") and self.settings.via_ir is not None: version_settings["viaIR"] = self.settings.via_ir settings[solc_version] = version_settings # TODO: Filter out libraries that are not used for this version. - libs = self.libraries - if libs: + if libs := self.libraries: version_settings["libraries"] = libs return settings + def _get_used_remappings( + self, sources, remappings: Dict[str, str], base_path: Optional[Path] = None + ) -> Dict[str, str]: + base_path = base_path or self.project_manager.contracts_folder + remappings = remappings or self.get_import_remapping(base_path=base_path) + if not remappings: + # No remappings used at all. + return {} + + # Filter out unused import remapping. + return { + k: v + for source in ( + x + for sources in self.get_imports(list(sources), base_path=base_path).values() + for x in sources + if x.startswith(".cache") + ) + for parent_key in ( + os.path.sep.join(source.split(os.path.sep)[:3]) for source in [source] + ) + for k, v in [(k, v) for k, v in remappings.items() if parent_key in v] + } + def get_standard_input_json( - self, contract_filepaths: List[Path], base_path: Optional[Path] = None + self, contract_filepaths: Sequence[Path], base_path: Optional[Path] = None ) -> Dict[Version, Dict]: base_path = base_path or self.config_manager.contracts_folder files_by_solc_version = self.get_version_map(contract_filepaths, base_path=base_path) @@ -455,12 +482,13 @@ def get_standard_input_json( return input_jsons def compile( - self, contract_filepaths: List[Path], base_path: Optional[Path] = None + self, contract_filepaths: Sequence[Path], base_path: Optional[Path] = None ) -> List[ContractType]: base_path = base_path or self.config_manager.contracts_folder - solc_versions_by_contract_name: Dict[str, Version] = {} + contract_versions: Dict[str, Version] = {} contract_types: List[ContractType] = [] input_jsons = self.get_standard_input_json(contract_filepaths, base_path=base_path) + for solc_version, input_json in input_jsons.items(): logger.info(f"Compiling using Solidity compiler '{solc_version}'.") cleaned_version = Version(solc_version.base_version) @@ -479,6 +507,11 @@ def compile( raise SolcCompileError(err) from err contracts = output.get("contracts", {}) + + # Perf back-out. + if not contracts: + continue + input_contract_names: List[str] = [] for source_id, contracts_out in contracts.items(): for name, _ in contracts_out.items(): @@ -497,7 +530,7 @@ def classify_ast(_node: ASTNode): for source_id, contracts_out in contracts.items(): ast_data = output["sources"][source_id]["ast"] - ast = ASTNode.parse_obj(ast_data) + ast = ASTNode.model_validate(ast_data) classify_ast(ast) for contract_name, ct_data in contracts_out.items(): @@ -524,16 +557,18 @@ def classify_ast(_node: ASTNode): self._contracts_needing_libraries.add(contract_path) continue - previously_compiled_version = solc_versions_by_contract_name.get(contract_name) - if previously_compiled_version: - # Don't add previously compiled contract type unless it was compiled - # using a greater Solidity version. - if previously_compiled_version >= solc_version: + if previous_version := contract_versions.get(contract_name, None): + if previous_version < solc_version: + # Keep the smallest version for max compat. continue + else: + # Remove the previously compiled contract types and re-compile. contract_types = [ ct for ct in contract_types if ct.name != contract_name ] + if contract_name in contract_versions: + del contract_versions[contract_name] ct_data["contractName"] = contract_name ct_data["sourceId"] = str( @@ -545,9 +580,36 @@ def classify_ast(_node: ASTNode): ct_data["devdoc"] = load_dict(ct_data["devdoc"]) ct_data["sourcemap"] = evm_data["bytecode"]["sourceMap"] ct_data["ast"] = ast - contract_type = ContractType.parse_obj(ct_data) + contract_type = ContractType.model_validate(ct_data) contract_types.append(contract_type) - solc_versions_by_contract_name[contract_name] = solc_version + contract_versions[contract_name] = solc_version + + # Output compiler data used. + compilers_used: Dict[Version, Compiler] = {} + for ct in contract_types: + if not ct.name: + # Won't happen, but just for mypy. + continue + + vers = contract_versions[ct.name] + settings = input_jsons[vers]["settings"] + if vers in compilers_used and ct.name not in (compilers_used[vers].contractTypes or []): + compilers_used[vers].contractTypes = [ + *(compilers_used[vers].contractTypes or []), + ct.name, + ] + + elif vers not in compilers_used: + compilers_used[vers] = Compiler( + name=self.name.lower(), + version=f"{vers}", + contractTypes=[ct.name], + settings=settings, + ) + + # First, output compiler information to manifest. + compilers_ls = list(compilers_used.values()) + self.project_manager.local_project.add_compiler_data(compilers_ls) return contract_types @@ -566,7 +628,7 @@ def compile_code( else: if selected_version := select_version(pragma, self.available_versions): version = selected_version - install_solc(version, show_progress=False) + install_solc(version, show_progress=True) else: raise SolcInstallError() @@ -574,7 +636,7 @@ def compile_code( version = latest_installed elif latest := self.latest_version: - install_solc(latest, show_progress=False) + install_solc(latest, show_progress=True) version = latest else: @@ -609,7 +671,7 @@ def compile_code( def _get_unmapped_imports( self, - contract_filepaths: List[Path], + contract_filepaths: Sequence[Path], base_path: Optional[Path] = None, ) -> Dict[str, List[Tuple[str, str]]]: contracts_path = base_path or self.config_manager.contracts_folder @@ -639,37 +701,45 @@ def _get_unmapped_imports( def get_imports( self, - contract_filepaths: List[Path], + contract_filepaths: Sequence[Path], base_path: Optional[Path] = None, ) -> Dict[str, List[str]]: - # NOTE: Process import remappings _before_ getting the full contract set. contracts_path = base_path or self.config_manager.contracts_folder - import_remapping = self.get_import_remapping(base_path=contracts_path) - contract_filepaths_set = verify_contract_filepaths(contract_filepaths) - imports_dict: Dict[str, List[str]] = {} - for src_path, import_strs in get_import_lines(contract_filepaths_set).items(): - import_set = set() - for import_str in import_strs: - import_item = _import_str_to_source_id( - import_str, src_path, contracts_path, import_remapping - ) - import_set.add(import_item) + def build_map(paths: Set[Path], prev: Optional[Dict] = None) -> Dict[str, List[str]]: + result: Dict[str, List[str]] = prev or {} - source_id = str(get_relative_path(src_path, contracts_path)) - imports_dict[str(source_id)] = list(import_set) + for src_path, import_strs in get_import_lines(paths).items(): + source_id = str(get_relative_path(src_path, contracts_path)) + if source_id in result: + continue - return imports_dict + import_set = { + _import_str_to_source_id(import_str, src_path, contracts_path, import_remapping) + for import_str in import_strs + } + result[source_id] = list(import_set) + + # Add imports of imports. + import_paths = {contracts_path / p for p in import_set if p not in result} + result = {**result, **build_map(import_paths, prev=result)} + + return result + + # NOTE: Process import remapping list _before_ getting the full contract set. + import_remapping = self.get_import_remapping(base_path=contracts_path) + contract_filepaths_set = verify_contract_filepaths(contract_filepaths) + return build_map(contract_filepaths_set) def get_version_map( self, - contract_filepaths: Union[Path, List[Path]], + contract_filepaths: Union[Path, Sequence[Path]], base_path: Optional[Path] = None, ) -> Dict[Version, Set[Path]]: # Ensure `.cache` folder is built before getting version map. - _ = self.get_import_remapping(base_path=base_path) + self.get_import_remapping(base_path=base_path) - if not isinstance(contract_filepaths, (list, tuple)): + if not isinstance(contract_filepaths, Sequence): contract_filepaths = [contract_filepaths] base_path = base_path or self.project_manager.contracts_folder @@ -704,7 +774,7 @@ def get_version_map( and not any(source_by_pragma_spec.values()) and (latest := self.latest_version) ): - install_solc(latest, show_progress=False) + install_solc(latest, show_progress=True) # Adjust best-versions based on imports. files_by_solc_version: Dict[Version, Set[Path]] = {} @@ -798,7 +868,7 @@ def _get_pramga_spec_from_str(self, source_str: str) -> Optional[SpecifierSet]: return pragma_spec elif compiler_version := select_version(pragma_spec, self.available_versions): - install_solc(compiler_version, show_progress=False) + install_solc(compiler_version, show_progress=True) else: # Attempt to use the best-installed version. @@ -925,7 +995,7 @@ def flatten_contract(self, path: Path, **kwargs) -> Content: source = "\n".join([pragma, source]) lines = source.splitlines() line_dict = {i + 1: line for i, line in enumerate(lines)} - return Content(__root__=line_dict) + return Content(root=line_dict) def remove_imports(flattened_contract: str) -> str: diff --git a/pyproject.toml b/pyproject.toml index d5ac261..10d07d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,3 +38,6 @@ force_grid_wrap = 0 include_trailing_comma = true multi_line_output = 3 use_parentheses = true + +[tool.mdformat] +number = true diff --git a/setup.py b/setup.py index d7e3067..b4506b6 100644 --- a/setup.py +++ b/setup.py @@ -5,27 +5,26 @@ extras_require = { "test": [ # `test` GitHub Action jobs uses this "pytest>=6.0", # Core testing package - "pytest-xdist", # multi-process runner + "pytest-xdist", # Multi-process runner "pytest-cov", # Coverage analyzer plugin "pytest-mock", # For creating and using mocks "hypothesis>=6.2.0,<7.0", # Strategy-based fuzzer ], "lint": [ - "black>=23.11.0,<24", # Auto-formatter and linter - "mypy>=1.7.0,<2", # Static type analyzer - "types-requests", # Needed due to mypy typeshed - "types-setuptools", # Needed due to mypy typeshed + "black>=23.12.0,<24", # Auto-formatter and linter + "mypy>=1.7.1,<2", # Static type analyzer + "types-requests", # Needed for mypy type shed + "types-setuptools", # Needed for mypy type shed "types-pkg-resources", # Needed for type checking tests "flake8>=6.1.0,<7", # Style linter "isort>=5.10.1,<6", # Import sorting linter "mdformat>=0.7.17", # Auto-formatter for markdown "mdformat-gfm>=0.3.5", # Needed for formatting GitHub-flavored markdown "mdformat-frontmatter>=0.4.1", # Needed for frontmatters-style headers in issue templates - "pydantic<2.0", # Needed for successful type check. TODO: Remove after full v2 support. ], "doc": [ - "Sphinx>=3.4.3,<4", # Documentation generator - "sphinx_rtd_theme>=0.1.9,<1", # Readthedocs.org theme + "Sphinx>=6.1.3,<7", # Documentation generator + "sphinx_rtd_theme>=1.2.0,<2", # Readthedocs.org theme "towncrier>=19.2.0, <20", # Generate release notes ], "release": [ # `release` GitHub Action job uses this @@ -68,8 +67,9 @@ include_package_data=True, install_requires=[ "py-solc-x>=2.0.2,<3", - "eth-ape>=0.6.25,<0.7", + "eth-ape>=0.7.0,<0.8", "ethpm-types", # Use the version ape requires + "eth-pydantic-types", # Use the version ape requires "packaging", # Use the version ape requires "requests", ], diff --git a/tests/ape-config.yaml b/tests/ape-config.yaml index 48747f0..eecbaa9 100644 --- a/tests/ape-config.yaml +++ b/tests/ape-config.yaml @@ -50,5 +50,8 @@ solidity: # Needed for Vault - "@openzeppelin/contracts=OpenZeppelin/v4.7.1" + # Testing multiple versions of same dependency. + - "@oz/contracts=OpenZeppelin/v4.5.0" + # Using evm_version compatible with older and newer solidity versions. evm_version: constantinople diff --git a/tests/conftest.py b/tests/conftest.py index 224f852..5bed070 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,10 +3,11 @@ from distutils.dir_util import copy_tree from pathlib import Path from tempfile import mkdtemp +from unittest import mock import ape import pytest -import solcx # type: ignore +import solcx from ape_solidity.compiler import Extension @@ -100,7 +101,28 @@ def compiler_manager(): @pytest.fixture def compiler(compiler_manager): - return compiler_manager.registered_compilers[Extension.SOL.value] + return compiler_manager.solidity + + +@pytest.fixture(autouse=True) +def ignore_other_compilers(mocker, compiler_manager, compiler): + """ + Having ape-vyper installed causes the random Vyper file + (that exists for testing purposes) to get compiled and + vyper to get repeatedly installed in a temporary directory. + Avoid that by tricking Ape into thinking ape-vyper is not + installed (if it is). + """ + existing_compilers = compiler_manager.registered_compilers + ape_pm = compiler_manager.ethpm + valid_compilers = { + ext: c for ext, c in existing_compilers.items() if ext in [x.value for x in Extension] + } + path = "ape.managers.compilers.CompilerManager.registered_compilers" + mock_registered_compilers = mocker.patch(path, new_callable=mock.PropertyMock) + + # Only ethpm (.json) and Solidity extensions allowed. + mock_registered_compilers.return_value = {".json": ape_pm, **valid_compilers} @pytest.fixture diff --git a/tests/contracts/RandomVyperFile.vy b/tests/contracts/RandomVyperFile.vy index 87442f6..125b14d 100644 --- a/tests/contracts/RandomVyperFile.vy +++ b/tests/contracts/RandomVyperFile.vy @@ -1,4 +1,4 @@ -# @version 0.3.6 +# @version 0.3.10 # NOTE: This file only exists to prove it does not interfere # (we had found bugs related to this) diff --git a/tests/contracts/SpacesInPragma.sol b/tests/contracts/SpacesInPragma.sol index 5684da3..444e1c2 100644 --- a/tests/contracts/SpacesInPragma.sol +++ b/tests/contracts/SpacesInPragma.sol @@ -2,7 +2,7 @@ // This file exists to test a bug that occurred when the user has a pragma // like this one. It was failing to register as a proper NpmSpec because // of the spaces between the operator and the version. -pragma solidity >= 0.4.19 < 0.5.0; + pragma solidity >= 0.4.19 < 0.5.0; interface SpacesInPragma { // This syntax fails on version >= 5.0, thus we are testing diff --git a/tests/test_compiler.py b/tests/test_compiler.py index d05e558..8af011e 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -import solcx # type: ignore +import solcx from ape import reverts from ape.contracts import ContractContainer from ape.exceptions import CompilerError @@ -234,7 +234,7 @@ def test_get_version_map(project, compiler): assert all([f in version_map[expected_version] for f in file_paths[:-1]]) latest_version_sources = version_map[latest_version] - assert len(latest_version_sources) == 10, "Did the import remappings load correctly?" + assert len(latest_version_sources) >= 10, "Did the import remappings load correctly?" assert file_paths[-1] in latest_version_sources # Will fail if the import remappings have not loaded yet. @@ -256,68 +256,79 @@ def test_get_version_map_raises_on_non_solidity_sources(compiler, vyper_source_p def test_compiler_data_in_manifest(project): - manifest = project.extract_manifest() - compilers = [c for c in manifest.compilers if c.name == "solidity"] - latest_version = max(c.version for c in compilers) - - compiler_latest = [c for c in compilers if str(c.version) == latest_version][0] - compiler_0812 = [c for c in compilers if str(c.version) == "0.8.12+commit.f00d7308"][0] - compiler_0612 = [c for c in compilers if str(c.version) == "0.6.12+commit.27d51765"][0] - compiler_0426 = [c for c in compilers if str(c.version) == "0.4.26+commit.4563c3fc"][0] - - # Compiler name test - for compiler in (compiler_latest, compiler_0812, compiler_0612, compiler_0426): - assert compiler.name == "solidity" - assert compiler.settings["optimizer"] == DEFAULT_OPTIMIZER - assert compiler.settings["evmVersion"] == "constantinople" - - # No remappings for sources in the following compilers - assert ( - "remappings" not in compiler_0812.settings - ), f"Remappings found: {compiler_0812.settings['remappings']}" - - assert ( - "@openzeppelin/contracts=.cache/OpenZeppelin/v4.7.1" - in compiler_latest.settings["remappings"] - ) - assert "@vault=.cache/vault/v0.4.5" in compiler_latest.settings["remappings"] - assert "@vaultmain=.cache/vault/master" in compiler_latest.settings["remappings"] - common_suffix = ".cache/TestDependency/local" - expected_remappings = ( - "@remapping_2_brownie=.cache/BrownieDependency/local", - "@dependency_remapping=.cache/DependencyOfDependency/local", - f"@remapping_2={common_suffix}", - f"@remapping/contracts={common_suffix}", - "@styleofbrownie=.cache/BrownieStyleDependency/local", - ) - actual_remappings = compiler_latest.settings["remappings"] - assert all(x in actual_remappings for x in expected_remappings) - assert all( - b >= a for a, b in zip(actual_remappings, actual_remappings[1:]) - ), "Import remappings should be sorted" - assert f"@remapping/contracts={common_suffix}" in compiler_0426.settings["remappings"] - assert "UseYearn" in compiler_latest.contractTypes - assert "@gnosis=.cache/gnosis/v1.3.0" in compiler_latest.settings["remappings"] - - # Compiler contract types test - assert set(compiler_0812.contractTypes) == { - "ImportSourceWithEqualSignVersion", - "ImportSourceWithNoPrefixVersion", - "ImportingLessConstrainedVersion", - "IndirectlyImportingMoreConstrainedVersion", - "IndirectlyImportingMoreConstrainedVersionCompanion", - "SpecificVersionNoPrefix", - "SpecificVersionRange", - "SpecificVersionWithEqualSign", - "CompilesOnce", - "IndirectlyImportingMoreConstrainedVersionCompanionImport", - } - assert set(compiler_0612.contractTypes) == {"RangedVersion", "VagueVersion"} - assert set(compiler_0426.contractTypes) == { - "ExperimentalABIEncoderV2", - "SpacesInPragma", - "ImportOlderDependency", - } + def run_test(manifest): + compilers = [c for c in manifest.compilers if c.name == "solidity"] + latest_version = max(c.version for c in compilers) + + compiler_latest = [c for c in compilers if str(c.version) == latest_version][0] + compiler_0812 = [c for c in compilers if str(c.version) == "0.8.12+commit.f00d7308"][0] + compiler_0612 = [c for c in compilers if str(c.version) == "0.6.12+commit.27d51765"][0] + compiler_0426 = [c for c in compilers if str(c.version) == "0.4.26+commit.4563c3fc"][0] + + # Compiler name test + for compiler in (compiler_latest, compiler_0812, compiler_0612, compiler_0426): + assert compiler.name == "solidity" + assert compiler.settings["optimizer"] == DEFAULT_OPTIMIZER + assert compiler.settings["evmVersion"] == "constantinople" + + # No remappings for sources in the following compilers + assert ( + "remappings" not in compiler_0812.settings + ), f"Remappings found: {compiler_0812.settings['remappings']}" + + assert ( + "@openzeppelin/contracts=.cache/OpenZeppelin/v4.7.1" + in compiler_latest.settings["remappings"] + ) + assert "@vault=.cache/vault/v0.4.5" in compiler_latest.settings["remappings"] + assert "@vaultmain=.cache/vault/master" in compiler_latest.settings["remappings"] + common_suffix = ".cache/TestDependency/local" + expected_remappings = ( + "@remapping_2_brownie=.cache/BrownieDependency/local", + "@dependency_remapping=.cache/DependencyOfDependency/local", + f"@remapping_2={common_suffix}", + f"@remapping/contracts={common_suffix}", + "@styleofbrownie=.cache/BrownieStyleDependency/local", + ) + actual_remappings = compiler_latest.settings["remappings"] + assert all(x in actual_remappings for x in expected_remappings) + assert all( + b >= a for a, b in zip(actual_remappings, actual_remappings[1:]) + ), "Import remappings should be sorted" + assert f"@remapping/contracts={common_suffix}" in compiler_0426.settings["remappings"] + assert "UseYearn" in compiler_latest.contractTypes + assert "@gnosis=.cache/gnosis/v1.3.0" in compiler_latest.settings["remappings"] + + # Compiler contract types test + assert set(compiler_0812.contractTypes) == { + "ImportSourceWithEqualSignVersion", + "ImportSourceWithNoPrefixVersion", + "ImportingLessConstrainedVersion", + "IndirectlyImportingMoreConstrainedVersion", + "IndirectlyImportingMoreConstrainedVersionCompanion", + "SpecificVersionNoPrefix", + "SpecificVersionRange", + "SpecificVersionWithEqualSign", + "CompilesOnce", + "IndirectlyImportingMoreConstrainedVersionCompanionImport", + } + assert set(compiler_0612.contractTypes) == {"RangedVersion", "VagueVersion"} + assert set(compiler_0426.contractTypes) == { + "ExperimentalABIEncoderV2", + "SpacesInPragma", + "ImportOlderDependency", + } + + # Ensure compiled first so that the local cached manifest exists. + # We want to make ape-solidity has placed the compiler info in there. + project.load_contracts() + if man := project.local_project.manifest: + run_test(man) + else: + pytest.fail("Manifest was not cached after loading.") + + # The extracted manifest should produce the same result. + run_test(project.extract_manifest()) def test_get_versions(compiler, project): @@ -333,37 +344,54 @@ def test_get_versions(compiler, project): def test_get_compiler_settings(compiler, project): - source_a = "ImportSourceWithEqualSignVersion.sol" - source_b = "SpecificVersionNoPrefix.sol" - source_c = "CompilesOnce.sol" - source_d = "Imports.sol" # Uses mapped imports! - indirect_source = "SpecificVersionWithEqualSign.sol" - file_paths = [project.contracts_folder / x for x in (source_a, source_b, source_c, source_d)] + # We start with the following sources as inputs: + # `forced_812_*` are forced to compile using solc 0.8.12 because its + # import is hard-pinned to it. + forced_812_0 = "ImportSourceWithEqualSignVersion.sol" + forced_812_1 = "SpecificVersionNoPrefix.sol" + # The following are unspecified and not used by the above. + # Thus are compiled on the latest. + latest_0 = "CompilesOnce.sol" + latest_1 = "Imports.sol" # Uses mapped imports! + file_paths = [ + project.contracts_folder / x for x in (forced_812_0, forced_812_1, latest_0, latest_1) + ] + + # Actual should contain all the settings for every file used in a would-be compile. actual = compiler.get_compiler_settings(file_paths) + + # The following is indirectly used by 0.8.12 from an import. + forced_812_0_import = "SpecificVersionWithEqualSign.sol" + + # These are the versions we are checking in our expectations. v812 = Version("0.8.12+commit.f00d7308") latest = max(list(actual.keys())) - expected_remappings = ( - "@remapping_2_brownie=.cache/BrownieDependency/local", - "@dependency_remapping=.cache/DependencyOfDependency/local", - "@remapping_2=.cache/TestDependency/local", - "@remapping/contracts=.cache/TestDependency/local", - "@styleofbrownie=.cache/BrownieStyleDependency/local", - "@gnosis=.cache/gnosis/v1.3.0", - ) - expected_v812_contracts = (source_a, source_b, source_c, indirect_source) - expected_latest_contracts = ( + + expected_v812_contracts = [forced_812_0, forced_812_0_import, forced_812_1] + expected_latest_contracts = [ + latest_0, + latest_1, + # The following are expected imported sources. ".cache/BrownieDependency/local/BrownieContract.sol", "CompilesOnce.sol", ".cache/TestDependency/local/Dependency.sol", ".cache/DependencyOfDependency/local/DependencyOfDependency.sol", - source_d, "subfolder/Relativecontract.sol", ".cache/gnosis/v1.3.0/common/Enum.sol", - ) + ] + expected_remappings = [ + "@remapping_2_brownie=.cache/BrownieDependency/local", + "@dependency_remapping=.cache/DependencyOfDependency/local", + "@remapping_2=.cache/TestDependency/local", + "@remapping/contracts=.cache/TestDependency/local", + "@styleofbrownie=.cache/BrownieStyleDependency/local", + "@gnosis=.cache/gnosis/v1.3.0", + ] # Shared compiler defaults tests expected_source_lists = (expected_v812_contracts, expected_latest_contracts) for version, expected_sources in zip((v812, latest), expected_source_lists): + expected_sources.sort() output_selection = actual[version]["outputSelection"] assert actual[version]["optimizer"] == DEFAULT_OPTIMIZER for _, item_selection in output_selection.items(): @@ -373,7 +401,9 @@ def test_get_compiler_settings(compiler, project): elif key == "": # All sources assert selection == ["ast"] - actual_sources = [x for x in output_selection.keys()] + # Sort to help debug. + actual_sources = sorted([x for x in output_selection.keys()]) + for expected_source_id in expected_sources: assert ( expected_source_id in actual_sources @@ -401,7 +431,7 @@ def test_evm_version(compiler): def test_source_map(project, compiler): source_path = project.contracts_folder / "MultipleDefinitions.sol" result = compiler.compile([source_path])[-1] - assert result.sourcemap.__root__ == "124:87:0:-:0;;;;;;;;;;;;;;;;;;;" + assert result.sourcemap.root == "124:87:0:-:0;;;;;;;;;;;;;;;;;;;" def test_add_library(project, account, compiler, connection):