From c1cf255288b33a2efbfc31d6006aa25ee8114eaf Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 21 Nov 2023 21:22:33 +0800 Subject: [PATCH] Moved the environment assumption from `ConfigLoader` to `_ProjectSettings` (#3311) * update release notes Signed-off-by: Nok * Remove the default env in config loader - add the env in tests Signed-off-by: Nok Chan * remove unncessary `default_run_env` Signed-off-by: Nok Chan * fix tests Signed-off-by: Nok Chan * Added a couple of tests to make sure config loader standalone mode works Signed-off-by: Nok Chan * fix tests Signed-off-by: Nok Chan * lint Signed-off-by: Nok Chan * Fix tests Signed-off-by: Nok * update default Signed-off-by: Nok * fix test coverage Signed-off-by: Nok * fix test and lint Signed-off-by: Nok * rename test to 'without_environment' to reflect the change of defaults Signed-off-by: Nok * fix tests according to comments Signed-off-by: Nok * update test to test arbitary env explicitly Signed-off-by: Nok --------- Signed-off-by: Nok Signed-off-by: Nok Chan --- RELEASE.md | 2 +- .../settings.py | 2 + kedro/config/omegaconf_config.py | 8 +- kedro/framework/project/__init__.py | 4 +- .../settings.py | 2 + tests/config/test_omegaconf_config.py | 454 ++++++++++++++++-- tests/framework/context/test_context.py | 22 +- tests/framework/project/test_settings.py | 5 +- 8 files changed, 433 insertions(+), 66 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 2277bd6ac5..3873529804 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,7 +8,7 @@ * Added the functionality to choose a merging strategy for config files loaded with `OmegaConfigLoader`. * Modified the mechanism of importing datasets, raise more explicit error when dependencies are missing. * Added validation for configuration file used to override run commands via the CLI. - +* Moved the default environment `base` and `local` from config loader to `_ProjectSettings`. This enables the use of config loader as a standalone class without affecting existing Kedro Framework users. ## Bug fixes and other changes * Added a new field `add-ons` to `pyproject.toml` when a project is created. * Reduced `spaceflights` data to minimise waiting times during tutorial execution. diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py index 86a92b1c80..3207b1da0a 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py @@ -26,6 +26,8 @@ # CONFIG_LOADER_CLASS = OmegaConfigLoader # Keyword arguments to pass to the `CONFIG_LOADER_CLASS` constructor. # CONFIG_LOADER_ARGS = { +# "base_env": "base", +# "default_run_env": "local", # "config_patterns": { # "spark" : ["spark*/"], # "parameters": ["parameters*", "parameters*/**", "**/parameters*"], diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index c75a7713c9..f8ec549b38 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -80,8 +80,8 @@ def __init__( # noqa: PLR0913 runtime_params: dict[str, Any] = None, *, config_patterns: dict[str, list[str]] = None, - base_env: str = "base", - default_run_env: str = "local", + base_env: str = None, + default_run_env: str = None, custom_resolvers: dict[str, Callable] = None, merge_strategy: dict[str, str] = None, ): @@ -102,8 +102,8 @@ def __init__( # noqa: PLR0913 custom_resolvers: A dictionary of custom resolvers to be registered. For more information, see here: https://omegaconf.readthedocs.io/en/2.3_branch/custom_resolvers.html#custom-resolvers """ - self.base_env = base_env - self.default_run_env = default_run_env + self.base_env = base_env or "" + self.default_run_env = default_run_env or "" self.merge_strategy = merge_strategy or {} self.config_patterns = { diff --git a/kedro/framework/project/__init__.py b/kedro/framework/project/__init__.py index 485df57891..85eafc37c9 100644 --- a/kedro/framework/project/__init__.py +++ b/kedro/framework/project/__init__.py @@ -105,7 +105,9 @@ class _ProjectSettings(LazySettings): "CONFIG_LOADER_CLASS", default=_get_default_class("kedro.config.OmegaConfigLoader"), ) - _CONFIG_LOADER_ARGS = Validator("CONFIG_LOADER_ARGS", default={}) + _CONFIG_LOADER_ARGS = Validator( + "CONFIG_LOADER_ARGS", default={"base_env": "base", "default_run_env": "local"} + ) _DATA_CATALOG_CLASS = _IsSubclassValidator( "DATA_CATALOG_CLASS", default=_get_default_class("kedro.io.DataCatalog") ) diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py index 53c6f763bc..db58cab52d 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/settings.py @@ -29,6 +29,8 @@ # Keyword arguments to pass to the `CONFIG_LOADER_CLASS` constructor. # CONFIG_LOADER_ARGS = { +# "base_env": "base", +# "default_run_env": "local", # "config_patterns": { # "spark" : ["spark*/"], # "parameters": ["parameters*", "parameters*/**", "**/parameters*"], diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 31d8be44f0..640def26d4 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -158,7 +158,9 @@ class TestOmegaConfigLoader: @use_config_dir def test_load_core_config_dict_syntax(self, tmp_path): """Make sure core config can be fetched with a dict [] access.""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) params = conf["parameters"] catalog = conf["catalog"] @@ -168,7 +170,9 @@ def test_load_core_config_dict_syntax(self, tmp_path): @use_config_dir def test_load_core_config_get_syntax(self, tmp_path): """Make sure core config can be fetched with .get()""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) params = conf.get("parameters") catalog = conf.get("catalog") @@ -179,7 +183,9 @@ def test_load_core_config_get_syntax(self, tmp_path): def test_load_local_config_overrides_base(self, tmp_path): """Make sure that configs from `local/` override the ones from `base/`""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) params = conf["parameters"] catalog = conf["catalog"] @@ -193,14 +199,18 @@ def test_load_local_config_overrides_base(self, tmp_path): def test_load_base_config(self, tmp_path, base_config): """Test config loading if `local/` directory is empty""" (tmp_path / _DEFAULT_RUN_ENV).mkdir(exist_ok=True) - catalog = OmegaConfigLoader(str(tmp_path))["catalog"] + catalog = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] assert catalog == base_config @use_proj_catalog def test_duplicate_patterns(self, tmp_path, base_config): """Test config loading if the glob patterns cover the same file""" (tmp_path / _DEFAULT_RUN_ENV).mkdir(exist_ok=True) - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) catalog1 = conf["catalog"] catalog2 = conf["catalog"] assert catalog1 == catalog2 == base_config @@ -212,16 +222,24 @@ def test_subdirs_dont_exist(self, tmp_path, base_config): r"or is not a valid directory\: {}" ) with pytest.raises(MissingConfigException, match=pattern.format(".*base")): - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] with pytest.raises(MissingConfigException, match=pattern.format(".*local")): proj_catalog = tmp_path / _BASE_ENV / "catalog.yml" _write_yaml(proj_catalog, base_config) - OmegaConfigLoader(str(tmp_path))["catalog"] + print( + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] + ) @pytest.mark.usefixtures("create_config_dir", "proj_catalog", "proj_catalog_nested") def test_nested(self, tmp_path): """Test loading the config from subdirectories""" - config_loader = OmegaConfigLoader(str(tmp_path)) + config_loader = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) config_loader.default_run_env = "prod" prod_catalog = tmp_path / "prod" / "catalog.yml" @@ -246,7 +264,9 @@ def test_nested_subdirs_duplicate(self, tmp_path, base_config): r"\: cars, trains" ) with pytest.raises(ValueError, match=pattern): - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] @use_config_dir def test_multiple_nested_subdirs_duplicates( @@ -277,7 +297,9 @@ def test_multiple_nested_subdirs_duplicates( ) with pytest.raises(ValueError) as exc: - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] assert re.search(pattern_catalog_nested, str(exc.value)) assert re.search(pattern_catalog_local, str(exc.value)) assert re.search(pattern_nested_local, str(exc.value)) @@ -290,14 +312,18 @@ def test_bad_config_syntax(self, tmp_path): pattern = f"Invalid YAML or JSON file {conf_path.as_posix()}" with pytest.raises(ParserError, match=re.escape(pattern)): - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] def test_lots_of_duplicates(self, tmp_path): data = {str(i): i for i in range(100)} _write_yaml(tmp_path / _BASE_ENV / "catalog1.yml", data) _write_yaml(tmp_path / _BASE_ENV / "catalog2.yml", data) - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) pattern = ( r"Duplicate keys found in " r"(.*catalog2\.yml and .*catalog1\.yml|.*catalog1\.yml and .*catalog2\.yml)" @@ -335,7 +361,12 @@ def test_cannot_load_non_yaml_or_json_files(self, tmp_path): db_config_path = tmp_path / _BASE_ENV / "db.ini" _write_dummy_ini(db_config_path) - conf = OmegaConfigLoader(str(tmp_path), config_patterns=db_patterns) + conf = OmegaConfigLoader( + str(tmp_path), + config_patterns=db_patterns, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + ) pattern = ( r"No files of YAML or JSON format found in " r".*base or " @@ -357,7 +388,9 @@ def test_no_files_found(self, tmp_path): r"\[\'credentials\*\', \'credentials\*/\**\', \'\**/credentials\*\'\]" ) with pytest.raises(MissingConfigException, match=pattern): - OmegaConfigLoader(str(tmp_path))["credentials"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["credentials"] def test_empty_catalog_file(self, tmp_path): """Check that empty catalog file is read and returns an empty dict""" @@ -389,7 +422,10 @@ def test_overlapping_patterns(self, tmp_path, mocker): } catalog = OmegaConfigLoader( - conf_source=str(tmp_path), env="dev", config_patterns=catalog_patterns + conf_source=str(tmp_path), + base_env=_BASE_ENV, + env="dev", + config_patterns=catalog_patterns, )["catalog"] expected_catalog = { "env": "dev", @@ -420,7 +456,9 @@ def test_yaml_parser_error(self, tmp_path): f" line 3, position 10." ) with pytest.raises(ParserError, match=re.escape(msg)): - OmegaConfigLoader(str(tmp_path))["catalog"] + OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + )["catalog"] def test_customised_config_patterns(self, tmp_path): config_loader = OmegaConfigLoader( @@ -444,9 +482,12 @@ def test_customised_config_patterns(self, tmp_path): def test_default_destructive_merging_strategy(self, tmp_path, mlflow_config): mlflow_patterns = {"mlflow": ["mlflow*", "mlflow*/**", "**/mlflow*"]} - conf = OmegaConfigLoader(str(tmp_path), config_patterns=mlflow_patterns)[ - "mlflow" - ] + conf = OmegaConfigLoader( + str(tmp_path), + config_patterns=mlflow_patterns, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + )["mlflow"] assert conf == { "tracking": { @@ -462,6 +503,8 @@ def test_destructive_merging_strategy(self, tmp_path, mlflow_config): str(tmp_path), config_patterns=mlflow_patterns, merge_strategy={"mlflow": "destructive"}, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, )["mlflow"] assert conf == { @@ -478,6 +521,8 @@ def test_soft_merging_strategy(self, tmp_path, mlflow_config): str(tmp_path), config_patterns=mlflow_patterns, merge_strategy={"mlflow": "soft"}, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, )["mlflow"] assert conf == { @@ -497,12 +542,16 @@ def test_unsupported_merge_strategy(self, tmp_path, mlflow_config): str(tmp_path), config_patterns=mlflow_patterns, merge_strategy={"mlflow": "hard"}, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, )["mlflow"] @use_config_dir def test_adding_extra_keys_to_confloader(self, tmp_path): """Make sure extra keys can be added directly to the config loader instance.""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) catalog = conf["catalog"] conf["spark"] = {"spark_config": "emr.blabla"} @@ -513,7 +562,9 @@ def test_adding_extra_keys_to_confloader(self, tmp_path): def test_bypass_catalog_config_loading(self, tmp_path): """Make sure core config loading can be bypassed by setting the key and values directly on the config loader instance.""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) conf["catalog"] = {"catalog_config": "something_new"} assert conf["catalog"] == {"catalog_config": "something_new"} @@ -522,7 +573,9 @@ def test_bypass_catalog_config_loading(self, tmp_path): @use_credentials_env_variable_yml def test_load_credentials_from_env_variables(self, tmp_path): """Load credentials from environment variables""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" assert conf["credentials"]["user"]["name"] == "test_user" @@ -532,7 +585,9 @@ def test_load_credentials_from_env_variables(self, tmp_path): @use_catalog_env_variable_yml def test_env_resolver_not_used_for_catalog(self, tmp_path): """Check that the oc.env resolver is not used for catalog loading""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) os.environ["TEST_DATASET"] = "test_dataset" with pytest.raises(errors.UnsupportedInterpolationType): conf["catalog"]["test"]["file_path"] @@ -542,7 +597,9 @@ def test_env_resolver_not_used_for_catalog(self, tmp_path): def test_env_resolver_is_cleared_after_loading(self, tmp_path): """Check that the ``oc.env`` resolver is cleared after loading credentials in the case that it was not registered beforehand.""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" assert conf["credentials"]["user"]["name"] == "test_user" @@ -553,7 +610,9 @@ def test_env_resolver_is_cleared_after_loading(self, tmp_path): def test_env_resolver_is_registered_after_loading(self, tmp_path): """Check that the ``oc.env`` resolver is registered after loading credentials in the case that it was registered beforehand""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) OmegaConf.register_new_resolver("oc.env", oc.env) os.environ["TEST_USERNAME"] = "test_user" os.environ["TEST_KEY"] = "test_key" @@ -574,7 +633,11 @@ def test_load_config_from_tar_file(self, tmp_path): ] ) - conf = OmegaConfigLoader(conf_source=f"{tmp_path}/tar_conf.tar.gz") + conf = OmegaConfigLoader( + conf_source=f"{tmp_path}/tar_conf.tar.gz", + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + ) catalog = conf["catalog"] assert catalog["trains"]["type"] == "MemoryDataset" @@ -598,14 +661,20 @@ def zipdir(path, ziph): ) as zipf: zipdir(tmp_path, zipf) - conf = OmegaConfigLoader(conf_source=f"{tmp_path}/Python.zip") + conf = OmegaConfigLoader( + conf_source=f"{tmp_path}/Python.zip", + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + ) catalog = conf["catalog"] assert catalog["trains"]["type"] == "MemoryDataset" @use_config_dir def test_variable_interpolation_with_correct_env(self, tmp_path): """Make sure the parameters is interpolated with the correct environment""" - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) params = conf["parameters"] # Making sure it is not override by local/parameters_global.yml assert params["interpolated_param"] == "base" @@ -613,7 +682,12 @@ def test_variable_interpolation_with_correct_env(self, tmp_path): @use_config_dir def test_runtime_params_override_interpolated_value(self, tmp_path): """Make sure interpolated value is updated correctly with runtime_params""" - conf = OmegaConfigLoader(str(tmp_path), runtime_params={"test_env": "dummy"}) + conf = OmegaConfigLoader( + str(tmp_path), + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + runtime_params={"test_env": "dummy"}, + ) params = conf["parameters"] assert params["interpolated_param"] == "dummy" @@ -629,6 +703,8 @@ def test_runtime_params_not_propogate_non_parameters_config(self, tmp_path): str(tmp_path), config_patterns={"spark": ["spark*", "spark*/**", "**/spark*"]}, runtime_params=runtime_params, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, ) parameters = conf["parameters"] catalog = conf["catalog"] @@ -646,8 +722,7 @@ def test_ignore_hidden_keys(self, tmp_path): _write_yaml(tmp_path / _BASE_ENV / "catalog1.yml", {"k1": "v1", "_k2": "v2"}) _write_yaml(tmp_path / _BASE_ENV / "catalog2.yml", {"k3": "v3", "_k2": "v4"}) - conf = OmegaConfigLoader(str(tmp_path)) - conf.default_run_env = "" + conf = OmegaConfigLoader(str(tmp_path), base_env=_BASE_ENV) catalog = conf["catalog"] assert catalog.keys() == {"k1", "k3"} @@ -671,8 +746,7 @@ def test_variable_interpolation_in_catalog_with_templates(self, tmp_path): } _write_yaml(base_catalog, catalog_config) - conf = OmegaConfigLoader(str(tmp_path)) - conf.default_run_env = "" + conf = OmegaConfigLoader(str(tmp_path), base_env=_BASE_ENV) assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" def test_variable_interpolation_in_catalog_with_separate_templates_file( @@ -690,8 +764,7 @@ def test_variable_interpolation_in_catalog_with_separate_templates_file( _write_yaml(base_catalog, catalog_config) _write_yaml(tmp_catalog, template) - conf = OmegaConfigLoader(str(tmp_path)) - conf.default_run_env = "" + conf = OmegaConfigLoader(str(tmp_path), base_env=_BASE_ENV) assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" def test_custom_resolvers(self, tmp_path): @@ -710,8 +783,11 @@ def test_custom_resolvers(self, tmp_path): "oc.env": oc.env, } os.environ["VAR"] = "my_env_variable" - conf = OmegaConfigLoader(tmp_path, custom_resolvers=custom_resolvers) - conf.default_run_env = "" + conf = OmegaConfigLoader( + tmp_path, + custom_resolvers=custom_resolvers, + base_env=_BASE_ENV, + ) assert conf["parameters"]["model_options"]["param1"] == 7 assert conf["parameters"]["model_options"]["param2"] == 3 assert conf["parameters"]["model_options"]["param3"] == "my_env_variable" @@ -722,7 +798,7 @@ def test_globals(self, tmp_path): "x": 0, } _write_yaml(globals_params, globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) # OmegaConfigLoader has globals resolver assert OmegaConf.has_resolver("globals") # Globals is readable in a dict way @@ -746,7 +822,7 @@ def test_globals_resolution(self, tmp_path): _write_yaml(base_params, param_config) _write_yaml(globals_params, globals_config) _write_yaml(base_catalog, catalog_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) # Globals are resolved correctly in parameter assert conf["parameters"]["my_param"] == globals_config["x"] # The default value is used if the key does not exist @@ -769,7 +845,7 @@ def test_globals_nested(self, tmp_path): } _write_yaml(base_params, param_config) _write_yaml(globals_params, globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) assert conf["parameters"]["my_param"] == globals_config["x"] # Nested globals are accessible with dot notation assert conf["parameters"]["my_nested_param"] == globals_config["nested"]["y"] @@ -796,7 +872,9 @@ def test_globals_across_env(self, tmp_path): _write_yaml(local_params, local_param_config) _write_yaml(base_globals, base_globals_config) _write_yaml(local_globals, local_globals_config) - conf = OmegaConfigLoader(tmp_path) + conf = OmegaConfigLoader( + tmp_path, base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) # Local global overwrites the base global value assert conf["parameters"]["param1"] == local_globals_config["y"] # Base global value is accessible to local params @@ -813,7 +891,7 @@ def test_globals_default(self, tmp_path): base_globals_config = {"x": {"DUMMY": 3}} _write_yaml(base_params, base_param_config) _write_yaml(base_globals, base_globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) # Default value is being used as int assert conf["parameters"]["int"] == 1 # Default value is being used as str @@ -837,7 +915,7 @@ def test_globals_default_none(self, tmp_path): } _write_yaml(base_params, base_param_config) _write_yaml(base_globals, base_globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) # Default value can be 0 or null assert conf["parameters"]["zero"] == 0 assert conf["parameters"]["null"] is None @@ -857,7 +935,7 @@ def test_globals_missing_default(self, tmp_path): } _write_yaml(base_params, param_config) _write_yaml(globals_params, globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) with pytest.raises( InterpolationResolutionError, @@ -876,7 +954,7 @@ def test_bad_globals_underscore(self, tmp_path): } _write_yaml(base_params, base_param_config) _write_yaml(base_globals, base_globals_config) - conf = OmegaConfigLoader(tmp_path, default_run_env="") + conf = OmegaConfigLoader(tmp_path, base_env=_BASE_ENV) with pytest.raises( InterpolationResolutionError, match=r"Keys starting with '_' are not supported for globals.", @@ -887,7 +965,9 @@ def test_bad_globals_underscore(self, tmp_path): "hidden_path", ["/User/.hidden/dummy.yml", "/User/dummy/.hidden.yml"] ) def test_is_hidden_config(self, tmp_path, hidden_path): - conf = OmegaConfigLoader(str(tmp_path)) + conf = OmegaConfigLoader( + str(tmp_path), base_env=_BASE_ENV, default_run_env=_DEFAULT_RUN_ENV + ) assert conf._is_hidden(hidden_path) @pytest.mark.parametrize( @@ -949,7 +1029,10 @@ def test_runtime_params_resolution(self, tmp_path): _write_yaml(base_params, param_config) _write_yaml(base_catalog, catalog_config) conf = OmegaConfigLoader( - tmp_path, default_run_env="", runtime_params=runtime_params + tmp_path, + base_env=_BASE_ENV, + default_run_env="", + runtime_params=runtime_params, ) # runtime are resolved correctly in parameter assert conf["parameters"]["my_runtime_param"] == runtime_params["x"] @@ -968,7 +1051,10 @@ def test_runtime_params_missing_default(self, tmp_path): } _write_yaml(base_params, param_config) conf = OmegaConfigLoader( - tmp_path, default_run_env="", runtime_params=runtime_params + tmp_path, + base_env=_BASE_ENV, + default_run_env="", + runtime_params=runtime_params, ) with pytest.raises( InterpolationResolutionError, @@ -979,15 +1065,18 @@ def test_runtime_params_missing_default(self, tmp_path): def test_runtime_params_in_globals_not_allowed(self, tmp_path): base_globals = tmp_path / _BASE_ENV / "globals.yml" local_globals = tmp_path / _DEFAULT_RUN_ENV / "globals.yml" + runtime_params = { "x": 45, } + base_globals_config = { "my_global_var": "${runtime_params:x}", } local_globals_config = { "my_local_var": "${runtime_params:x}", # x does exist but shouldn't be allowed in globals } + _write_yaml(base_globals, base_globals_config) _write_yaml(local_globals, local_globals_config) @@ -997,15 +1086,19 @@ def test_runtime_params_in_globals_not_allowed(self, tmp_path): ): OmegaConfigLoader( tmp_path, - base_env="", - default_run_env="local", + default_run_env=_DEFAULT_RUN_ENV, runtime_params=runtime_params, ) with pytest.raises( UnsupportedInterpolationType, match=r"The `runtime_params:` resolver is not supported for globals.", ): - OmegaConfigLoader(tmp_path, runtime_params=runtime_params) + OmegaConfigLoader( + tmp_path, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + runtime_params=runtime_params, + ) def test_runtime_params_default_global(self, tmp_path): base_globals = tmp_path / _BASE_ENV / "globals.yml" @@ -1027,7 +1120,7 @@ def test_runtime_params_default_global(self, tmp_path): _write_yaml(base_catalog, catalog_config) _write_yaml(base_globals, globals_config) conf = OmegaConfigLoader( - tmp_path, default_run_env="", runtime_params=runtime_params + tmp_path, base_env=_BASE_ENV, runtime_params=runtime_params ) # runtime params are resolved correctly in catalog using global default assert conf["catalog"]["companies"]["type"] == globals_config["dataset"]["type"] @@ -1075,7 +1168,12 @@ def test_unsupported_interpolation_globals(self, tmp_path): UnsupportedInterpolationType, match=r"Unsupported interpolation type non_existent_resolver", ): - OmegaConfigLoader(tmp_path, runtime_params=runtime_params) + OmegaConfigLoader( + tmp_path, + base_env=_BASE_ENV, + default_run_env=_DEFAULT_RUN_ENV, + runtime_params=runtime_params, + ) with pytest.raises( UnsupportedInterpolationType, match=r"Unsupported interpolation type non_existent_resolver", @@ -1107,3 +1205,253 @@ def test_override_globals(self, tmp_path): } assert conf["parameters"]["my_global"] == 89 assert conf["parameters"]["my_second_global"] == 24 + + +class TestOmegaConfigLoaderStandalone: + """ + Test OmegaConfigLoader in standalone mode (defaults settings without environments) + """ + + def test_load_config_without_environment(self, tmp_path): + base_catalog = tmp_path / "catalog.yml" + catalog_config = { + "companies": { + "type": "MemoryDataset", + "filepath": "data/01_raw/companies.csv", + }, + } + _write_yaml(base_catalog, catalog_config) + + conf = OmegaConfigLoader(tmp_path) + assert ( + conf["catalog"]["companies"]["type"] == catalog_config["companies"]["type"] + ) + + def test_load_config_only_base_environment(self, tmp_path): + dummy_env = "dummy_base_env" + base_parameters = tmp_path / dummy_env / "parameters.yml" + base_parameters_config = {"dummy": "base"} + + _write_yaml(base_parameters, base_parameters_config) + + conf = OmegaConfigLoader(tmp_path, base_env=dummy_env) + assert conf["parameters"]["dummy"] == "base" + + def test_load_config_only_default_run_environment(self, tmp_path): + dummy_env = "dummy_base_env" + default_env_parameters = tmp_path / dummy_env / "parameters.yml" + default_env_parameters_config = {"dummy": "default"} + + _write_yaml(default_env_parameters, default_env_parameters_config) + + conf = OmegaConfigLoader(tmp_path, default_run_env=dummy_env) + assert conf["parameters"]["dummy"] == "default" + + def test_variable_interpolation_in_catalog_with_templates(self, tmp_path): + base_catalog = tmp_path / "catalog.yml" + catalog_config = { + "companies": { + "type": "${_pandas.type}", + "filepath": "data/01_raw/companies.csv", + }, + "_pandas": {"type": "pandas.CSVDataset"}, + } + _write_yaml(base_catalog, catalog_config) + + conf = OmegaConfigLoader( + str(tmp_path), + ) + assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" + + def test_variable_interpolation_in_catalog_with_separate_templates_file( + self, tmp_path + ): + base_catalog = tmp_path / "catalog.yml" + catalog_config = { + "companies": { + "type": "${_pandas.type}", + "filepath": "data/01_raw/companies.csv", + } + } + tmp_catalog = tmp_path / "catalog_temp.yml" + template = {"_pandas": {"type": "pandas.CSVDataset"}} + _write_yaml(base_catalog, catalog_config) + _write_yaml(tmp_catalog, template) + + conf = OmegaConfigLoader(str(tmp_path)) + assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataset" + + def test_globals(self, tmp_path): + globals_params = tmp_path / "globals.yml" + globals_config = { + "x": 0, + } + _write_yaml(globals_params, globals_config) + conf = OmegaConfigLoader(tmp_path) + # OmegaConfigLoader has globals resolver + assert OmegaConf.has_resolver("globals") + # Globals is readable in a dict way + assert conf["globals"] == globals_config + + def test_globals_resolution(self, tmp_path): + base_params = tmp_path / "parameters.yml" + base_catalog = tmp_path / "catalog.yml" + globals_params = tmp_path / "globals.yml" + param_config = { + "my_param": "${globals:x}", + "my_param_default": "${globals:y,34}", # y does not exist in globals + } + catalog_config = { + "companies": { + "type": "${globals:dataset_type}", + "filepath": "data/01_raw/companies.csv", + }, + } + globals_config = {"x": 34, "dataset_type": "pandas.CSVDataset"} + _write_yaml(base_params, param_config) + _write_yaml(globals_params, globals_config) + _write_yaml(base_catalog, catalog_config) + conf = OmegaConfigLoader(tmp_path) + # Globals are resolved correctly in parameter + assert conf["parameters"]["my_param"] == globals_config["x"] + # The default value is used if the key does not exist + assert conf["parameters"]["my_param_default"] == 34 + # Globals are resolved correctly in catalog + assert conf["catalog"]["companies"]["type"] == globals_config["dataset_type"] + + def test_globals_nested(self, tmp_path): + base_params = tmp_path / "parameters.yml" + globals_params = tmp_path / "globals.yml" + param_config = { + "my_param": "${globals:x}", + "my_nested_param": "${globals:nested.y}", + } + globals_config = { + "x": 34, + "nested": { + "y": 42, + }, + } + _write_yaml(base_params, param_config) + _write_yaml(globals_params, globals_config) + conf = OmegaConfigLoader(tmp_path) + assert conf["parameters"]["my_param"] == globals_config["x"] + # Nested globals are accessible with dot notation + assert conf["parameters"]["my_nested_param"] == globals_config["nested"]["y"] + + def test_globals_default(self, tmp_path): + base_params = tmp_path / "parameters.yml" + base_globals = tmp_path / "globals.yml" + base_param_config = { + "int": "${globals:x.NOT_EXIST, 1}", + "str": "${globals: x.NOT_EXIST, '2'}", + "dummy": "${globals: x.DUMMY.DUMMY, '2'}", + } + base_globals_config = {"x": {"DUMMY": 3}} + _write_yaml(base_params, base_param_config) + _write_yaml(base_globals, base_globals_config) + conf = OmegaConfigLoader(tmp_path) + # Default value is being used as int + assert conf["parameters"]["int"] == 1 + # Default value is being used as str + assert conf["parameters"]["str"] == "2" + # Test when x.DUMMY is not a dictionary it should still work + assert conf["parameters"]["dummy"] == "2" + + def test_globals_default_none(self, tmp_path): + base_params = tmp_path / "parameters.yml" + base_globals = tmp_path / "globals.yml" + base_param_config = { + "zero": "${globals: x.NOT_EXIST, 0}", + "null": "${globals: x.NOT_EXIST, null}", + "null2": "${globals: x.y}", + } + base_globals_config = { + "x": { + "z": 23, + "y": None, + }, + } + _write_yaml(base_params, base_param_config) + _write_yaml(base_globals, base_globals_config) + conf = OmegaConfigLoader(tmp_path) + # Default value can be 0 or null + assert conf["parameters"]["zero"] == 0 + assert conf["parameters"]["null"] is None + # Global value is null + assert conf["parameters"]["null2"] is None + + def test_runtime_params_resolution(self, tmp_path): + base_params = tmp_path / "parameters.yml" + base_catalog = tmp_path / "catalog.yml" + runtime_params = { + "x": 45, + "dataset": { + "type": "pandas.CSVDataset", + }, + } + param_config = { + "my_runtime_param": "${runtime_params:x}", + "my_param_default": "${runtime_params:y,34}", # y does not exist in globals + } + catalog_config = { + "companies": { + "type": "${runtime_params:dataset.type}", + "filepath": "data/01_raw/companies.csv", + }, + } + _write_yaml(base_params, param_config) + _write_yaml(base_catalog, catalog_config) + conf = OmegaConfigLoader( + tmp_path, + runtime_params=runtime_params, + ) + # runtime are resolved correctly in parameter + assert conf["parameters"]["my_runtime_param"] == runtime_params["x"] + # The default value is used if the key does not exist + assert conf["parameters"]["my_param_default"] == 34 + # runtime params are resolved correctly in catalog + assert conf["catalog"]["companies"]["type"] == runtime_params["dataset"]["type"] + + def test_runtime_params_in_globals_not_allowed(self, tmp_path): + base_globals = tmp_path / "globals.yml" + runtime_params = { + "x": 45, + } + base_globals_config = { + "my_global_var": "${runtime_params:x}", + } + + _write_yaml(base_globals, base_globals_config) + + with pytest.raises( + UnsupportedInterpolationType, + match=r"The `runtime_params:` resolver is not supported for globals.", + ): + OmegaConfigLoader( + tmp_path, + runtime_params=runtime_params, + ) + + def test_runtime_params_default_global(self, tmp_path): + base_globals = tmp_path / "globals.yml" + base_catalog = tmp_path / "catalog.yml" + runtime_params = { + "x": 45, + } + globals_config = { + "dataset": { + "type": "pandas.CSVDataset", + } + } + catalog_config = { + "companies": { + "type": "${runtime_params:type, ${globals:dataset.type, 'MemoryDataset'}}", + "filepath": "data/01_raw/companies.csv", + }, + } + _write_yaml(base_catalog, catalog_config) + _write_yaml(base_globals, globals_config) + conf = OmegaConfigLoader(tmp_path, runtime_params=runtime_params) + # runtime params are resolved correctly in catalog using global default + assert conf["catalog"]["companies"]["type"] == globals_config["dataset"]["type"] diff --git a/tests/framework/context/test_context.py b/tests/framework/context/test_context.py index c305c8c2ea..6faf71a5cd 100644 --- a/tests/framework/context/test_context.py +++ b/tests/framework/context/test_context.py @@ -16,7 +16,7 @@ from pandas.testing import assert_frame_equal from kedro import __version__ as kedro_version -from kedro.config import MissingConfigException, OmegaConfigLoader +from kedro.config import MissingConfigException from kedro.framework.context import KedroContext from kedro.framework.context.context import ( _convert_paths_to_absolute_posix, @@ -26,9 +26,10 @@ from kedro.framework.project import ( ValidationError, _ProjectSettings, - configure_project, pipelines, ) +from kedro.framework.session import KedroSession +from kedro.framework.startup import bootstrap_project MOCK_PACKAGE_NAME = "mock_package_name" @@ -120,16 +121,20 @@ def prepare_project_dir(tmp_path, base_config, local_config, env): env_catalog = tmp_path / "conf" / str(env) / "catalog.yml" env_credentials = tmp_path / "conf" / str(env) / "credentials.yml" parameters = tmp_path / "conf" / "base" / "parameters.json" - db_config_path = tmp_path / "conf" / "base" / "db.ini" project_parameters = {"param1": 1, "param2": 2, "param3": {"param4": 3}} + + # Create configurations _write_yaml(proj_catalog, base_config) _write_yaml(env_catalog, local_config) _write_yaml(env_credentials, local_config) _write_json(parameters, project_parameters) - _write_dummy_ini(db_config_path) _write_toml(tmp_path / "pyproject.toml", pyproject_toml_payload) + # Create the necessary files in src/ + (tmp_path / "src" / MOCK_PACKAGE_NAME).mkdir(parents=True, exist_ok=True) + (tmp_path / "src" / MOCK_PACKAGE_NAME / "__init__.py").write_text(" ") + @pytest.fixture def mock_settings_file_bad_data_catalog_class(tmpdir): @@ -188,8 +193,13 @@ def extra_params(request): @pytest.fixture def dummy_context(tmp_path, prepare_project_dir, env, extra_params): - configure_project(MOCK_PACKAGE_NAME) - config_loader = OmegaConfigLoader(str(tmp_path / "conf"), env=env) + bootstrap_project(tmp_path) + + session = KedroSession.create( + project_path=tmp_path, env=env, extra_params=extra_params + ) + context = session.load_context() + config_loader = context.config_loader context = KedroContext( MOCK_PACKAGE_NAME, str(tmp_path), diff --git a/tests/framework/project/test_settings.py b/tests/framework/project/test_settings.py index d74fb60d5b..74a2ac50ca 100644 --- a/tests/framework/project/test_settings.py +++ b/tests/framework/project/test_settings.py @@ -90,7 +90,10 @@ def test_settings_without_configure_project_shows_default_values(): assert settings.CONTEXT_CLASS is KedroContext assert settings.CONF_SOURCE == "conf" assert settings.CONFIG_LOADER_CLASS == OmegaConfigLoader - assert settings.CONFIG_LOADER_ARGS == {} + assert settings.CONFIG_LOADER_ARGS == { + "base_env": "base", + "default_run_env": "local", + } assert settings.DATA_CATALOG_CLASS == DataCatalog