Integration WIP

- Core tests passing

hydra/_internal/config_loader_impl.py

@@ -26,9 +26,11 @@
     convert_overrides_to_defaults,
     expand_defaults_list,
 )
+from hydra._internal.new_defaults_list import create_defaults_list
 from hydra.core import DefaultElement
-from hydra.core.config_loader import ConfigLoader, LoadTrace
+from hydra.core.config_loader import ConfigLoader, LoadTrace, LoadTrace2
 from hydra.core.config_search_path import ConfigSearchPath
+from hydra.core.new_default_element import ResultDefault
 from hydra.core.object_type import ObjectType
 from hydra.core.override_parser.overrides_parser import OverridesParser
 from hydra.core.override_parser.types import Override, ValueType
@@ -173,17 +175,115 @@ def load_configuration(
         from_shell: bool = True,
     ) -> DictConfig:
         try:
-            cfg = self._load_configuration_impl(
+            cfg = self._load_configuration_impl2(
                 config_name=config_name,
                 overrides=overrides,
                 run_mode=run_mode,
                 strict=strict,
                 from_shell=from_shell,
             )
+            #
+            # cfg = self._load_configuration_impl(
+            #     config_name=config_name,
+            #     overrides=overrides,
+            #     run_mode=run_mode,
+            #     strict=strict,
+            #     from_shell=from_shell,
+            # )
+            #
+            # def are_same(cfg1, cfg2) -> bool:
+            #     cfg1 = copy.deepcopy(cfg1)
+            #     cfg2 = copy.deepcopy(cfg2)
+            #     cfg1.hydra.composition_trace = []
+            #     cfg2.hydra.composition_trace = []
+            #
+            #     return cfg1 == cfg2
+            # assert are_same(cfg, cfg2)
             return cfg
         except OmegaConfBaseException as e:
             raise ConfigCompositionException().with_traceback(sys.exc_info()[2]) from e
 
+    def _load_configuration_impl2(
+        self,
+        config_name: Optional[str],
+        overrides: List[str],
+        run_mode: RunMode,
+        strict: Optional[bool] = None,
+        from_shell: bool = True,
+    ) -> DictConfig:
+        self.ensure_main_config_source_available()
+        caching_repo = CachingConfigRepository(self.repository)
+
+        if config_name is not None and not caching_repo.config_exists(config_name):
+            self._missing_config_error(
+                config_name=config_name,
+                msg=f"Cannot find primary config : {config_name}, check that it's in your config search path",
+                with_search_path=True,
+            )
+
+        if strict is None:
+            strict = self.default_strict
+
+        parser = OverridesParser.create()
+        parsed_overrides = parser.parse_overrides(overrides=overrides)
+
+        # TODO: decide where the error checking in this function should be.
+        #  This function can parse and validate the overrides without splitting them into types
+        config_overrides = ConfigLoaderImpl.parse_overrides(
+            overrides=overrides, run_mode=run_mode, from_shell=from_shell
+        )
+
+        defaults_list = create_defaults_list(
+            repo=caching_repo,
+            config_name=config_name,
+            overrides_list=parser.parse_overrides(overrides=overrides),
+            prepend_hydra=True,
+            skip_missing=run_mode == RunMode.MULTIRUN,
+        )
+
+        config_overrides = defaults_list.config_overrides
+
+        cfg, composition_trace = self._compose_config_from_defaults_list2(
+            primary_config_name=config_name,
+            defaults=defaults_list.defaults,
+            repo=caching_repo,
+        )
+
+        OmegaConf.set_struct(cfg, strict)
+        OmegaConf.set_readonly(cfg.hydra, False)
+
+        # Apply command line overrides after enabling strict flag
+        ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
+        app_overrides = []
+        for override in parsed_overrides:
+            if override.is_hydra_override():
+                cfg.hydra.overrides.hydra.append(override.input_line)
+            else:
+                cfg.hydra.overrides.task.append(override.input_line)
+                app_overrides.append(override)
+
+        with open_dict(cfg.hydra):
+            from hydra import __version__
+
+            cfg.hydra.runtime.version = __version__
+            cfg.hydra.runtime.cwd = os.getcwd()
+
+            cfg.hydra.composition_trace = composition_trace
+            if "name" not in cfg.hydra.job:
+                cfg.hydra.job.name = JobRuntime().get("name")
+            cfg.hydra.job.override_dirname = get_overrides_dirname(
+                overrides=app_overrides,
+                kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
+                item_sep=cfg.hydra.job.config.override_dirname.item_sep,
+                exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys,
+            )
+            cfg.hydra.job.config_name = config_name
+
+            for key in cfg.hydra.job.env_copy:
+                cfg.hydra.job.env_set[key] = os.environ[key]
+
+        return cfg
+
     def _load_configuration_impl(
         self,
         config_name: Optional[str],
@@ -353,6 +453,77 @@ def _apply_overrides_to_config(overrides: List[Override], cfg: DictConfig) -> No
                     f"Error merging override {override.input_line}"
                 ) from ex
 
+    def _load_single_config2(
+        self, default: ResultDefault, repo: IConfigRepository, is_primary: bool
+    ) -> Tuple[ConfigResult, LoadTrace2]:
+        config_path = default.config_path
+        package = default.package
+
+        ret = repo.load_config(
+            config_path=config_path,
+            is_primary_config=is_primary,
+            package_override=package,
+        )
+        assert ret is not None
+
+        if not isinstance(ret.config, DictConfig):
+            raise ValueError(
+                f"Config {config_path} must be a Dictionary, got {type(ret).__name__}"
+            )
+
+        schema_provider = None
+        if not ret.is_schema_source:
+            schema = None
+            try:
+                schema_source = repo.get_schema_source()
+                schema = schema_source.load_config(
+                    ConfigSource._normalize_file_name(filename=config_path),
+                    is_primary_config=is_primary,
+                    # TODO: really weird that this seems to work.
+                    #  hopefully things will be cleaner after package cleanup
+                    package_override=None,
+                )
+            except ConfigLoadError:
+                # schema not found, ignore
+                pass
+
+            if schema is not None:
+                try:
+                    # if config has a hydra node, remove it during validation and add it back.
+                    # This allows overriding Hydra's configuration without declaring this node
+                    # in every program
+                    hydra = None
+                    hydra_config_group = (
+                        default.config_path is not None
+                        and default.config_path.startswith("hydra/")
+                    )
+                    if "hydra" in ret.config and not hydra_config_group:
+                        hydra = ret.config.pop("hydra")
+                    schema_provider = schema.provider
+                    merged = OmegaConf.merge(schema.config, ret.config)
+                    assert isinstance(merged, DictConfig)
+
+                    if hydra is not None:
+                        with open_dict(merged):
+                            merged.hydra = hydra
+                    ret.config = merged
+                except OmegaConfBaseException as e:
+                    raise ConfigCompositionException(
+                        f"Error merging '{config_path}' with schema"
+                    ) from e
+
+                assert isinstance(merged, DictConfig)
+
+        trace = LoadTrace2(
+            config_path=default.config_path,
+            package=default.package,
+            parent=default.parent,
+            is_self=default.is_self,
+            search_path=ret.path,
+            provider=ret.provider,
+        )
+        return ret, trace
+
     def _load_single_config(
         self,
         default: DefaultElement,
@@ -437,6 +608,48 @@ def get_group_options(
     ) -> List[str]:
         return self.repository.get_group_options(group_name, results_filter)
 
+    def _compose_config_from_defaults_list2(
+        self,
+        primary_config_name: str,
+        defaults: List[ResultDefault],
+        repo: IConfigRepository,
+    ) -> Tuple[DictConfig, List[LoadTrace2]]:
+        composition_trace = []
+        cfg = OmegaConf.create()
+        for default in defaults:
+            # TODO: if primary is critical return it in ResultDefault
+            primary = default.config_path == primary_config_name
+
+            loaded, trace = self._load_single_config2(
+                default=default, repo=repo, is_primary=primary
+            )
+            merged = OmegaConf.merge(cfg, loaded.config)
+            assert isinstance(merged, DictConfig)
+            cfg = merged
+            assert cfg is not None
+            composition_trace.append(trace)
+            # if default.skip_load:
+            #     trace = LoadTrace(
+            #         config_group=default.config_group,
+            #         config_name=default.config_name,
+            #         package=default.get_subject_package(),
+            #         parent=default.parent,
+            #         skip_reason=default.skip_load_reason,
+            #     )
+            # else:
+            #     loaded, trace = self._load_single_config(default=default, repo=repo)
+            #     # should not happen, generation of defaults list already verified that this exists
+            #     merged = OmegaConf.merge(cfg, loaded.config)
+            #     assert isinstance(merged, DictConfig)
+            #     cfg = merged
+            #     assert cfg is not None
+            # composition_trace.append(trace)
+
+        # This is primarily cosmetic
+        cfg._metadata.ref_type = cfg._metadata.object_type
+
+        return cfg, composition_trace
+
     def _compose_config_from_defaults_list(
         self,
         defaults: List[DefaultElement],

hydra/_internal/hydra.py

@@ -11,7 +11,7 @@
 from omegaconf import Container, DictConfig, OmegaConf, open_dict
 
 from hydra._internal.utils import get_column_widths, run_and_report
-from hydra.core.config_loader import ConfigLoader, LoadTrace
+from hydra.core.config_loader import ConfigLoader, LoadTrace2
 from hydra.core.config_search_path import ConfigSearchPath
 from hydra.core.hydra_config import HydraConfig
 from hydra.core.plugins import Plugins
@@ -411,44 +411,40 @@ def _print_composition_trace(self, cfg: DictConfig) -> None:
         self._log_header("Composition trace", filler="*")
         box: List[List[str]] = [
             [
-                "Config group",
-                "Config name",
+                "Config Path",
+                "Package",
                 "Search path",
                 "Provider",
-                "Schema provider",
             ]
         ]
-        composition_trace = [LoadTrace(**x) for x in cfg.hydra.composition_trace]
+        composition_trace = [LoadTrace2(**x) for x in cfg.hydra.composition_trace]
         for trace in composition_trace:
             box.append(
                 [
-                    trace.config_group if trace.config_group is not None else "",
-                    trace.config_name if trace.config_name is not None else "",
+                    trace.config_path if trace.config_path is not None else "",
+                    trace.package if trace.package is not None else "",
                     trace.search_path if trace.search_path is not None else "",
                     trace.provider if trace.provider is not None else "",
-                    trace.schema_provider if trace.schema_provider is not None else "",
                 ]
             )
         padding = get_column_widths(box)
         del box[0]
 
-        header = "| {} | {} | {} | {} | {} |".format(
-            "Config group".ljust(padding[0]),
-            "Config name".ljust(padding[1]),
+        header = "| {} | {} | {} | {} |".format(
+            "Config Path".ljust(padding[0]),
+            "Package".ljust(padding[1]),
             "Search path".ljust(padding[2]),
             "Provider".ljust(padding[3]),
-            "Schema provider".ljust(padding[4]),
         )
         self._log_header(header=header, filler="-")
 
         for row in box:
             log.debug(
-                "| {} | {} | {} | {} | {} |".format(
+                "| {} | {} | {} | {} |".format(
                     row[0].ljust(padding[0]),
                     row[1].ljust(padding[1]),
                     row[2].ljust(padding[2]),
                     row[3].ljust(padding[3]),
-                    row[4].ljust(padding[4]),
                 )
             )
 

hydra/conf/__init__.py

@@ -156,11 +156,28 @@ class HydraConf:
     verbose: Any = False
 
 
-cstore = ConfigStore.instance()
+cs = ConfigStore.instance()
 
-cstore.store(
+cs.store(
     name="hydra_config",
     node=HydraConf,
     package="hydra",
     provider="hydra",
 )
+
+cs.store(
+    group="hydra",
+    name="config",
+    node=HydraConf(
+        defaults=[
+            {"hydra_logging": "default"},
+            {"job_logging": "default"},
+            {"launcher": "basic"},
+            {"sweeper": "basic"},
+            {"output": "default"},
+            {"help": "default"},
+            {"hydra_help": "default"},
+        ]
+    ),
+    provider="hydra",
+)

hydra/core/config_loader.py

@@ -11,6 +11,7 @@
 from hydra.types import RunMode
 
 
+# TODO: cleanup old LogTrace
 @dataclass
 class LoadTrace:
     config_group: Optional[str] = None
@@ -23,6 +24,16 @@ class LoadTrace:
     skip_reason: Optional[str] = None
 
 
+@dataclass
+class LoadTrace2:
+    config_path: Optional[str] = None
+    package: Optional[str] = None
+    parent: Optional[str] = None
+    is_self: bool = False
+    search_path: Optional[str] = None
+    provider: Optional[str] = None
+
+
 class ConfigLoader(ABC):
     """
     Config loader interface

tests/defaults_list/test_defaults_list.py

@@ -81,7 +81,6 @@
 #  - (Y) Test deprecation message when attempting to override hydra configs without override: true
 #  - (Y) duplicate entries in final defaults list raises an error
 
-# TODO: Integrate with Hydra
 #  - (Y) replace old defaults list computation
 #  - (Y) enable --info=defaults output
 #  - (Y) ensure all tests are passing

tests/test_basic_launcher.py

@@ -19,9 +19,9 @@ class TestBasicLauncher(LauncherTestSuite):
         pytest.param(
             {
                 "defaults": [
-                    {"hydra/launcher": "basic"},
-                    {"hydra/hydra_logging": "hydra_debug"},
-                    {"hydra/job_logging": "disabled"},
+                    {"hydra/launcher": "basic", "override": True},
+                    {"hydra/hydra_logging": "hydra_debug", "override": True},
+                    {"hydra/job_logging": "disabled", "override": True},
                 ]
             },
             ["-m"],