`GridSearch` fails when using `grid_search(pipeline_space)`

[timurcarstensen]

@dataclass
class GridSearch:
    """Evaluates a fixed list of configurations in order."""

    pipeline_space: SearchSpace
    """The search space from which the configurations are derived."""

    configs_list: list[dict[str, Any]]
    """The list of configurations to evaluate."""

    def __call__(
        self,
        trials: Mapping[str, Trial],
        budget_info: BudgetInfo | None,
        n: int | None = None,
    ) -> SampledConfig | list[SampledConfig]:
        assert n is None, "TODO"
        _num_previous_configs = len(trials)
        if _num_previous_configs > len(self.configs_list) - 1:
            raise ValueError("Grid search exhausted!")

        rng = random.Random()
        configs = rng.sample(self.configs_list, len(self.configs_list))

        config = configs[_num_previous_configs]
        config_id = str(_num_previous_configs)
        return SampledConfig(config=config, id=config_id, previous_config_id=None)

budget_info: BudgetInfo | None should be budget_info: BudgetInfo | None = None, right?
@eddiebergman

[eddiebergman]

Nope, the signature is this:

neps/neps/optimizers/optimizer.py

Lines 57 to 79 in 1c7f9d2

	class AskFunction(Protocol):
	"""Interface to implement the ask of optimizer."""
	@abstractmethod
	def __call__(
	self,
	trials: Mapping[str, Trial],
	budget_info: BudgetInfo | None,
	n: int | None = None,
	) -> SampledConfig | list[SampledConfig]:
	"""Sample a new configuration.
	Args:
	trials: All of the trials that are known about.
	budget_info: information about the budget constraints.
	n: The number of configurations to sample. If you do not support
	sampling multiple configurations at once, you should raise
	a `ValueError`.
	Returns:
	The sampled configuration(s)
	"""
	...

Are you using it in the vanilla way, i.e. neps.run(..., algorithm="grid_search")

Ahh, I see I never added this to the set of optimizers that gets tested, I would have assumed it work but apparently it wasn't there:

neps/tests/test_state/test_neps_state.py

Lines 83 to 123 in 1c7f9d2

	JUST_SKIP = [
	"multifidelity_tpe",
	]
	OPTIMIZER_FAILS_WITH_FIDELITY = [
	"random_search",
	"bayesian_optimization_cost_aware",
	"bayesian_optimization",
	"bayesian_optimization_prior",
	"pibo",
	"cost_cooling_bayesian_optimization",
	"cost_cooling",
	]
	# There's no programattic way to check if a class requires a fidelity.
	# See issue #118, #119, #120
	OPTIMIZER_REQUIRES_FIDELITY = [
	"successive_halving",
	"successive_halving_prior",
	"asha",
	"asha_prior",
	"hyperband",
	"hyperband_prior",
	"async_hb",
	"async_hb_prior",
	"priorband",
	"priorband_sh",
	"priorband_asha",
	"priorband_async",
	"priorband_bo",
	"bayesian_optimization_cost_aware",
	"mobster",
	"ifbo",
	]
	REQUIRES_PRIOR = {
	"priorband",
	"priorband_bo",
	"priorband_asha",
	"priorband_asha_hyperband",
	}
	REQUIRES_COST = ["cost_cooling_bayesian_optimization", "cost_cooling"]