Create fit evaluator, no resampling strategy and fix bug for test statistics

Fix mypy and flake

Fix check for X_test while making test data loader

fix bug in lookahead hyperparameters where lookahead was repeated for the hyperparameter name

Make passing tests in api easier

Fix bug in trainer weighted loss code for regression

Author: ravinkohli

autoPyTorch/api/base_task.py

@@ -40,7 +40,7 @@
 )
 from autoPyTorch.data.base_validator import BaseInputValidator
 from autoPyTorch.datasets.base_dataset import BaseDataset, BaseDatasetPropertiesType
-from autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutValTypes
+from autoPyTorch.datasets.resampling_strategy import CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes
 from autoPyTorch.ensemble.ensemble_builder import EnsembleBuilderManager
 from autoPyTorch.ensemble.singlebest_ensemble import SingleBest
 from autoPyTorch.evaluation.abstract_evaluator import fit_and_suppress_warnings
@@ -172,7 +172,9 @@ def __init__(
         include_components: Optional[Dict[str, Any]] = None,
         exclude_components: Optional[Dict[str, Any]] = None,
         backend: Optional[Backend] = None,
-        resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+        resampling_strategy: Union[CrossValTypes,
+                                   HoldoutValTypes,
+                                   NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation,
         resampling_strategy_args: Optional[Dict[str, Any]] = None,
         search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None,
         task_type: Optional[str] = None
@@ -1390,9 +1392,6 @@ def fit_pipeline(
         disable_file_output: Optional[List[Union[str, DisableFileOutputParameters]]] = None,
     ) -> Tuple[Optional[BasePipeline], RunInfo, RunValue, BaseDataset]:
         """
-        Fit a pipeline on the given task for the budget.
-        A pipeline configuration can be specified if None,
-        uses default
         Fit uses the estimator pipeline_config attribute, which the user
         can interact via the get_pipeline_config()/set_pipeline_config()
         methods.
@@ -1494,6 +1493,7 @@ def fit_pipeline(
             (BaseDataset):
                 Dataset created from the given tensors
         """
+        self.dataset_name = dataset.dataset_name
 
         if dataset is None:
             if (

autoPyTorch/api/tabular_classification.py

@@ -15,6 +15,7 @@
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
     HoldoutValTypes,
+    NoResamplingStrategyTypes
 )
 from autoPyTorch.datasets.tabular_dataset import TabularDataset
 from autoPyTorch.evaluation.utils import DisableFileOutputParameters
@@ -81,9 +82,11 @@ def __init__(
         output_directory: Optional[str] = None,
         delete_tmp_folder_after_terminate: bool = True,
         delete_output_folder_after_terminate: bool = True,
-        include_components: Optional[Dict[str, Any]] = None,
-        exclude_components: Optional[Dict[str, Any]] = None,
-        resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+        include_components: Optional[Dict] = None,
+        exclude_components: Optional[Dict] = None,
+        resampling_strategy: Union[CrossValTypes,
+                                   HoldoutValTypes,
+                                   NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation,
         resampling_strategy_args: Optional[Dict[str, Any]] = None,
         backend: Optional[Backend] = None,
         search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None

autoPyTorch/api/tabular_regression.py

@@ -15,6 +15,7 @@
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
     HoldoutValTypes,
+    NoResamplingStrategyTypes
 )
 from autoPyTorch.datasets.tabular_dataset import TabularDataset
 from autoPyTorch.evaluation.utils import DisableFileOutputParameters
@@ -82,9 +83,11 @@ def __init__(
         output_directory: Optional[str] = None,
         delete_tmp_folder_after_terminate: bool = True,
         delete_output_folder_after_terminate: bool = True,
-        include_components: Optional[Dict[str, Any]] = None,
-        exclude_components: Optional[Dict[str, Any]] = None,
-        resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+        include_components: Optional[Dict] = None,
+        exclude_components: Optional[Dict] = None,
+        resampling_strategy:Union[CrossValTypes,
+                                    HoldoutValTypes,
+                                    NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation,
         resampling_strategy_args: Optional[Dict[str, Any]] = None,
         backend: Optional[Backend] = None,
         search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None

autoPyTorch/datasets/base_dataset.py

@@ -21,7 +21,10 @@
     DEFAULT_RESAMPLING_PARAMETERS,
     HoldOutFunc,
     HoldOutFuncs,
-    HoldoutValTypes
+    HoldoutValTypes,
+    get_no_resampling_validators,
+    NoResamplingStrategyTypes,
+    NO_RESAMPLING_FN
 )
 from autoPyTorch.utils.common import FitRequirement
 
@@ -78,7 +81,9 @@ def __init__(
         dataset_name: Optional[str] = None,
         val_tensors: Optional[BaseDatasetInputType] = None,
         test_tensors: Optional[BaseDatasetInputType] = None,
-        resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+        resampling_strategy: Union[CrossValTypes,
+                                   HoldoutValTypes,
+                                   NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation,
         resampling_strategy_args: Optional[Dict[str, Any]] = None,
         shuffle: Optional[bool] = True,
         seed: Optional[int] = 42,
@@ -95,7 +100,7 @@ def __init__(
                 validation data
             test_tensors (An optional tuple of objects that have a __len__ and a __getitem__ attribute):
                 test data
-            resampling_strategy (Union[CrossValTypes, HoldoutValTypes]),
+            resampling_strategy (Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]),
                 (default=HoldoutValTypes.holdout_validation):
                 strategy to split the training data.
             resampling_strategy_args (Optional[Dict[str, Any]]): arguments
@@ -117,9 +122,16 @@ def __init__(
         if not hasattr(train_tensors[0], 'shape'):
             type_check(train_tensors, val_tensors)
         self.train_tensors, self.val_tensors, self.test_tensors = train_tensors, val_tensors, test_tensors
+<<<<<<< HEAD
         self.cross_validators: Dict[str, CrossValFunc] = {}
         self.holdout_validators: Dict[str, HoldOutFunc] = {}
         self.random_state = np.random.RandomState(seed=seed)
+=======
+        self.cross_validators: Dict[str, CROSS_VAL_FN] = {}
+        self.holdout_validators: Dict[str, HOLDOUT_FN] = {}
+        self.no_resampling_validators: Dict[str, NO_RESAMPLING_FN] = {}
+        self.rng = np.random.RandomState(seed=seed)
+>>>>>>> Fix mypy and flake
         self.shuffle = shuffle
         self.resampling_strategy = resampling_strategy
         self.resampling_strategy_args = resampling_strategy_args
@@ -144,6 +156,8 @@ def __init__(
         # Make sure cross validation splits are created once
         self.cross_validators = CrossValFuncs.get_cross_validators(*CrossValTypes)
         self.holdout_validators = HoldOutFuncs.get_holdout_validators(*HoldoutValTypes)
+        self.no_resampling_validators = get_no_resampling_validators(*NoResamplingStrategyTypes)
+
         self.splits = self.get_splits_from_resampling_strategy()
 
         # We also need to be able to transform the data, be it for pre-processing
@@ -211,7 +225,7 @@ def __len__(self) -> int:
     def _get_indices(self) -> np.ndarray:
         return self.random_state.permutation(len(self)) if self.shuffle else np.arange(len(self))
 
-    def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], List[int]]]:
+    def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], Optional[List[int]]]]:
         """
         Creates a set of splits based on a resampling strategy provided
 
@@ -242,6 +256,8 @@ def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], List[int]
                     num_splits=cast(int, num_splits),
                 )
             )
+        elif isinstance(self.resampling_strategy, NoResamplingStrategyTypes):
+            splits.append((self.no_resampling_validators[self.resampling_strategy.name](self._get_indices()), None))
         else:
             raise ValueError(f"Unsupported resampling strategy={self.resampling_strategy}")
         return splits
@@ -313,7 +329,7 @@ def create_holdout_val_split(
             self.random_state, val_share, self._get_indices(), **kwargs)
         return train, val
 
-    def get_dataset_for_training(self, split_id: int) -> Tuple[Dataset, Dataset]:
+    def get_dataset_for_training(self, split_id: int, train: bool) -> Dataset:
         """
         The above split methods employ the Subset to internally subsample the whole dataset.
 
@@ -327,8 +343,7 @@ def get_dataset_for_training(self, split_id: int) -> Tuple[Dataset, Dataset]:
             Dataset: the reduced dataset to be used for testing
         """
         # Subset creates a dataset. Splits is a (train_indices, test_indices) tuple
-        return (TransformSubset(self, self.splits[split_id][0], train=True),
-                TransformSubset(self, self.splits[split_id][1], train=False))
+        return TransformSubset(self, self.splits[split_id][0], train=train)
 
     def replace_data(self, X_train: BaseDatasetInputType,
                      X_test: Optional[BaseDatasetInputType]) -> 'BaseDataset':

autoPyTorch/datasets/image_dataset.py

@@ -24,6 +24,7 @@
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
     HoldoutValTypes,
+    NoResamplingStrategyTypes
 )
 
 IMAGE_DATASET_INPUT = Union[Dataset, Tuple[Union[np.ndarray, List[str]], np.ndarray]]
@@ -39,7 +40,7 @@ class ImageDataset(BaseDataset):
             validation data
         test (Union[Dataset, Tuple[Union[np.ndarray, List[str]], np.ndarray]]):
             testing data
-        resampling_strategy (Union[CrossValTypes, HoldoutValTypes]),
+        resampling_strategy (Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]),
             (default=HoldoutValTypes.holdout_validation):
             strategy to split the training data.
         resampling_strategy_args (Optional[Dict[str, Any]]): arguments
@@ -57,7 +58,9 @@ def __init__(self,
                  train: IMAGE_DATASET_INPUT,
                  val: Optional[IMAGE_DATASET_INPUT] = None,
                  test: Optional[IMAGE_DATASET_INPUT] = None,
-                 resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+                 resampling_strategy: Union[CrossValTypes,
+                                            HoldoutValTypes,
+                                            NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation,
                  resampling_strategy_args: Optional[Dict[str, Any]] = None,
                  shuffle: Optional[bool] = True,
                  seed: Optional[int] = 42,

autoPyTorch/datasets/resampling_strategy.py

@@ -32,6 +32,11 @@ def __call__(self, random_state: np.random.RandomState, val_share: float,
         ...
 
 
+class NO_RESAMPLING_FN(Protocol):
+    def __call__(self, indices: np.ndarray) -> np.ndarray:
+        ...
+
+
 class CrossValTypes(IntEnum):
     """The type of cross validation
 
@@ -76,8 +81,14 @@ def is_stratified(self) -> bool:
         return getattr(self, self.name) in stratified
 
 
+class NoResamplingStrategyTypes(IntEnum):
+    no_resampling = 8
+    shuffle_no_resampling = 9
+
+
 # TODO: replace it with another way
-RESAMPLING_STRATEGIES = [CrossValTypes, HoldoutValTypes]
+RESAMPLING_STRATEGIES = [CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]
+
 
 DEFAULT_RESAMPLING_PARAMETERS: Dict[Union[HoldoutValTypes, CrossValTypes], Dict[str, Any]] = {
     HoldoutValTypes.holdout_validation: {
@@ -98,7 +109,13 @@ def is_stratified(self) -> bool:
     CrossValTypes.time_series_cross_validation: {
         'num_splits': 5,
     },
-}
+    NoResamplingStrategyTypes.no_resampling: {
+        'shuffle': False
+    },
+    NoResamplingStrategyTypes.shuffle_no_resampling: {
+        'shuffle': True
+    }
+}  # type: Dict[Union[HoldoutValTypes, CrossValTypes, NoResamplingStrategyTypes], Dict[str, Any]]
 
 
 class HoldOutFuncs():
@@ -225,3 +242,55 @@ def get_cross_validators(cls, *cross_val_types: CrossValTypes) -> Dict[str, Cros
             for cross_val_type in cross_val_types
         }
         return cross_validators
+
+
+def get_no_resampling_validators(*no_resampling: NoResamplingStrategyTypes) -> Dict[str, NO_RESAMPLING_FN]:
+    no_resampling_strategies = {}  # type: Dict[str, NO_RESAMPLING_FN]
+    for strategy in no_resampling:
+        no_resampling_fn = globals()[strategy.name]
+        no_resampling_strategies[strategy.name] = no_resampling_fn
+    return no_resampling_strategies
+
+
+def no_resampling(indices: np.ndarray) -> np.ndarray:
+    """
+    Returns the indices without performing
+    any operation on them. To be used for
+    fitting on the whole dataset.
+    This strategy is not compatible with
+    HPO search.
+    Args:
+        indices:  array of indices
+
+    Returns:
+        np.ndarray: array of indices
+    """
+    return indices
+
+
+def shuffle_no_resampling(indices: np.ndarray, **kwargs: Any) -> np.ndarray:
+    """
+    Returns the indices after shuffling them.
+    To be used for fitting on the whole dataset.
+    This strategy is not compatible with HPO search.
+    Args:
+        indices:  array of indices
+
+    Returns:
+        np.ndarray: shuffled array of indices
+    """
+    if 'random_state' in kwargs:
+        if isinstance(kwargs['random_state'], np.random.RandomState):
+            kwargs['random_state'].shuffle(indices)
+        elif isinstance(kwargs['random_state'], int):
+            np.random.seed(kwargs['random_state'])
+            np.random.shuffle(indices)
+        else:
+            raise ValueError("Illegal value for 'random_state' entered. "
+                             "Expected it to be {} or {} but got {}".format(int,
+                                                                            np.random.RandomState,
+                                                                            type(kwargs['random_state'])))
+    else:
+        np.random.shuffle(indices)
+
+    return indices

autoPyTorch/datasets/tabular_dataset.py

@@ -21,6 +21,7 @@
 from autoPyTorch.datasets.resampling_strategy import (
     CrossValTypes,
     HoldoutValTypes,
+    NoResamplingStrategyTypes
 )
 
 
@@ -32,7 +33,7 @@ class TabularDataset(BaseDataset):
             Y (Union[np.ndarray, pd.Series]): training data targets.
             X_test (Optional[Union[np.ndarray, pd.DataFrame]]):  input testing data.
             Y_test (Optional[Union[np.ndarray, pd.DataFrame]]): testing data targets
-            resampling_strategy (Union[CrossValTypes, HoldoutValTypes]),
+            resampling_strategy (Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]),
                 (default=HoldoutValTypes.holdout_validation):
                 strategy to split the training data.
             resampling_strategy_args (Optional[Dict[str, Any]]):
@@ -55,7 +56,9 @@ def __init__(self,
                  Y: Union[np.ndarray, pd.Series],
                  X_test: Optional[Union[np.ndarray, pd.DataFrame]] = None,
                  Y_test: Optional[Union[np.ndarray, pd.DataFrame]] = None,
-                 resampling_strategy: Union[CrossValTypes, HoldoutValTypes] = HoldoutValTypes.holdout_validation,
+                 resampling_strategy: Union[CrossValTypes,
+                                            HoldoutValTypes,
+                                            NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation,
                  resampling_strategy_args: Optional[Dict[str, Any]] = None,
                  shuffle: Optional[bool] = True,
                  seed: Optional[int] = 42,

autoPyTorch/evaluation/abstract_evaluator.py

@@ -663,9 +663,9 @@ def _loss(self, y_true: np.ndarray, y_hat: np.ndarray) -> Dict[str, float]:
             y_true, y_hat, self.task_type, metrics)
 
     def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float],
-                  opt_pred: np.ndarray, valid_pred: Optional[np.ndarray],
-                  test_pred: Optional[np.ndarray], additional_run_info: Optional[Dict],
-                  file_output: bool, status: StatusType
+                  valid_pred: Optional[np.ndarray], test_pred: Optional[np.ndarray],
+                  additional_run_info: Optional[Dict], file_output: bool, status: StatusType,
+                  opt_pred: Optional[np.ndarray],
                   ) -> Optional[Tuple[float, float, int, Dict]]:
         """This function does everything necessary after the fitting is done:
 
@@ -707,6 +707,9 @@ def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float],
                 Additional run information, like train/test loss
         """
 
+        assert opt_pred is not None, "Cases where 'opt_pred' is None should be handled " \
+                                     "specifically with special child classes"
+
         self.duration = time.time() - self.starttime
 
         if file_output: