Additional metrics during train (#194)

* Added additional metrics to fit dictionary

* Added in test also

Fix mypy and flake after rebase, added random state to mixup and cutout and changs no resampling for new code

fix bug in setup.py

Author: ravinkohli

autoPyTorch/datasets/base_dataset.py

@@ -82,13 +82,7 @@ def __init__(
         dataset_name: Optional[str] = None,
         val_tensors: Optional[BaseDatasetInputType] = None,
         test_tensors: Optional[BaseDatasetInputType] = None,
-<<<<<<< HEAD
         resampling_strategy: ResamplingStrategies = HoldoutValTypes.holdout_validation,
-=======
-        resampling_strategy: Union[CrossValTypes,
-                                   HoldoutValTypes,
-                                   NoResamplingStrategyTypes] = HoldoutValTypes.holdout_validation,
->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics
         resampling_strategy_args: Optional[Dict[str, Any]] = None,
         shuffle: Optional[bool] = True,
         seed: Optional[int] = 42,
@@ -105,12 +99,7 @@ def __init__(
                 validation data
             test_tensors (An optional tuple of objects that have a __len__ and a __getitem__ attribute):
                 test data
-<<<<<<< HEAD
             resampling_strategy (RESAMPLING_STRATEGIES: default=HoldoutValTypes.holdout_validation):
-=======
-            resampling_strategy (Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]),
-                (default=HoldoutValTypes.holdout_validation):
->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics
                 strategy to split the training data.
             resampling_strategy_args (Optional[Dict[str, Any]]): arguments
                 required for the chosen resampling strategy. If None, uses
@@ -132,17 +121,11 @@ def __init__(
         if not hasattr(train_tensors[0], 'shape'):
             type_check(train_tensors, val_tensors)
         self.train_tensors, self.val_tensors, self.test_tensors = train_tensors, val_tensors, test_tensors
-<<<<<<< HEAD
         self.cross_validators: Dict[str, CrossValFunc] = {}
         self.holdout_validators: Dict[str, HoldOutFunc] = {}
         self.no_resampling_validators: Dict[str, NoResamplingFunc] = {}
         self.random_state = np.random.RandomState(seed=seed)
-=======
-        self.cross_validators: Dict[str, CROSS_VAL_FN] = {}
-        self.holdout_validators: Dict[str, HOLDOUT_FN] = {}
-        self.no_resampling_validators: Dict[str, NO_RESAMPLING_FN] = {}
-        self.rng = np.random.RandomState(seed=seed)
->>>>>>> Fix mypy and flake
+        self.no_resampling_validators: Dict[str, NoResamplingFunc] = {}
         self.shuffle = shuffle
         self.resampling_strategy = resampling_strategy
         self.resampling_strategy_args = resampling_strategy_args
@@ -167,11 +150,8 @@ def __init__(
         # Make sure cross validation splits are created once
         self.cross_validators = CrossValFuncs.get_cross_validators(*CrossValTypes)
         self.holdout_validators = HoldOutFuncs.get_holdout_validators(*HoldoutValTypes)
-<<<<<<< HEAD
+
         self.no_resampling_validators = NoResamplingFuncs.get_no_resampling_validators(*NoResamplingStrategyTypes)
-=======
-        self.no_resampling_validators = get_no_resampling_validators(*NoResamplingStrategyTypes)
->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics
 
         self.splits = self.get_splits_from_resampling_strategy()
 
@@ -272,12 +252,8 @@ def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], Optional[
                 )
             )
         elif isinstance(self.resampling_strategy, NoResamplingStrategyTypes):
-<<<<<<< HEAD
             splits.append((self.no_resampling_validators[self.resampling_strategy.name](self.random_state,
                                                                                         self._get_indices()), None))
-=======
-            splits.append((self.no_resampling_validators[self.resampling_strategy.name](self._get_indices()), None))
->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics
         else:
             raise ValueError(f"Unsupported resampling strategy={self.resampling_strategy}")
         return splits
@@ -349,11 +325,7 @@ def create_holdout_val_split(
             self.random_state, val_share, self._get_indices(), **kwargs)
         return train, val
 
-<<<<<<< HEAD
     def get_dataset(self, split_id: int, train: bool) -> Dataset:
-=======
-    def get_dataset_for_training(self, split_id: int, train: bool) -> Dataset:
->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics
         """
         The above split methods employ the Subset to internally subsample the whole dataset.
 
@@ -368,7 +340,6 @@ def get_dataset_for_training(self, split_id: int, train: bool) -> Dataset:
             Dataset: the reduced dataset to be used for testing
         """
         # Subset creates a dataset. Splits is a (train_indices, test_indices) tuple
-<<<<<<< HEAD
         if split_id >= len(self.splits):  # old version: split_id > len(self.splits)
             raise IndexError(f"self.splits index out of range, got split_id={split_id}"
                              f" (>= num_splits={len(self.splits)})")
@@ -377,9 +348,6 @@ def get_dataset_for_training(self, split_id: int, train: bool) -> Dataset:
             raise ValueError("Specified fold (or subset) does not exist")
 
         return TransformSubset(self, indices, train=train)
-=======
-        return TransformSubset(self, self.splits[split_id][0], train=train)
->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics
 
     def replace_data(self, X_train: BaseDatasetInputType,
                      X_test: Optional[BaseDatasetInputType]) -> 'BaseDataset':

autoPyTorch/datasets/resampling_strategy.py

@@ -39,8 +39,10 @@ def __call__(self, random_state: np.random.RandomState, val_share: float,
         ...
 
 
-class NO_RESAMPLING_FN(Protocol):
-    def __call__(self, indices: np.ndarray) -> np.ndarray:
+class NoResamplingFunc(Protocol):
+    def __call__(self,
+                 random_state: np.random.RandomState,
+                 indices: np.ndarray) -> np.ndarray:
         ...
 
 
@@ -90,22 +92,13 @@ def is_stratified(self) -> bool:
 
 class NoResamplingStrategyTypes(IntEnum):
     no_resampling = 8
-<<<<<<< HEAD
 
     def is_stratified(self) -> bool:
         return False
 
 
 # TODO: replace it with another way
 ResamplingStrategies = Union[CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]
-=======
-    shuffle_no_resampling = 9
-
-
-# TODO: replace it with another way
-RESAMPLING_STRATEGIES = [CrossValTypes, HoldoutValTypes, NoResamplingStrategyTypes]
-
->>>>>>> Create fit evaluator, no resampling strategy and fix bug for test statistics
 
 DEFAULT_RESAMPLING_PARAMETERS: Dict[
     ResamplingStrategies,

autoPyTorch/pipeline/components/training/trainer/cutout_utils.py

@@ -10,6 +10,8 @@
 
 import numpy as np
 
+from sklearn.utils import check_random_state
+
 from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES
 from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter
@@ -35,7 +37,12 @@ def __init__(self, patch_ratio: float,
         """
         self.use_stochastic_weight_averaging = use_stochastic_weight_averaging
         self.weighted_loss = weighted_loss
-        self.random_state = random_state
+        if random_state is None:
+            # A trainer components need a random state for
+            # sampling -- for example in MixUp training
+            self.random_state = check_random_state(1)
+        else:
+            self.random_state = random_state
         self.use_snapshot_ensemble = use_snapshot_ensemble
         self.se_lastk = se_lastk
         self.use_lookahead_optimizer = use_lookahead_optimizer

autoPyTorch/pipeline/components/training/trainer/mixup_utils.py

@@ -10,6 +10,8 @@
 
 import numpy as np
 
+from sklearn.utils import check_random_state
+
 from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES
 from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead
 from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter
@@ -34,7 +36,12 @@ def __init__(self, alpha: float,
         """
         self.use_stochastic_weight_averaging = use_stochastic_weight_averaging
         self.weighted_loss = weighted_loss
-        self.random_state = random_state
+        if random_state is None:
+            # A trainer components need a random state for
+            # sampling -- for example in MixUp training
+            self.random_state = check_random_state(1)
+        else:
+            self.random_state = random_state
         self.use_snapshot_ensemble = use_snapshot_ensemble
         self.se_lastk = se_lastk
         self.use_lookahead_optimizer = use_lookahead_optimizer

setup.py

@@ -58,7 +58,7 @@
             "pre-commit",
             "pytest-cov",
             'pytest-forked',
-            "pytest-mock"
+            "pytest-mock",
             "codecov",
             "pep8",
             "mypy",

test/test_api/test_api.py

@@ -36,7 +36,7 @@
 from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner import _traditional_learners
 from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy
 
-from test.test_api.api_utils import print_debug_information
+from test.test_api.api_utils import print_debug_information  # noqa E402
 
 
 CV_NUM_SPLITS = 2