[FIX] fix seed in splits

Author: franchuterivera

autoPyTorch/datasets/base_dataset.py

@@ -118,7 +118,7 @@ def __init__(
         self.train_tensors, self.val_tensors, self.test_tensors = train_tensors, val_tensors, test_tensors
         self.cross_validators: Dict[str, CrossValFunc] = {}
         self.holdout_validators: Dict[str, HoldOutFunc] = {}
-        self.rng = np.random.RandomState(seed=seed)
+        self.random_state = np.random.RandomState(seed=seed)
         self.shuffle = shuffle
         self.resampling_strategy = resampling_strategy
         self.resampling_strategy_args = resampling_strategy_args
@@ -205,7 +205,7 @@ def __len__(self) -> int:
         return self.train_tensors[0].shape[0]
 
     def _get_indices(self) -> np.ndarray:
-        return self.rng.permutation(len(self)) if self.shuffle else np.arange(len(self))
+        return self.random_state.permutation(len(self)) if self.shuffle else np.arange(len(self))
 
     def get_splits_from_resampling_strategy(self) -> List[Tuple[List[int], List[int]]]:
         """
@@ -271,7 +271,7 @@ def create_cross_val_splits(
             # we need additional information about the data for stratification
             kwargs["stratify"] = self.train_tensors[-1]
         splits = self.cross_validators[cross_val_type.name](
-            num_splits, self._get_indices(), **kwargs)
+            self.random_state, num_splits, self._get_indices(), **kwargs)
         return splits
 
     def create_holdout_val_split(
@@ -305,7 +305,8 @@ def create_holdout_val_split(
         if holdout_val_type.is_stratified():
             # we need additional information about the data for stratification
             kwargs["stratify"] = self.train_tensors[-1]
-        train, val = self.holdout_validators[holdout_val_type.name](val_share, self._get_indices(), **kwargs)
+        train, val = self.holdout_validators[holdout_val_type.name](
+            self.random_state, val_share, self._get_indices(), **kwargs)
         return train, val
 
     def get_dataset_for_training(self, split_id: int) -> Tuple[Dataset, Dataset]:

autoPyTorch/datasets/resampling_strategy.py

@@ -18,14 +18,16 @@
 # Use callback protocol as workaround, since callable with function fields count 'self' as argument
 class CrossValFunc(Protocol):
     def __call__(self,
+                 random_state: np.random.RandomState,
                  num_splits: int,
                  indices: np.ndarray,
                  stratify: Optional[Any]) -> List[Tuple[np.ndarray, np.ndarray]]:
         ...
 
 
 class HoldOutFunc(Protocol):
-    def __call__(self, val_share: float, indices: np.ndarray, stratify: Optional[Any]
+    def __call__(self, random_state: np.random.RandomState, val_share: float,
+                 indices: np.ndarray, stratify: Optional[Any]
                  ) -> Tuple[np.ndarray, np.ndarray]:
         ...
 
@@ -85,35 +87,42 @@ def is_stratified(self) -> bool:
         'val_share': 0.33,
     },
     CrossValTypes.k_fold_cross_validation: {
-        'num_splits': 3,
+        'num_splits': 5,
     },
     CrossValTypes.stratified_k_fold_cross_validation: {
-        'num_splits': 3,
+        'num_splits': 5,
     },
     CrossValTypes.shuffle_split_cross_validation: {
-        'num_splits': 3,
+        'num_splits': 5,
     },
     CrossValTypes.time_series_cross_validation: {
-        'num_splits': 3,
+        'num_splits': 5,
     },
 }  # type: Dict[Union[HoldoutValTypes, CrossValTypes], Dict[str, Any]]
 
 
 class HoldOutFuncs():
     @staticmethod
-    def holdout_validation(val_share: float,
+    def holdout_validation(random_state: np.random.RandomState,
+                           val_share: float,
                            indices: np.ndarray,
                            **kwargs: Any
                            ) -> Tuple[np.ndarray, np.ndarray]:
-        train, val = train_test_split(indices, test_size=val_share, shuffle=False)
+        shuffle = kwargs.get('shuffle', True)
+        train, val = train_test_split(indices, test_size=val_share,
+                                      shuffle=shuffle,
+                                      random_state=random_state if shuffle else None,
+                                      )
         return train, val
 
     @staticmethod
-    def stratified_holdout_validation(val_share: float,
+    def stratified_holdout_validation(random_state: np.random.RandomState,
+                                      val_share: float,
                                       indices: np.ndarray,
                                       **kwargs: Any
                                       ) -> Tuple[np.ndarray, np.ndarray]:
-        train, val = train_test_split(indices, test_size=val_share, shuffle=True, stratify=kwargs["stratify"])
+        train, val = train_test_split(indices, test_size=val_share, shuffle=True, stratify=kwargs["stratify"],
+                                      random_state=random_state)
         return train, val
 
     @classmethod
@@ -128,34 +137,38 @@ def get_holdout_validators(cls, *holdout_val_types: HoldoutValTypes) -> Dict[str
 
 class CrossValFuncs():
     @staticmethod
-    def shuffle_split_cross_validation(num_splits: int,
+    def shuffle_split_cross_validation(random_state: np.random.RandomState,
+                                       num_splits: int,
                                        indices: np.ndarray,
                                        **kwargs: Any
                                        ) -> List[Tuple[np.ndarray, np.ndarray]]:
-        cv = ShuffleSplit(n_splits=num_splits)
+        cv = ShuffleSplit(n_splits=num_splits, random_state=random_state)
         splits = list(cv.split(indices))
         return splits
 
     @staticmethod
-    def stratified_shuffle_split_cross_validation(num_splits: int,
+    def stratified_shuffle_split_cross_validation(random_state: np.random.RandomState,
+                                                  num_splits: int,
                                                   indices: np.ndarray,
                                                   **kwargs: Any
                                                   ) -> List[Tuple[np.ndarray, np.ndarray]]:
-        cv = StratifiedShuffleSplit(n_splits=num_splits)
+        cv = StratifiedShuffleSplit(n_splits=num_splits, random_state=random_state)
         splits = list(cv.split(indices, kwargs["stratify"]))
         return splits
 
     @staticmethod
-    def stratified_k_fold_cross_validation(num_splits: int,
+    def stratified_k_fold_cross_validation(random_state: np.random.RandomState,
+                                           num_splits: int,
                                            indices: np.ndarray,
                                            **kwargs: Any
                                            ) -> List[Tuple[np.ndarray, np.ndarray]]:
-        cv = StratifiedKFold(n_splits=num_splits)
+        cv = StratifiedKFold(n_splits=num_splits, random_state=random_state)
         splits = list(cv.split(indices, kwargs["stratify"]))
         return splits
 
     @staticmethod
-    def k_fold_cross_validation(num_splits: int,
+    def k_fold_cross_validation(random_state: np.random.RandomState,
+                                num_splits: int,
                                 indices: np.ndarray,
                                 **kwargs: Any
                                 ) -> List[Tuple[np.ndarray, np.ndarray]]:
@@ -169,12 +182,14 @@ def k_fold_cross_validation(num_splits: int,
         Returns:
             splits (List[Tuple[List, List]]): list of tuples of training and validation indices
         """
-        cv = KFold(n_splits=num_splits)
+        shuffle = kwargs.get('shuffle', True)
+        cv = KFold(n_splits=num_splits, random_state=random_state if shuffle else None, shuffle=shuffle)
         splits = list(cv.split(indices))
         return splits
 
     @staticmethod
-    def time_series_cross_validation(num_splits: int,
+    def time_series_cross_validation(random_state: np.random.RandomState,
+                                     num_splits: int,
                                      indices: np.ndarray,
                                      **kwargs: Any
                                      ) -> List[Tuple[np.ndarray, np.ndarray]]:
@@ -196,7 +211,7 @@ def time_series_cross_validation(num_splits: int,
                  ([0, 1, 2], [3])]
 
         """
-        cv = TimeSeriesSplit(n_splits=num_splits)
+        cv = TimeSeriesSplit(n_splits=num_splits, random_state=random_state)
         splits = list(cv.split(indices))
         return splits