Fix: MLPRegressor tests (#1367)

* Fix: MLPRegressor tests

* Fix: Ordering of statements in test

* Fix: MLP n_calls

Author: eddiebergman

test/test_pipeline/components/regression/test_base.py

@@ -1,4 +1,4 @@
-from typing import Type
+from typing import Type, Container
 
 import unittest
 
@@ -41,31 +41,29 @@ def test_default_boston(self):
                     Regressor=self.module
                 )
 
+            score = sklearn.metrics.r2_score(y_true=targets, y_pred=predictions)
+
+            # Special treatment for Gaussian Process Regression
             if "default_boston_le_ge" in self.res:
-                # Special treatment for Gaussian Process Regression
-                self.assertLessEqual(
-                    sklearn.metrics.r2_score(y_true=targets, y_pred=predictions),
-                    self.res["default_boston_le_ge"][0]
-                )
-                self.assertGreaterEqual(
-                    sklearn.metrics.r2_score(y_true=targets, y_pred=predictions),
-                    self.res["default_boston_le_ge"][1]
-                )
+                upper, lower = self.res["default_boston_le_ge"]
+                assert lower <= score <= upper
+
             else:
-                score = sklearn.metrics.r2_score(targets, predictions)
                 fixture = self.res["default_boston"]
+                places = self.res.get("default_boston_places", 7)
+
                 if score < -1e10:
-                    print(f"score = {score}, fixture = {fixture}")
                     score = np.log(-score)
                     fixture = np.log(-fixture)
-                self.assertAlmostEqual(
-                    fixture,
-                    score,
-                    places=self.res.get("default_boston_places", 7),
-                )
 
-            if self.res.get("boston_n_calls"):
-                self.assertEqual(self.res["boston_n_calls"], n_calls)
+                self.assertAlmostEqual(fixture, score, places)
+
+            if "boston_n_calls" in self.res:
+                expected = self.res["boston_n_calls"]
+                if isinstance(expected, Container):
+                    assert n_calls in expected
+                else:
+                    assert n_calls == expected
 
     def test_default_boston_iterative_fit(self):
 
@@ -84,23 +82,28 @@ def test_default_boston_iterative_fit(self):
 
             score = sklearn.metrics.r2_score(targets, predictions)
             fixture = self.res["default_boston_iterative"]
+            places = self.res.get("default_boston_iterative_places", 7)
 
             if score < -1e10:
                 print(f"score = {score}, fixture = {fixture}")
                 score = np.log(-score)
                 fixture = np.log(-fixture)
 
-            self.assertAlmostEqual(
-                fixture,
-                score,
-                places=self.res.get("default_boston_iterative_places", 7),
-            )
+            self.assertAlmostEqual(fixture, score, places)
 
             if self.step_hyperparameter is not None:
-                self.assertEqual(
-                    getattr(regressor.estimator, self.step_hyperparameter['name']),
-                    self.res.get("boston_iterative_n_iter", self.step_hyperparameter['value'])
-                )
+                param_name = self.step_hyperparameter['name']
+                default = self.step_hyperparameter['value']
+
+                value = getattr(regressor.estimator, param_name)
+                expected = self.res.get("boston_iterative_n_iter", default)
+
+                # To currently allow for MLPRegressor which is indeterministic,
+                # we can have multiple values
+                if isinstance(expected, Container):
+                    assert value in expected
+                else:
+                    assert value == expected
 
     def test_default_boston_iterative_sparse_fit(self):
 

test/test_pipeline/components/regression/test_mlp.py

@@ -1,3 +1,5 @@
+from typing import Any, Dict
+
 import sklearn.neural_network
 
 from autosklearn.pipeline.components.regression.mlp import MLPRegressor
@@ -22,21 +24,36 @@ class MLPComponentTest(BaseRegressionComponentTest):
     #
     # These seem to have consistent CPU's so I'm unsure what the underlying reason
     # for this to randomly fail only sometimes on Github runners
+    #
+    # Edit: If changing, please tracke what values were failing
+    #
+    # Seems there is a consistently different values for boston so:
+    # * include two valuess for n_iter in 'boston_iterative_n_iter'
+    #   known-values = [236, 331]
+    #
+    # * decreased places from 6 -> 5 in 'default_boston_{sparse,_iterative_sparse}'
+    #   to check for for iterations and expanded the default places for checking
+    #   know-values = [-0.10972947168054104, -0.10973142976866268]
+    #
+    # * decreased places from 3 -> 1 in 'default_boston_places'
+    #   known-values = [0.29521793994422807, 0.2750079862455884]
+    #
+    # * Include two value for 'boston_n_calls'
+    #   known-values = [8, 9]
     __test__ = True
-
     __test__ = True
 
-    res = dict()
+    res: Dict[str, Any] = {}
     res["default_boston"] = 0.2750079862455884
-    res["default_boston_places"] = 3
-    res["boston_n_calls"] = 8
-    res["boston_iterative_n_iter"] = 236
+    res["default_boston_places"] = 1
+    res["boston_n_calls"] = [8, 9]
+    res["boston_iterative_n_iter"] = [236, 331]
     res["default_boston_iterative"] = res["default_boston"]
     res["default_boston_iterative_places"] = 1
     res["default_boston_sparse"] = -0.10972947168054104
-    res["default_boston_sparse_places"] = 6
+    res["default_boston_sparse_places"] = 5
     res["default_boston_iterative_sparse"] = res["default_boston_sparse"]
-    res["default_boston_iterative_sparse_places"] = 6
+    res["default_boston_iterative_sparse_places"] = res["default_boston_sparse_places"]
     res["default_diabetes"] = 0.35917389841850555
     res["diabetes_n_calls"] = 9
     res["diabetes_iterative_n_iter"] = 435