[Fix] unit test

Author: franchuterivera

.coveragerc

@@ -0,0 +1,26 @@
+# .coveragerc to control coverage.py
+[run]
+branch = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+[html]
+directory = coverage_html_report

.github/workflows/pytest.yml

@@ -32,7 +32,7 @@ jobs:
         echo "::set-output name=BEFORE::$(git status --porcelain -b)"
     - name: Run tests
       run: |
-        if [ ${{ matrix.code-cov }} ]; then codecov='--cov=autoPyTorch --cov-report=xml'; fi
+        if [ ${{ matrix.code-cov }} ]; then codecov='--cov=autoPyTorch --cov-report=xml --cov-config=.coveragerc'; fi
         python -m pytest --forked --durations=20 --timeout=600 --timeout-method=signal -v $codecov test
     - name: Check for files left behind by test
       if: ${{ always() }}

autoPyTorch/datasets/tabular_dataset.py

@@ -24,18 +24,6 @@
 )
 
 
-class Value2Index(object):
-    def __init__(self, values: list):
-        assert all(not (pd.isna(v)) for v in values)
-        self.values = {v: i for i, v in enumerate(values)}
-
-    def __getitem__(self, item: Any) -> int:
-        if pd.isna(item):
-            return 0
-        else:
-            return self.values[item] + 1
-
-
 class TabularDataset(BaseDataset):
     """
         Base class for datasets used in AutoPyTorch

autoPyTorch/search_space/__init__.py

@@ -1,3 +1,4 @@
+import logging.handlers
 import os
 import re
 import shutil
@@ -299,6 +300,7 @@ def get_fit_dictionary(X, y, validator, backend):
         'metrics_during_training': True,
         'split_id': 0,
         'backend': backend,
+        'logger_port': logging.handlers.DEFAULT_TCP_LOGGING_PORT,
     }
     backend.save_datamanager(datamanager)
     return fit_dictionary

autoPyTorch/search_space/search_space.py

@@ -5,6 +5,8 @@
 import unittest
 from test.test_api.utils import dummy_do_dummy_prediction, dummy_eval_function
 
+import ConfigSpace as CS
+
 import numpy as np
 
 import pandas as pd
@@ -97,15 +99,15 @@ def test_tabular_classification(openml_id, resampling_strategy, backend, resampl
     assert len(loaded_datamanager.train_tensors) == len(estimator.dataset.train_tensors)
 
     expected_files = [
-        'smac3-output/run_1/configspace.json',
-        'smac3-output/run_1/runhistory.json',
-        'smac3-output/run_1/scenario.txt',
-        'smac3-output/run_1/stats.json',
-        'smac3-output/run_1/train_insts.txt',
-        'smac3-output/run_1/trajectory.json',
+        'smac3-output/run_42/configspace.json',
+        'smac3-output/run_42/runhistory.json',
+        'smac3-output/run_42/scenario.txt',
+        'smac3-output/run_42/stats.json',
+        'smac3-output/run_42/train_insts.txt',
+        'smac3-output/run_42/trajectory.json',
         '.autoPyTorch/datamanager.pkl',
         '.autoPyTorch/ensemble_read_preds.pkl',
-        '.autoPyTorch/start_time_1',
+        '.autoPyTorch/start_time_42',
         '.autoPyTorch/ensemble_history.json',
         '.autoPyTorch/ensemble_read_losses.pkl',
         '.autoPyTorch/true_targets_ensemble.npy',
@@ -212,6 +214,9 @@ def test_tabular_classification(openml_id, resampling_strategy, backend, resampl
     # Test refit on dummy data
     estimator.refit(dataset=backend.load_datamanager())
 
+    # Make sure that a configuration space is stored in the estimator
+    assert isinstance(estimator.get_search_space(), CS.ConfigurationSpace)
+
 
 @pytest.mark.parametrize('openml_name', ("boston", ))
 @unittest.mock.patch('autoPyTorch.evaluation.train_evaluator.eval_function',
@@ -284,15 +289,15 @@ def test_tabular_regression(openml_name, resampling_strategy, backend, resamplin
     assert len(loaded_datamanager.train_tensors) == len(estimator.dataset.train_tensors)
 
     expected_files = [
-        'smac3-output/run_1/configspace.json',
-        'smac3-output/run_1/runhistory.json',
-        'smac3-output/run_1/scenario.txt',
-        'smac3-output/run_1/stats.json',
-        'smac3-output/run_1/train_insts.txt',
-        'smac3-output/run_1/trajectory.json',
+        'smac3-output/run_42/configspace.json',
+        'smac3-output/run_42/runhistory.json',
+        'smac3-output/run_42/scenario.txt',
+        'smac3-output/run_42/stats.json',
+        'smac3-output/run_42/train_insts.txt',
+        'smac3-output/run_42/trajectory.json',
         '.autoPyTorch/datamanager.pkl',
         '.autoPyTorch/ensemble_read_preds.pkl',
-        '.autoPyTorch/start_time_1',
+        '.autoPyTorch/start_time_42',
         '.autoPyTorch/ensemble_history.json',
         '.autoPyTorch/ensemble_read_losses.pkl',
         '.autoPyTorch/true_targets_ensemble.npy',

test/conftest.py

@@ -1,10 +1,15 @@
+import logging
+import re
 import unittest
+from unittest.mock import MagicMock
 
 import numpy as np
 
 import pytest
 
-from autoPyTorch.api.base_task import BaseTask
+from autoPyTorch.api.base_task import BaseTask, _pipeline_predict
+from autoPyTorch.constants import TABULAR_CLASSIFICATION, TABULAR_REGRESSION
+from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline
 
 
 # ====
@@ -29,6 +34,10 @@ def test_nonsupported_arguments(fit_dictionary_tabular):
     api.ensemble_ = unittest.mock.MagicMock()
     with pytest.raises(ValueError, match=r".*No metric found. Either fit/search has not been.*"):
         api.score(np.ones(10), np.ones(10))
+    api.task_type = None
+    api._metric = MagicMock()
+    with pytest.raises(ValueError, match=r".*AutoPytorch failed to infer a task type*"):
+        api.score(np.ones(10), np.ones(10))
     api._metric = unittest.mock.MagicMock()
     with pytest.raises(ValueError, match=r".*No valid model found in run history.*"):
         api._load_models()
@@ -44,3 +53,37 @@ def returnfalse():
         api.predict(np.ones((10, 10)))
     with pytest.raises(ValueError, match=r".*No ensemble found. Either fit has not yet.*"):
         api.predict(np.ones((10, 10)))
+
+
+def test_pipeline_predict_function():
+    X = np.ones((10, 10))
+    pipeline = MagicMock()
+    pipeline.predict.return_value = np.full((10,), 3)
+    pipeline.predict_proba.return_value = np.full((10, 2), 3)
+
+    # First handle the classification case
+    task = TABULAR_CLASSIFICATION
+    with pytest.raises(ValueError, match='prediction probability not within'):
+        _pipeline_predict(pipeline, X, 5, logging.getLogger, task)
+    pipeline.predict_proba.return_value = np.zeros((10, 2))
+    predictions = _pipeline_predict(pipeline, X, 5, logging.getLogger(), task)
+    assert np.shape(predictions) == (10, 2)
+
+    task = TABULAR_REGRESSION
+    predictions = _pipeline_predict(pipeline, X, 5, logging.getLogger(), task)
+    assert np.shape(predictions) == (10,)
+    # Trigger warning msg with different shape for prediction
+    pipeline.predict.return_value = np.full((12,), 3)
+    predictions = _pipeline_predict(pipeline, X, 5, logging.getLogger(), task)
+
+
+@pytest.mark.parametrize("fit_dictionary_tabular", ['classification_categorical_only'], indirect=True)
+def test_show_models(fit_dictionary_tabular):
+    api = BaseTask()
+    api.ensemble_ = MagicMock()
+    api.models_ = [TabularClassificationPipeline(dataset_properties=fit_dictionary_tabular['dataset_properties'])]
+    api.ensemble_.get_models_with_weights.return_value = [(1.0, api.models_[0])]
+    # Expect the default configuration
+    expected = (r"0\s+|\s+SimpleImputer,OneHotEncoder,NoScaler,NoFeaturePreprocessing\s+"
+                r"|\s+no embedding,ShapedMLPBackbone,FullyConnectedHead,nn.Sequential\s+|\s+1")
+    assert re.search(expected, api.show_models()) is not None

test/test_api/test_api.py

@@ -10,12 +10,15 @@
 
 import pynisher
 
+import pytest
+
 from smac.runhistory.runhistory import RunInfo
 from smac.stats.stats import Stats
 from smac.tae import StatusType
+from smac.utils.constants import MAXINT
 
 from autoPyTorch.evaluation.tae import ExecuteTaFuncWithQueue, get_cost_of_crash
-from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy
+from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy, log_loss
 
 this_directory = os.path.dirname(__file__)
 sys.path.append(this_directory)
@@ -389,3 +392,8 @@ def test_silent_exception_in_target_function(self):
         self.assertNotIn('exitcode', info[1].additional_info)
         self.assertNotIn('exit_status', info[1].additional_info)
         self.assertNotIn('traceback', info[1])
+
+
+@pytest.mark.parametrize("metric,expected", [(accuracy, 1.0), (log_loss, MAXINT)])
+def test_get_cost_of_crash(metric, expected):
+    assert get_cost_of_crash(metric) == expected