Additional metrics during train (#194)

* Added additional metrics to fit dictionary

* Added in test also

Author: ravinkohli

test/test_api/test_api.py

@@ -522,6 +522,7 @@ def test_portfolio_selection(openml_id, backend, n_samples):
         resampling_strategy=HoldoutValTypes.holdout_validation,
     )
 
+<<<<<<< HEAD
     with unittest.mock.patch.object(estimator, '_do_dummy_prediction', new=dummy_do_dummy_prediction):
         estimator.search(
             X_train=X_train, y_train=y_train,
@@ -533,6 +534,62 @@ def test_portfolio_selection(openml_id, backend, n_samples):
             portfolio_selection=os.path.join(os.path.dirname(__file__),
                                              "../../autoPyTorch/configs/greedy_portfolio.json")
         )
+=======
+    dataset = estimator.get_dataset(X_train=X_train,
+                                    y_train=y_train,
+                                    X_test=X_test,
+                                    y_test=y_test,
+                                    resampling_strategy=resampling_strategy,
+                                    resampling_strategy_args=resampling_strategy_args)
+
+    configuration = estimator.get_search_space(dataset).get_default_configuration()
+    pipeline, run_info, run_value, dataset = estimator.fit_pipeline(dataset=dataset,
+                                                                    configuration=configuration,
+                                                                    run_time_limit_secs=50,
+                                                                    budget_type='epochs',
+                                                                    budget=30,
+                                                                    disable_file_output=disable_file_output,
+                                                                    eval_metric='balanced_accuracy'
+                                                                    )
+    assert isinstance(dataset, BaseDataset)
+    assert isinstance(run_info, RunInfo)
+    assert isinstance(run_info.config, Configuration)
+
+    assert isinstance(run_value, RunValue)
+    assert 'SUCCESS' in str(run_value.status)
+
+    if not disable_file_output:
+
+        if resampling_strategy in CrossValTypes:
+            pytest.skip("Bug, Can't predict with cross validation pipeline")
+            assert isinstance(pipeline, BaseEstimator)
+            X_test = dataset.test_tensors[0]
+            preds = pipeline.predict(X_test)
+            assert isinstance(preds, np.ndarray)
+
+            score = accuracy(dataset.test_tensors[1], preds)
+            assert isinstance(score, float)
+            assert score > 0.8
+        else:
+            # To make sure we fitted the model, there should be a
+            # run summary object
+            run_summary = pipeline.named_steps['trainer'].run_summary
+            assert run_summary is not None
+            # test to ensure balanced_accuracy is reported during training
+            assert 'balanced_accuracy' in run_summary.performance_tracker['train_metrics'][1].keys()
+
+            assert isinstance(pipeline, BasePipeline)
+            X_test = dataset.test_tensors[0]
+            preds = pipeline.predict(X_test)
+            assert isinstance(preds, np.ndarray)
+
+            score = accuracy(dataset.test_tensors[1], preds)
+            assert isinstance(score, float)
+            assert score > 0.8
+    else:
+        assert pipeline is None
+        assert run_value.cost < 0.2
+>>>>>>> Additional metrics during train (#194)
 
     successful_config_ids = [run_key.config_id for run_key, run_value in estimator.run_history.data.items(
     ) if 'SUCCESS' in str(run_value.status)]