Additional metrics during train (#194)

* Added additional metrics to fit dictionary

* Added in test also

Author: ravinkohli

autoPyTorch/evaluation/abstract_evaluator.py

@@ -378,6 +378,16 @@ def __init__(self, backend: Backend,
             'backend': self.backend,
             'logger_port': logger_port,
         })
+
+        # Update fit dictionary with metrics passed to the evaluator
+        metrics_dict: Dict[str, List[str]] = {'additional_metrics': []}
+        metrics_dict['additional_metrics'].append(self.metric.name)
+        if all_supported_metrics:
+            assert self.additional_metrics is not None
+            for metric in self.additional_metrics:
+                metrics_dict['additional_metrics'].append(metric.name)
+        self.fit_dictionary.update(metrics_dict)
+
         assert self.pipeline_class is not None, "Could not infer pipeline class"
         pipeline_config = pipeline_config if pipeline_config is not None \
             else self.pipeline_class.get_default_pipeline_options()

test/test_api/test_api.py

@@ -503,7 +503,8 @@ def test_pipeline_fit(openml_id,
                                                                     run_time_limit_secs=50,
                                                                     budget_type='epochs',
                                                                     budget=30,
-                                                                    disable_file_output=disable_file_output
+                                                                    disable_file_output=disable_file_output,
+                                                                    eval_metric='balanced_accuracy'
                                                                     )
     assert isinstance(dataset, BaseDataset)
     assert isinstance(run_info, RunInfo)
@@ -513,6 +514,7 @@ def test_pipeline_fit(openml_id,
     assert 'SUCCESS' in str(run_value.status)
 
     if not disable_file_output:
+
         if resampling_strategy in CrossValTypes:
             pytest.skip("Bug, Can't predict with cross validation pipeline")
             assert isinstance(pipeline, BaseEstimator)
@@ -524,11 +526,14 @@ def test_pipeline_fit(openml_id,
             assert isinstance(score, float)
             assert score > 0.8
         else:
-            assert isinstance(pipeline, BasePipeline)
             # To make sure we fitted the model, there should be a
-            # run summary object with accuracy
+            # run summary object
             run_summary = pipeline.named_steps['trainer'].run_summary
             assert run_summary is not None
+            # test to ensure balanced_accuracy is reported during training
+            assert 'balanced_accuracy' in run_summary.performance_tracker['train_metrics'][1].keys()
+
+            assert isinstance(pipeline, BasePipeline)
             X_test = dataset.test_tensors[0]
             preds = pipeline.predict(X_test)
             assert isinstance(preds, np.ndarray)