diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/automl/image_vertical/image_model_settings.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/automl/image_vertical/image_model_settings.py index aa27e6467f04..7c88e6286ad0 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/automl/image_vertical/image_model_settings.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/automl/image_vertical/image_model_settings.py @@ -86,6 +86,8 @@ class ImageModelSettingsObjectDetectionSchema(ImageModelSettingsSchema): allowed_values=[o.value for o in ValidationMetricType], casing_transform=camel_to_snake, ) + log_training_metrics = fields.Str() + log_validation_loss = fields.Str() @post_load def make(self, data, **kwargs): diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/automl_image_object_detection_base.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/automl_image_object_detection_base.py index 082d3517160b..7ec9b122090a 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/automl_image_object_detection_base.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/automl_image_object_detection_base.py @@ -11,6 +11,8 @@ ModelSize, StochasticOptimizer, ValidationMetricType, + LogTrainingMetrics, + LogValidationLoss, ) from azure.ai.ml._utils.utils import camel_to_snake from azure.ai.ml.entities._job.automl import SearchSpace @@ -56,6 +58,8 @@ def training_parameters(self, value: Union[Dict, ImageModelSettingsObjectDetecti learning_rate_scheduler=value.learning_rate_scheduler, model_size=value.model_size, validation_metric_type=value.validation_metric_type, + log_training_metrics=value.log_training_metrics, + log_validation_loss=value.log_validation_loss, ) elif value is None: self._training_parameters = value @@ -148,6 +152,8 @@ def set_training_parameters( tile_predictions_nms_threshold: Optional[float] = None, validation_iou_threshold: Optional[float] = None, validation_metric_type: Optional[Union[str, ValidationMetricType]] = None, + log_training_metrics: Optional[Union[str, LogTrainingMetrics]] = None, + log_validation_loss: Optional[Union[str, LogValidationLoss]] = None, ) -> None: """Setting Image training parameters for for AutoML Image Object Detection and Image Instance Segmentation tasks. @@ -288,6 +294,12 @@ def set_training_parameters( :keyword validation_metric_type: Metric computation method to use for validation metrics. Must be 'none', 'coco', 'voc', or 'coco_voc'. :type validation_metric_type: str or ~azure.mgmt.machinelearningservices.models.ValidationMetricType + :keyword log_training_metrics: indicates whether or not to log training metrics. Must + be 'Enable' or 'Disable' + :type log_training_metrics: str or ~azure.mgmt.machinelearningservices.models.LogTrainingMetrics + :keyword log_validation_loss: indicates whether or not to log validation loss. Must + be 'Enable' or 'Disable' + :type log_validation_loss: str or ~azure.mgmt.machinelearningservices.models.LogValidationLoss """ self._training_parameters = self._training_parameters or ImageModelSettingsObjectDetection() @@ -432,6 +444,16 @@ def set_training_parameters( if validation_metric_type is not None else self._training_parameters.validation_metric_type ) + self._training_parameters.log_training_metrics = ( + LogTrainingMetrics[camel_to_snake(log_training_metrics)] + if log_training_metrics is not None + else self._training_parameters.log_training_metrics + ) + self._training_parameters.log_validation_loss = ( + LogValidationLoss[camel_to_snake(log_validation_loss)] + if log_validation_loss is not None + else self._training_parameters.log_validation_loss + ) # pylint: enable=too-many-locals diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/image_model_settings.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/image_model_settings.py index d4633258dcb6..595885df8f70 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/image_model_settings.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/image_model_settings.py @@ -16,6 +16,8 @@ ModelSize, StochasticOptimizer, ValidationMetricType, + LogTrainingMetrics, + LogValidationLoss, ) from azure.ai.ml.entities._mixins import RestTranslatableMixin @@ -624,6 +626,12 @@ class ImageModelSettingsObjectDetection(ImageModelDistributionSettings): values include: "None", "Coco", "Voc", "CocoVoc". :type validation_metric_type: str or ~azure.mgmt.machinelearningservices.models.ValidationMetricType + :param log_training_metrics: indicates whether or not to log training metrics + :type log_training_metrics: str or + ~azure.mgmt.machinelearningservices.models.LogTrainingMetrics + :param log_validation_loss: indicates whether or not to log validation loss + :type log_validation_loss: str or + ~azure.mgmt.machinelearningservices.models.LogValidationLoss """ def __init__( @@ -672,6 +680,8 @@ def __init__( tile_predictions_nms_threshold: Optional[float] = None, validation_iou_threshold: Optional[float] = None, validation_metric_type: Optional[ValidationMetricType] = None, + log_training_metrics: Optional[LogTrainingMetrics] = None, + log_validation_loss: Optional[LogValidationLoss] = None, **kwargs, ): super(ImageModelSettingsObjectDetection, self).__init__( @@ -720,11 +730,11 @@ def __init__( self.tile_predictions_nms_threshold = tile_predictions_nms_threshold self.validation_iou_threshold = validation_iou_threshold self.validation_metric_type = validation_metric_type + self.log_training_metrics = log_training_metrics + self.log_validation_loss = log_validation_loss def _to_rest_object(self) -> RestImageModelSettingsObjectDetection: return RestImageModelSettingsObjectDetection( - # Temporary fix for https://msdata.visualstudio.com/Vienna/_workitems/edit/2385143 - log_training_metrics="Disable", advanced_settings=self.advanced_settings, ams_gradient=self.ams_gradient, beta1=self.beta1, @@ -768,6 +778,8 @@ def _to_rest_object(self) -> RestImageModelSettingsObjectDetection: tile_predictions_nms_threshold=self.tile_predictions_nms_threshold, validation_iou_threshold=self.validation_iou_threshold, validation_metric_type=self.validation_metric_type, + log_training_metrics=self.log_training_metrics, + log_validation_loss=self.log_validation_loss, ) @classmethod @@ -816,6 +828,8 @@ def _from_rest_object(cls, obj: RestImageModelSettingsObjectDetection) -> "Image tile_predictions_nms_threshold=obj.tile_predictions_nms_threshold, validation_iou_threshold=obj.validation_iou_threshold, validation_metric_type=obj.validation_metric_type, + log_training_metrics=obj.log_training_metrics, + log_validation_loss=obj.log_validation_loss, ) def __eq__(self, other: object) -> bool: @@ -837,6 +851,8 @@ def __eq__(self, other: object) -> bool: and self.tile_predictions_nms_threshold == other.tile_predictions_nms_threshold and self.validation_iou_threshold == other.validation_iou_threshold and self.validation_metric_type == other.validation_metric_type + and self.log_training_metrics == other.log_training_metrics + and self.log_validation_loss == other.log_validation_loss ) def __ne__(self, other: object) -> bool: diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py index bbeb42d2f797..f1a79f667b78 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py @@ -12,6 +12,8 @@ ModelSize, SamplingAlgorithmType, StochasticOptimizer, + LogTrainingMetrics, + LogValidationLoss, ) from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType @@ -128,24 +130,38 @@ def _check_data_type(data, expected_type, expected_path, msg): "settings, expected", [ ( - ("adam", "warmup_cosine", "coco_voc", "large"), + ("adam", "warmup_cosine", "coco_voc", "large", "enable", "enable"), ( StochasticOptimizer.ADAM, LearningRateScheduler.WARMUP_COSINE, ValidationMetricType.COCO_VOC, ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, ), ), ( - ("Adam", "WarmupCosine", "CocoVoc", "Large"), + ("Adam", "WarmupCosine", "CocoVoc", "Large", "Enable", "Enable"), ( StochasticOptimizer.ADAM, LearningRateScheduler.WARMUP_COSINE, ValidationMetricType.COCO_VOC, ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, + ), + ), + ( + (None, None, "coco_voc", "large", "enable", "enable"), + ( + None, + None, + ValidationMetricType.COCO_VOC, + ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, ), ), - ((None, None, "coco_voc", "large"), (None, None, ValidationMetricType.COCO_VOC, ModelSize.LARGE)), ], ids=["snake case", "camel case", "with None"], ) @@ -159,25 +175,33 @@ def test_image_set_training_parameters_with_valid_values(self, settings, expecte learning_rate_scheduler=settings[1], validation_metric_type=settings[2], model_size=settings[3], + log_training_metrics=settings[4], + log_validation_loss=settings[5], ) assert image_instance_segmentation_job.training_parameters.optimizer == expected[0] assert image_instance_segmentation_job.training_parameters.learning_rate_scheduler == expected[1] assert image_instance_segmentation_job.training_parameters.validation_metric_type == expected[2] assert image_instance_segmentation_job.training_parameters.model_size == expected[3] + assert image_instance_segmentation_job.training_parameters.log_training_metrics == expected[4] + assert image_instance_segmentation_job.training_parameters.log_validation_loss == expected[5] @pytest.mark.parametrize( "settings, expected", [ - (("adamW", None, None, None), pytest.raises(KeyError)), - ((None, "Warmup_Cosine", None, None), pytest.raises(KeyError)), - ((None, None, "Coco_Voc", "large"), pytest.raises(KeyError)), - ((None, None, None, "Extra_Large"), pytest.raises(KeyError)), + (("adamW", None, None, None, "Enable", "Enable"), pytest.raises(KeyError)), + ((None, "Warmup_Cosine", None, None, "Enable", "Enable"), pytest.raises(KeyError)), + ((None, None, "Coco_Voc", "large", "Enable", "Enable"), pytest.raises(KeyError)), + ((None, None, None, "Extra_Large", "Enable", "Enable"), pytest.raises(KeyError)), + ((None, None, None, None, "false", "Enable"), pytest.raises(KeyError)), + ((None, None, None, None, "Enable", "false"), pytest.raises(KeyError)), ], ids=[ "optimizer invalid", "learning rate scheduler invalid", "validation metric type invalid", "model size invalid", + "log_training_metrics invalid", + "log_validation_loss invalid", ], ) def test_image_set_training_parameters_with_invalid_values(self, settings, expected): @@ -191,30 +215,46 @@ def test_image_set_training_parameters_with_invalid_values(self, settings, expec learning_rate_scheduler=settings[1], validation_metric_type=settings[2], model_size=settings[3], + log_training_metrics=settings[4], + log_validation_loss=settings[5], ) @pytest.mark.parametrize( "settings, expected", [ ( - ("adam", "warmup_cosine", "coco_voc", "large"), + ("adam", "warmup_cosine", "coco_voc", "large", "enable", "enable"), ( StochasticOptimizer.ADAM, LearningRateScheduler.WARMUP_COSINE, ValidationMetricType.COCO_VOC, ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, ), ), ( - ("Adam", "WarmupCosine", "CocoVoc", "Large"), + ("Adam", "WarmupCosine", "CocoVoc", "Large", "Enable", "Enable"), ( StochasticOptimizer.ADAM, LearningRateScheduler.WARMUP_COSINE, ValidationMetricType.COCO_VOC, ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, + ), + ), + ( + (None, None, "coco_voc", "large", "enable", "enable"), + ( + None, + None, + ValidationMetricType.COCO_VOC, + ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, ), ), - ((None, None, "coco_voc", "large"), (None, None, ValidationMetricType.COCO_VOC, ModelSize.LARGE)), ], ids=["snake case", "camel case", "with None"], ) @@ -224,13 +264,18 @@ def test_image_set_training_parameters_with_settings_object(self, settings, expe learning_rate_scheduler=settings[1], validation_metric_type=settings[2], model_size=settings[3], + log_training_metrics=settings[4], + log_validation_loss=settings[5], ) image_instance_segmentation_job = image_instance_segmentation( training_data=Input(type=AssetTypes.MLTABLE, path="https://foo/bar/train.csv"), target_column_name="label", training_parameters=image_model_settings, ) + assert image_instance_segmentation_job.training_parameters.optimizer == expected[0] assert image_instance_segmentation_job.training_parameters.learning_rate_scheduler == expected[1] assert image_instance_segmentation_job.training_parameters.validation_metric_type == expected[2] assert image_instance_segmentation_job.training_parameters.model_size == expected[3] + assert image_instance_segmentation_job.training_parameters.log_training_metrics == expected[4] + assert image_instance_segmentation_job.training_parameters.log_validation_loss == expected[5] diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py index 0c27fda2d169..87e13be22453 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_object_detection.py @@ -12,6 +12,8 @@ ObjectDetectionPrimaryMetrics, SamplingAlgorithmType, StochasticOptimizer, + LogTrainingMetrics, + LogValidationLoss, ) from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType @@ -136,24 +138,38 @@ def _check_data_type(data, expected_type, expected_path, msg): "settings, expected", [ ( - ("adam", "warmup_cosine", "coco_voc", "large"), + ("adam", "warmup_cosine", "coco_voc", "large", "enable", "enable"), ( StochasticOptimizer.ADAM, LearningRateScheduler.WARMUP_COSINE, ValidationMetricType.COCO_VOC, ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, ), ), ( - ("Adam", "WarmupCosine", "CocoVoc", "Large"), + ("Adam", "WarmupCosine", "CocoVoc", "Large", "Enable", "Enable"), ( StochasticOptimizer.ADAM, LearningRateScheduler.WARMUP_COSINE, ValidationMetricType.COCO_VOC, ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, + ), + ), + ( + (None, None, "coco_voc", "large", "enable", "enable"), + ( + None, + None, + ValidationMetricType.COCO_VOC, + ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, ), ), - ((None, None, "coco_voc", "large"), (None, None, ValidationMetricType.COCO_VOC, ModelSize.LARGE)), ], ids=["snake case", "camel case", "with None"], ) @@ -167,25 +183,33 @@ def test_image_set_training_parameters_with_valid_values(self, settings, expecte learning_rate_scheduler=settings[1], validation_metric_type=settings[2], model_size=settings[3], + log_training_metrics=settings[4], + log_validation_loss=settings[5], ) assert image_object_detection_job.training_parameters.optimizer == expected[0] assert image_object_detection_job.training_parameters.learning_rate_scheduler == expected[1] assert image_object_detection_job.training_parameters.validation_metric_type == expected[2] assert image_object_detection_job.training_parameters.model_size == expected[3] + assert image_object_detection_job.training_parameters.log_training_metrics == expected[4] + assert image_object_detection_job.training_parameters.log_validation_loss == expected[5] @pytest.mark.parametrize( "settings, expected", [ - (("adamW", None, None, None), pytest.raises(KeyError)), - ((None, "Warmup_Cosine", None, None), pytest.raises(KeyError)), - ((None, None, "Coco_Voc", "large"), pytest.raises(KeyError)), - ((None, None, None, "Extra_Large"), pytest.raises(KeyError)), + (("adamW", None, None, None, "Enable", "Enable"), pytest.raises(KeyError)), + ((None, "Warmup_Cosine", None, None, "Enable", "Enable"), pytest.raises(KeyError)), + ((None, None, "Coco_Voc", "large", "Enable", "Enable"), pytest.raises(KeyError)), + ((None, None, None, "Extra_Large", "Enable", "Enable"), pytest.raises(KeyError)), + ((None, None, None, None, "false", "Enable"), pytest.raises(KeyError)), + ((None, None, None, None, "Enable", "false"), pytest.raises(KeyError)), ], ids=[ "optimizer invalid", "learning rate scheduler invalid", "validation metric type invalid", "model size invalid", + "log_training_metrics invalid", + "log_validation_loss invalid", ], ) def test_image_set_training_parameters_with_invalid_values(self, settings, expected): @@ -199,30 +223,46 @@ def test_image_set_training_parameters_with_invalid_values(self, settings, expec learning_rate_scheduler=settings[1], validation_metric_type=settings[2], model_size=settings[3], + log_training_metrics=settings[4], + log_validation_loss=settings[5], ) @pytest.mark.parametrize( "settings, expected", [ ( - ("adam", "warmup_cosine", "coco_voc", "large"), + ("adam", "warmup_cosine", "coco_voc", "large", "enable", "enable"), ( StochasticOptimizer.ADAM, LearningRateScheduler.WARMUP_COSINE, ValidationMetricType.COCO_VOC, ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, ), ), ( - ("Adam", "WarmupCosine", "CocoVoc", "Large"), + ("Adam", "WarmupCosine", "CocoVoc", "Large", "Enable", "Enable"), ( StochasticOptimizer.ADAM, LearningRateScheduler.WARMUP_COSINE, ValidationMetricType.COCO_VOC, ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, + ), + ), + ( + (None, None, "coco_voc", "large", "enable", "enable"), + ( + None, + None, + ValidationMetricType.COCO_VOC, + ModelSize.LARGE, + LogTrainingMetrics.ENABLE, + LogValidationLoss.ENABLE, ), ), - ((None, None, "coco_voc", "large"), (None, None, ValidationMetricType.COCO_VOC, ModelSize.LARGE)), ], ids=["snake case", "camel case", "with None"], ) @@ -232,6 +272,8 @@ def test_image_set_training_parameters_with_settings_object(self, settings, expe learning_rate_scheduler=settings[1], validation_metric_type=settings[2], model_size=settings[3], + log_training_metrics=settings[4], + log_validation_loss=settings[5], ) image_object_detection_job = image_object_detection( training_data=Input(type=AssetTypes.MLTABLE, path="https://foo/bar/train.csv"), @@ -242,3 +284,5 @@ def test_image_set_training_parameters_with_settings_object(self, settings, expe assert image_object_detection_job.training_parameters.learning_rate_scheduler == expected[1] assert image_object_detection_job.training_parameters.validation_metric_type == expected[2] assert image_object_detection_job.training_parameters.model_size == expected[3] + assert image_object_detection_job.training_parameters.log_training_metrics == expected[4] + assert image_object_detection_job.training_parameters.log_validation_loss == expected[5] diff --git a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py index 66960cabb73b..bc0bc554fda8 100644 --- a/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py +++ b/sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_schema.py @@ -148,7 +148,6 @@ def expected_image_model_settings_classification() -> RestImageModelSettingsClas @pytest.fixture def expected_image_model_settings_object_detection() -> RestImageModelSettingsObjectDetection: return RestImageModelSettingsObjectDetection( - log_training_metrics="Disable", checkpoint_frequency=1, early_stopping=True, early_stopping_delay=2,