Skip to content

Commit

Permalink
Expose the parameters log_training_metrics and log_validation_loss fo…
Browse files Browse the repository at this point in the history 
…r object detection and instance segmentation (#31479)

* add log_training_metrics and log_validation_loss params into image_model_settings

* add log params to automl_image_object_detection_base, use enum types, update unit tests

* code reformatting using black

* add log params to docstrings

* remove hardcoded log_training_metrics from unit tests
  • Loading branch information
@bhimar
bhimar authored Aug 8, 2023
1 parent 91d92f4 commit ae2c36e
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 23 deletions.
2 changes: 2 additions & 0 deletions sdk/ml/azure-ai-ml/azure/ai/ml/_schema/automl/image_vertical/image_model_settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ class ImageModelSettingsObjectDetectionSchema(ImageModelSettingsSchema):
allowed_values=[o.value for o in ValidationMetricType],
casing_transform=camel_to_snake,
)
log_training_metrics = fields.Str()
log_validation_loss = fields.Str()

@post_load
def make(self, data, **kwargs):
Expand Down
22 changes: 22 additions & 0 deletions .../azure-ai-ml/azure/ai/ml/entities/_job/automl/image/automl_image_object_detection_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
ModelSize,
StochasticOptimizer,
ValidationMetricType,
LogTrainingMetrics,
LogValidationLoss,
)
from azure.ai.ml._utils.utils import camel_to_snake
from azure.ai.ml.entities._job.automl import SearchSpace
Expand Down Expand Up @@ -56,6 +58,8 @@ def training_parameters(self, value: Union[Dict, ImageModelSettingsObjectDetecti
learning_rate_scheduler=value.learning_rate_scheduler,
model_size=value.model_size,
validation_metric_type=value.validation_metric_type,
log_training_metrics=value.log_training_metrics,
log_validation_loss=value.log_validation_loss,
)
elif value is None:
self._training_parameters = value
Expand Down Expand Up @@ -148,6 +152,8 @@ def set_training_parameters(
tile_predictions_nms_threshold: Optional[float] = None,
validation_iou_threshold: Optional[float] = None,
validation_metric_type: Optional[Union[str, ValidationMetricType]] = None,
log_training_metrics: Optional[Union[str, LogTrainingMetrics]] = None,
log_validation_loss: Optional[Union[str, LogValidationLoss]] = None,
) -> None:
"""Setting Image training parameters for for AutoML Image Object Detection and Image Instance Segmentation
tasks.
Expand Down Expand Up @@ -288,6 +294,12 @@ def set_training_parameters(
:keyword validation_metric_type: Metric computation method to use for validation metrics. Must
be 'none', 'coco', 'voc', or 'coco_voc'.
:type validation_metric_type: str or ~azure.mgmt.machinelearningservices.models.ValidationMetricType
:keyword log_training_metrics: indicates whether or not to log training metrics. Must
be 'Enable' or 'Disable'
:type log_training_metrics: str or ~azure.mgmt.machinelearningservices.models.LogTrainingMetrics
:keyword log_validation_loss: indicates whether or not to log validation loss. Must
be 'Enable' or 'Disable'
:type log_validation_loss: str or ~azure.mgmt.machinelearningservices.models.LogValidationLoss
"""
self._training_parameters = self._training_parameters or ImageModelSettingsObjectDetection()

Expand Down Expand Up @@ -432,6 +444,16 @@ def set_training_parameters(
if validation_metric_type is not None
else self._training_parameters.validation_metric_type
)
self._training_parameters.log_training_metrics = (
LogTrainingMetrics[camel_to_snake(log_training_metrics)]
if log_training_metrics is not None
else self._training_parameters.log_training_metrics
)
self._training_parameters.log_validation_loss = (
LogValidationLoss[camel_to_snake(log_validation_loss)]
if log_validation_loss is not None
else self._training_parameters.log_validation_loss
)

# pylint: enable=too-many-locals

Expand Down
20 changes: 18 additions & 2 deletions sdk/ml/azure-ai-ml/azure/ai/ml/entities/_job/automl/image/image_model_settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
ModelSize,
StochasticOptimizer,
ValidationMetricType,
LogTrainingMetrics,
LogValidationLoss,
)
from azure.ai.ml.entities._mixins import RestTranslatableMixin

Expand Down Expand Up @@ -624,6 +626,12 @@ class ImageModelSettingsObjectDetection(ImageModelDistributionSettings):
values include: "None", "Coco", "Voc", "CocoVoc".
:type validation_metric_type: str or
~azure.mgmt.machinelearningservices.models.ValidationMetricType
:param log_training_metrics: indicates whether or not to log training metrics
:type log_training_metrics: str or
~azure.mgmt.machinelearningservices.models.LogTrainingMetrics
:param log_validation_loss: indicates whether or not to log validation loss
:type log_validation_loss: str or
~azure.mgmt.machinelearningservices.models.LogValidationLoss
"""

def __init__(
Expand Down Expand Up @@ -672,6 +680,8 @@ def __init__(
tile_predictions_nms_threshold: Optional[float] = None,
validation_iou_threshold: Optional[float] = None,
validation_metric_type: Optional[ValidationMetricType] = None,
log_training_metrics: Optional[LogTrainingMetrics] = None,
log_validation_loss: Optional[LogValidationLoss] = None,
**kwargs,
):
super(ImageModelSettingsObjectDetection, self).__init__(
Expand Down Expand Up @@ -720,11 +730,11 @@ def __init__(
self.tile_predictions_nms_threshold = tile_predictions_nms_threshold
self.validation_iou_threshold = validation_iou_threshold
self.validation_metric_type = validation_metric_type
self.log_training_metrics = log_training_metrics
self.log_validation_loss = log_validation_loss

def _to_rest_object(self) -> RestImageModelSettingsObjectDetection:
return RestImageModelSettingsObjectDetection(
# Temporary fix for https://msdata.visualstudio.com/Vienna/_workitems/edit/2385143
log_training_metrics="Disable",
advanced_settings=self.advanced_settings,
ams_gradient=self.ams_gradient,
beta1=self.beta1,
Expand Down Expand Up @@ -768,6 +778,8 @@ def _to_rest_object(self) -> RestImageModelSettingsObjectDetection:
tile_predictions_nms_threshold=self.tile_predictions_nms_threshold,
validation_iou_threshold=self.validation_iou_threshold,
validation_metric_type=self.validation_metric_type,
log_training_metrics=self.log_training_metrics,
log_validation_loss=self.log_validation_loss,
)

@classmethod
Expand Down Expand Up @@ -816,6 +828,8 @@ def _from_rest_object(cls, obj: RestImageModelSettingsObjectDetection) -> "Image
tile_predictions_nms_threshold=obj.tile_predictions_nms_threshold,
validation_iou_threshold=obj.validation_iou_threshold,
validation_metric_type=obj.validation_metric_type,
log_training_metrics=obj.log_training_metrics,
log_validation_loss=obj.log_validation_loss,
)

def __eq__(self, other: object) -> bool:
Expand All @@ -837,6 +851,8 @@ def __eq__(self, other: object) -> bool:
and self.tile_predictions_nms_threshold == other.tile_predictions_nms_threshold
and self.validation_iou_threshold == other.validation_iou_threshold
and self.validation_metric_type == other.validation_metric_type
and self.log_training_metrics == other.log_training_metrics
and self.log_validation_loss == other.log_validation_loss
)

def __ne__(self, other: object) -> bool:
Expand Down
65 changes: 55 additions & 10 deletions sdk/ml/azure-ai-ml/tests/automl_job/unittests/test_automl_image_instance_segmentation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
ModelSize,
SamplingAlgorithmType,
StochasticOptimizer,
LogTrainingMetrics,
LogValidationLoss,
)
from azure.ai.ml._restclient.v2023_04_01_preview.models import UserIdentity as RestUserIdentity
from azure.ai.ml._restclient.v2023_04_01_preview.models import ValidationMetricType
Expand Down Expand Up @@ -128,24 +130,38 @@ def _check_data_type(data, expected_type, expected_path, msg):
"settings, expected",
[
(
("adam", "warmup_cosine", "coco_voc", "large"),
("adam", "warmup_cosine", "coco_voc", "large", "enable", "enable"),
(
StochasticOptimizer.ADAM,
LearningRateScheduler.WARMUP_COSINE,
ValidationMetricType.COCO_VOC,
ModelSize.LARGE,
LogTrainingMetrics.ENABLE,
LogValidationLoss.ENABLE,
),
),
(
("Adam", "WarmupCosine", "CocoVoc", "Large"),
("Adam", "WarmupCosine", "CocoVoc", "Large", "Enable", "Enable"),
(
StochasticOptimizer.ADAM,
LearningRateScheduler.WARMUP_COSINE,
ValidationMetricType.COCO_VOC,
ModelSize.LARGE,
LogTrainingMetrics.ENABLE,
LogValidationLoss.ENABLE,
),
),
(
(None, None, "coco_voc", "large", "enable", "enable"),
(
None,
None,
ValidationMetricType.COCO_VOC,
ModelSize.LARGE,
LogTrainingMetrics.ENABLE,
LogValidationLoss.ENABLE,
),
),
((None, None, "coco_voc", "large"), (None, None, ValidationMetricType.COCO_VOC, ModelSize.LARGE)),
],
ids=["snake case", "camel case", "with None"],
)
Expand All @@ -159,25 +175,33 @@ def test_image_set_training_parameters_with_valid_values(self, settings, expecte
learning_rate_scheduler=settings[1],
validation_metric_type=settings[2],
model_size=settings[3],
log_training_metrics=settings[4],
log_validation_loss=settings[5],
)
assert image_instance_segmentation_job.training_parameters.optimizer == expected[0]
assert image_instance_segmentation_job.training_parameters.learning_rate_scheduler == expected[1]
assert image_instance_segmentation_job.training_parameters.validation_metric_type == expected[2]
assert image_instance_segmentation_job.training_parameters.model_size == expected[3]
assert image_instance_segmentation_job.training_parameters.log_training_metrics == expected[4]
assert image_instance_segmentation_job.training_parameters.log_validation_loss == expected[5]

@pytest.mark.parametrize(
"settings, expected",
[
(("adamW", None, None, None), pytest.raises(KeyError)),
((None, "Warmup_Cosine", None, None), pytest.raises(KeyError)),
((None, None, "Coco_Voc", "large"), pytest.raises(KeyError)),
((None, None, None, "Extra_Large"), pytest.raises(KeyError)),
(("adamW", None, None, None, "Enable", "Enable"), pytest.raises(KeyError)),
((None, "Warmup_Cosine", None, None, "Enable", "Enable"), pytest.raises(KeyError)),
((None, None, "Coco_Voc", "large", "Enable", "Enable"), pytest.raises(KeyError)),
((None, None, None, "Extra_Large", "Enable", "Enable"), pytest.raises(KeyError)),
((None, None, None, None, "false", "Enable"), pytest.raises(KeyError)),
((None, None, None, None, "Enable", "false"), pytest.raises(KeyError)),
],
ids=[
"optimizer invalid",
"learning rate scheduler invalid",
"validation metric type invalid",
"model size invalid",
"log_training_metrics invalid",
"log_validation_loss invalid",
],
)
def test_image_set_training_parameters_with_invalid_values(self, settings, expected):
Expand All @@ -191,30 +215,46 @@ def test_image_set_training_parameters_with_invalid_values(self, settings, expec
learning_rate_scheduler=settings[1],
validation_metric_type=settings[2],
model_size=settings[3],
log_training_metrics=settings[4],
log_validation_loss=settings[5],
)

@pytest.mark.parametrize(
"settings, expected",
[
(
("adam", "warmup_cosine", "coco_voc", "large"),
("adam", "warmup_cosine", "coco_voc", "large", "enable", "enable"),
(
StochasticOptimizer.ADAM,
LearningRateScheduler.WARMUP_COSINE,
ValidationMetricType.COCO_VOC,
ModelSize.LARGE,
LogTrainingMetrics.ENABLE,
LogValidationLoss.ENABLE,
),
),
(
("Adam", "WarmupCosine", "CocoVoc", "Large"),
("Adam", "WarmupCosine", "CocoVoc", "Large", "Enable", "Enable"),
(
StochasticOptimizer.ADAM,
LearningRateScheduler.WARMUP_COSINE,
ValidationMetricType.COCO_VOC,
ModelSize.LARGE,
LogTrainingMetrics.ENABLE,
LogValidationLoss.ENABLE,
),
),
(
(None, None, "coco_voc", "large", "enable", "enable"),
(
None,
None,
ValidationMetricType.COCO_VOC,
ModelSize.LARGE,
LogTrainingMetrics.ENABLE,
LogValidationLoss.ENABLE,
),
),
((None, None, "coco_voc", "large"), (None, None, ValidationMetricType.COCO_VOC, ModelSize.LARGE)),
],
ids=["snake case", "camel case", "with None"],
)
Expand All @@ -224,13 +264,18 @@ def test_image_set_training_parameters_with_settings_object(self, settings, expe
learning_rate_scheduler=settings[1],
validation_metric_type=settings[2],
model_size=settings[3],
log_training_metrics=settings[4],
log_validation_loss=settings[5],
)
image_instance_segmentation_job = image_instance_segmentation(
training_data=Input(type=AssetTypes.MLTABLE, path="https://foo/bar/train.csv"),
target_column_name="label",
training_parameters=image_model_settings,
)

assert image_instance_segmentation_job.training_parameters.optimizer == expected[0]
assert image_instance_segmentation_job.training_parameters.learning_rate_scheduler == expected[1]
assert image_instance_segmentation_job.training_parameters.validation_metric_type == expected[2]
assert image_instance_segmentation_job.training_parameters.model_size == expected[3]
assert image_instance_segmentation_job.training_parameters.log_training_metrics == expected[4]
assert image_instance_segmentation_job.training_parameters.log_validation_loss == expected[5]
Loading

0 comments on commit ae2c36e

Please sign in to comment.