Merge branch 'main' into wanhan/parallel_for_assets

.github/CODEOWNERS

@@ -6,27 +6,20 @@
 ################
 # As of 1/26/2023 these paths have no owners:
 
-# /
+# /**
 # /.devcontainer/
 # /.vscode/
 # /conda-releaselogs/
 # /doc/
 # /scripts/
 # /tools/
 
-###########
-# Eng Sys
-###########
-/eng/                                                                @scbedd @weshaggard @benbp
-/**/tests.yml                                                        @scbedd @benbp
-/**/ci.yml                                                           @scbedd @benbp
-
 ################
 # Automation
 ################
 
 # Git Hub integration and bot rules
-/.github/                                       @jsquire @ronniegeraghty
+/.github/                                                            @jsquire @ronniegeraghty
 
 ###########
 # SDK
@@ -921,6 +914,12 @@
 # Management Plane
 /**/*mgmt*/                                                          @Wzb123456789 @msyyc
 
+
+###########
+# Eng Sys
+###########
+/eng/                                                                @scbedd @weshaggard @benbp
+
 # Add owners for notifications for specific pipelines
 /eng/pipelines/templates/jobs/tests-nightly-python.yml               @lmazuel @mccoyp
 /eng/pipelines/aggregate-reports.yml                                 @lmazuel @mccoyp @YalinLi0312

sdk/appconfiguration/azure-appconfiguration/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Release History
 
+## 1.4.0b2 (Unreleased)
+
+### Features Added
+
+### Breaking Changes
+
+### Bugs Fixed
+
+### Other Changes
+
 ## 1.4.0b1 (2022-02-07)
 
 ### Other Changes

sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_version.py

@@ -3,4 +3,4 @@
 # Licensed under the MIT License.
 # ------------------------------------
 
-VERSION = "1.4.0b1"
+VERSION = "1.4.0b2"

sdk/cosmos/azure-cosmos/test/test_env.py

@@ -25,6 +25,7 @@
 from azure.cosmos import PartitionKey
 import test_config
 import os
+import uuid
 
 pytestmark = pytest.mark.cosmosEmulator
 
@@ -72,7 +73,7 @@ def tearDownClass(cls):
 
     def test_insert(self):
         # create a document using the document definition
-        d = {'id': '1',
+        d = {'id': str(uuid.uuid4()),
                  'name': 'sample document',
                  'spam': 'eggs',
                  'cnt': '1',

sdk/cosmos/azure-cosmos/test/test_headers.py

@@ -45,8 +45,8 @@ class HeadersTest(unittest.TestCase):
     def setUpClass(cls):
         cls.client = cosmos_client.CosmosClient(cls.host, cls.masterKey)
         cls.database = cls.client.create_database_if_not_exists(test_config._test_config.TEST_DATABASE_ID)
-        cls.container = cls.database.create_container(id=test_config._test_config.TEST_COLLECTION_MULTI_PARTITION_ID,
-                                                      partition_key=PartitionKey(path="/id"))
+        cls.container = cls.database.create_container_if_not_exists(
+            id=test_config._test_config.TEST_COLLECTION_MULTI_PARTITION_ID, partition_key=PartitionKey(path="/id"))
 
     def side_effect_dedicated_gateway_max_age_thousand(self, *args, **kwargs):
         # Extract request headers from args

sdk/identity/azure-identity/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Release History
 
+## 1.13.0b3 (Unreleased)
+
+### Features Added
+
+### Breaking Changes
+
+### Bugs Fixed
+
+### Other Changes
+
 ## 1.13.0b2 (2023-02-07)
 
 ### Features Added

sdk/identity/azure-identity/azure/identity/_version.py

@@ -2,4 +2,4 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 # ------------------------------------
-VERSION = "1.13.0b2"
+VERSION = "1.13.0b3"

sdk/ml/azure-ai-ml/CHANGELOG.md

@@ -1,6 +1,8 @@
 # Release History
 
-## 1.4.0 (Unreleased)
+## 1.5.0 (Unreleased)
+
+## 1.4.0 (2023-02-07)
 
 ### Features Added
 - Added dedicated classes for each type of job service and updated the docstrings. The classes added are `JupyterLabJobService, SshJobService, TensorBoardJobService, VsCodeJobService` with a few properties specific to the type.

sdk/ml/azure-ai-ml/azure/ai/ml/_azure_environments.py

@@ -10,6 +10,11 @@
 
 from azure.ai.ml._utils.utils import _get_mfe_url_override
 from azure.ai.ml.constants._common import AZUREML_CLOUD_ENV_NAME
+from azure.ai.ml.constants._common import ArmConstants
+from azure.core.rest import HttpRequest
+from azure.mgmt.core import ARMPipelineClient
+
+
 
 module_logger = logging.getLogger(__name__)
 
@@ -56,6 +61,19 @@ class EndpointURLS:  # pylint: disable=too-few-public-methods,no-init
     },
 }
 
+_requests_pipeline = None
+
+def _get_cloud(cloud: str):
+    if cloud in _environments:
+        return _environments[cloud]
+    arm_url = os.environ.get(ArmConstants.METADATA_URL_ENV_NAME,ArmConstants.DEFAULT_URL)
+    arm_clouds = _get_clouds_by_metadata_url(arm_url)
+    try:
+        new_cloud = arm_clouds[cloud]
+        _environments.update(new_cloud)
+        return new_cloud
+    except KeyError:
+        raise Exception('Unknown cloud environment "{0}".'.format(cloud))
 
 def _get_default_cloud_name():
     """Return AzureCloud as the default cloud."""
@@ -74,17 +92,18 @@ def _get_cloud_details(cloud: str = AzureEnvironments.ENV_DEFAULT):
             AzureEnvironments.ENV_DEFAULT,
         )
         cloud = _get_default_cloud_name()
-    try:
-        azure_environment = _environments[cloud]
-        module_logger.debug("Using the cloud configuration: '%s'.", azure_environment)
-    except KeyError:
-        raise Exception('Unknown cloud environment "{0}".'.format(cloud))
-    return azure_environment
+    return _get_cloud(cloud)
 
 
 def _set_cloud(cloud: str = AzureEnvironments.ENV_DEFAULT):
+    """Sets the current cloud
+
+    :param cloud: cloud name
+    """
     if cloud is not None:
-        if cloud not in _environments:
+        try:
+            _get_cloud(cloud)
+        except Exception:
             raise Exception('Unknown cloud environment supplied: "{0}".'.format(cloud))
     else:
         cloud = _get_default_cloud_name()
@@ -189,3 +208,74 @@ def _resource_to_scopes(resource):
     """
     scope = resource + "/.default"
     return [scope]
+
+def _get_registry_discovery_url(cloud, cloud_suffix=""):
+    """Get or generate the registry discovery url
+
+        :param cloud: configuration of the cloud to get the registry_discovery_url from
+        :param cloud_suffix: the suffix to use for the cloud, in the case that the registry_discovery_url
+            must be generated
+        :return: string of discovery url
+    """
+    cloud_name = cloud["name"]
+    if cloud_name in _environments:
+        return _environments[cloud_name].registry_url
+
+    registry_discovery_region = os.environ.get(
+        ArmConstants.REGISTRY_DISCOVERY_REGION_ENV_NAME,
+        ArmConstants.REGISTRY_DISCOVERY_DEFAULT_REGION
+    )
+    registry_discovery_region_default = "https://{}{}.api.azureml.{}/".format(
+        cloud_name.lower(),
+        registry_discovery_region,
+        cloud_suffix
+    )
+    return os.environ.get(ArmConstants.REGISTRY_ENV_URL, registry_discovery_region_default)
+
+def _get_clouds_by_metadata_url(metadata_url):
+    """Get all the clouds by the specified metadata url
+
+        :return: list of the clouds
+    """
+    try:
+        module_logger.debug('Start : Loading cloud metadata from the url specified by %s', metadata_url)
+        client = ARMPipelineClient(base_url=metadata_url, policies=[])
+        HttpRequest("GET", metadata_url)
+        with client.send_request(HttpRequest("GET", metadata_url)) as meta_response:
+            arm_cloud_dict = meta_response.json()
+            cli_cloud_dict = _convert_arm_to_cli(arm_cloud_dict)
+            module_logger.debug('Finish : Loading cloud metadata from the url specified by %s', metadata_url)
+            return cli_cloud_dict
+    except Exception as ex:  # pylint: disable=broad-except
+        module_logger.warning("Error: Azure ML was unable to load cloud metadata from the url specified by %s. %s. "
+                        "This may be due to a misconfiguration of networking controls. Azure Machine Learning Python "
+                        "SDK requires outbound access to Azure Resource Manager. Please contact your networking team "
+                        "to configure outbound access to Azure Resource Manager on both Network Security Group and "
+                        "Firewall. For more details on required configurations, see "
+                        "https://docs.microsoft.com/azure/machine-learning/how-to-access-azureml-behind-firewall.",
+                        metadata_url, ex)
+        return {}
+
+def _convert_arm_to_cli(arm_cloud_metadata):
+    cli_cloud_metadata_dict = {}
+    if isinstance(arm_cloud_metadata, dict):
+        arm_cloud_metadata = [arm_cloud_metadata]
+
+    for cloud in arm_cloud_metadata:
+        try:
+            cloud_name = cloud["name"]
+            portal_endpoint = cloud["portal"]
+            cloud_suffix = ".".join(portal_endpoint.split('.')[2:]).replace("/", "")
+            registry_discovery_url = _get_registry_discovery_url(cloud, cloud_suffix)
+            cli_cloud_metadata_dict[cloud_name] = {
+                EndpointURLS.AZURE_PORTAL_ENDPOINT: cloud["portal"],
+                EndpointURLS.RESOURCE_MANAGER_ENDPOINT: cloud["resourceManager"],
+                EndpointURLS.ACTIVE_DIRECTORY_ENDPOINT: cloud["authentication"]["loginEndpoint"],
+                EndpointURLS.AML_RESOURCE_ID: "https://ml.azure.{}".format(cloud_suffix),
+                EndpointURLS.STORAGE_ENDPOINT: cloud["suffixes"]["storage"],
+                EndpointURLS.REGISTRY_DISCOVERY_ENDPOINT: registry_discovery_url
+            }
+        except KeyError as ex:
+            module_logger.warning("Property on cloud not found in arm cloud metadata: %s", ex)
+            continue
+    return cli_cloud_metadata_dict

sdk/ml/azure-ai-ml/azure/ai/ml/_version.py

@@ -2,4 +2,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-VERSION = "1.4.0"
+VERSION = "1.5.0"

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_common.py

@@ -276,6 +276,12 @@ class ArmConstants(object):
     AZURE_MGMT_KEYVAULT_API_VERSION = "2019-09-01"
     AZURE_MGMT_CONTAINER_REG_API_VERSION = "2019-05-01"
 
+    DEFAULT_URL = "https://management.azure.com/metadata/endpoints?api-version=2019-05-01"
+    METADATA_URL_ENV_NAME = "ARM_CLOUD_METADATA_URL"
+    REGISTRY_DISCOVERY_DEFAULT_REGION = "west"
+    REGISTRY_DISCOVERY_REGION_ENV_NAME = "REGISTRY_DISCOVERY_ENDPOINT_REGION"
+    REGISTRY_ENV_URL = "REGISTRY_DISCOVERY_ENDPOINT_URL"
+
 
 class HttpResponseStatusCode(object):
     NOT_FOUND = 404

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/pipeline_component.py

@@ -264,7 +264,7 @@ def _validate_binding_inputs(self, node: BaseNode) -> MutableValidationResult:
                 else:
                     # Raise exception if pipeline input is optional set by user but link to required inputs.
                     validation_result.append_error(
-                        yaml_path="inputs.{}".format(pipeline_input._arg_name),
+                        yaml_path="inputs.{}".format(pipeline_input._port_name),
                         message=f"Pipeline optional Input binding to required inputs: {required_bindings}",
                     )
         return validation_result

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/input.py

@@ -214,9 +214,9 @@ def __init__(
         **kwargs,
     ):
         super(Input, self).__init__(type=type)
-        # As an annotation, it is not allowed to initialize the _arg_name.
-        # The _arg_name will be updated by the annotated variable _arg_name.
-        self._arg_name = None
+        # As an annotation, it is not allowed to initialize the _port_name.
+        # The _port_name will be updated by the annotated variable _port_name.
+        self._port_name = None
         self.description = description
 
         if path is not None and not isinstance(path, str):
@@ -291,8 +291,8 @@ def _parse(self, val):
             if lower_val not in {"true", "false"}:
                 msg = "Boolean parameter '{}' only accept True/False, got {}."
                 raise ValidationException(
-                    message=msg.format(self._arg_name, val),
-                    no_personal_data_message=msg.format("[self._arg_name]", "[val]"),
+                    message=msg.format(self._port_name, val),
+                    no_personal_data_message=msg.format("[self._port_name]", "[val]"),
                     error_category=ErrorCategory.USER_ERROR,
                     target=ErrorTarget.PIPELINE,
                     error_type=ValidationErrorType.INVALID_VALUE,
@@ -314,11 +314,11 @@ def _parse_and_validate(self, val):
         return val
 
     def _update_name(self, name):
-        self._arg_name = name
+        self._port_name = name
 
     def _update_default(self, default_value):
         """Update provided default values."""
-        name = "" if not self._arg_name else f"{self._arg_name!r} "
+        name = "" if not self._port_name else f"{self._port_name!r} "
         msg_prefix = f"Default value of Input {name}"
 
         if not self._is_primitive_type and default_value is not None:
@@ -360,8 +360,8 @@ def _validate_or_throw(self, value):
         if not self.optional and value is None:
             msg = "Parameter {} cannot be None since it is not optional."
             raise ValidationException(
-                message=msg.format(self._arg_name),
-                no_personal_data_message=msg.format("[self._arg_name]"),
+                message=msg.format(self._port_name),
+                no_personal_data_message=msg.format("[self._port_name]"),
                 error_category=ErrorCategory.USER_ERROR,
                 target=ErrorTarget.PIPELINE,
                 error_type=ValidationErrorType.INVALID_VALUE,
@@ -370,8 +370,8 @@ def _validate_or_throw(self, value):
             if not isinstance(value, self._allowed_types):
                 msg = "Unexpected data type for parameter '{}'. Expected {} but got {}."
                 raise ValidationException(
-                    message=msg.format(self._arg_name, self._allowed_types, type(value)),
-                    no_personal_data_message=msg.format("[_arg_name]", self._allowed_types, type(value)),
+                    message=msg.format(self._port_name, self._allowed_types, type(value)),
+                    no_personal_data_message=msg.format("[_port_name]", self._allowed_types, type(value)),
                     error_category=ErrorCategory.USER_ERROR,
                     target=ErrorTarget.PIPELINE,
                     error_type=ValidationErrorType.INVALID_VALUE,
@@ -381,17 +381,17 @@ def _validate_or_throw(self, value):
             if self.min is not None and value < self.min:
                 msg = "Parameter '{}' should not be less than {}."
                 raise ValidationException(
-                    message=msg.format(self._arg_name, self.min),
-                    no_personal_data_message=msg.format("[_arg_name]", self.min),
+                    message=msg.format(self._port_name, self.min),
+                    no_personal_data_message=msg.format("[_port_name]", self.min),
                     error_category=ErrorCategory.USER_ERROR,
                     target=ErrorTarget.PIPELINE,
                     error_type=ValidationErrorType.INVALID_VALUE,
                 )
             if self.max is not None and value > self.max:
                 msg = "Parameter '{}' should not be greater than {}."
                 raise ValidationException(
-                    message=msg.format(self._arg_name, self.max),
-                    no_personal_data_message=msg.format("[_arg_name]", self.max),
+                    message=msg.format(self._port_name, self.max),
+                    no_personal_data_message=msg.format("[_port_name]", self.max),
                     error_category=ErrorCategory.USER_ERROR,
                     target=ErrorTarget.PIPELINE,
                     error_type=ValidationErrorType.INVALID_VALUE,

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/output.py

@@ -86,9 +86,9 @@ def __init__(self, type: Literal["uri_file"] = "uri_file", path=None, mode=None,
 
     def __init__(self, *, type=AssetTypes.URI_FOLDER, path=None, mode=None, description=None, **kwargs):
         super(Output, self).__init__(type=type)
-        # As an annotation, it is not allowed to initialize the _arg_name.
-        # The _arg_name will be updated by the annotated variable _arg_name.
-        self._arg_name = None
+        # As an annotation, it is not allowed to initialize the _port_name.
+        # The _port_name will be updated by the annotated variable _port_name.
+        self._port_name = None
         self.name = kwargs.pop('name', None)
         self.version = kwargs.pop('version', None)
         self._is_primitive_type = self.type in IOConstants.PRIMITIVE_STR_2_TYPE

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_inputs_outputs/utils.py

@@ -114,7 +114,7 @@ def _update_annotation_with_default(anno, name, default):
         complete_annotation = anno
         if _is_dsl_type_cls(anno):
             complete_annotation = anno()
-        complete_annotation._arg_name = name
+        complete_annotation._port_name = name
         if default is Input._EMPTY:
             return complete_annotation
         if isinstance(complete_annotation, Input):
@@ -128,7 +128,7 @@ def _update_annotation_with_default(anno, name, default):
             complete_annotation._update_default(default)
         if isinstance(complete_annotation, Output) and default is not None:
             msg = (
-                f"Default value of Output {complete_annotation._arg_name!r} cannot be set:"
+                f"Default value of Output {complete_annotation._port_name!r} cannot be set:"
                 f"Output has no default value."
             )
             raise UserErrorException(msg)