From d02bc458c64b2b82e708d137fe15117ce567b941 Mon Sep 17 00:00:00 2001 From: Jamie Parsons Date: Fri, 7 Jul 2023 15:44:23 +0100 Subject: [PATCH 1/5] Multiple instances of the same NF --- src/aosm/azext_aosm/_configuration.py | 14 ++ .../generate_nfd/cnf_nfd_generator.py | 2 +- .../azext_aosm/generate_nsd/nsd_generator.py | 122 ++++++------ .../templates/nf_template.bicep.j2 | 14 +- .../mock_nsd/input_multiple_instances.json | 14 ++ src/aosm/azext_aosm/tests/latest/test_nsd.py | 184 ++++++++++++++---- src/aosm/azext_aosm/tests/latest/test_vnf.py | 2 +- 7 files changed, 246 insertions(+), 106 deletions(-) create mode 100644 src/aosm/azext_aosm/tests/latest/mock_nsd/input_multiple_instances.json diff --git a/src/aosm/azext_aosm/_configuration.py b/src/aosm/azext_aosm/_configuration.py index 744edb79c8f..10585c2e761 100644 --- a/src/aosm/azext_aosm/_configuration.py +++ b/src/aosm/azext_aosm/_configuration.py @@ -84,6 +84,10 @@ "network_function_type": ( "Type of nf in the definition. Valid values are 'cnf' or 'vnf'" ), + "multiple_instances": ( + "Whether the NSD should allow arbitrary numbers of this type of NF. If set to " + "false only a single instance will be allowed. Defaults to false." + ), "helm_package_name": "Name of the Helm package", "path_to_chart": ( "File path of Helm Chart on local disk. Accepts .tgz, .tar or .tar.gz" @@ -221,6 +225,14 @@ class NSConfiguration(Configuration): nsdg_name: str = DESCRIPTION_MAP["nsdg_name"] nsd_version: str = DESCRIPTION_MAP["nsd_version"] nsdv_description: str = DESCRIPTION_MAP["nsdv_description"] + multiple_instances: bool = DESCRIPTION_MAP["multiple_instances"] + + def __post_init__(self): + """ + Finish setting up the instance. + """ + if self.multiple_instances == DESCRIPTION_MAP["multiple_instances"]: + self.multiple_instances = False def validate(self): """Validate that all of the configuration parameters are set.""" @@ -269,6 +281,8 @@ def validate(self): raise ValueError("NSDG name must be set") if self.nsd_version == DESCRIPTION_MAP["nsd_version"] or "": raise ValueError("NSD Version must be set") + if not isinstance(self.multiple_instances, bool): + raise ValueError("multiple_instances must be a boolean") @property def output_directory_for_build(self) -> Path: diff --git a/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py b/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py index dbf2bc7a314..16f947f72b8 100644 --- a/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py +++ b/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py @@ -557,6 +557,7 @@ def traverse_dict( :param d: The dictionary to traverse. :param target: The regex to search for. """ + # pylint: disable=too-many-nested-blocks @dataclass class DictNode: @@ -575,7 +576,6 @@ class DictNode: # For each key-value pair in the popped item for key, value in node.sub_dict.items(): - # If the value is a dictionary if isinstance(value, dict): # Add the dictionary to the stack with the path diff --git a/src/aosm/azext_aosm/generate_nsd/nsd_generator.py b/src/aosm/azext_aosm/generate_nsd/nsd_generator.py index 18c81f3731d..4111377d9c2 100644 --- a/src/aosm/azext_aosm/generate_nsd/nsd_generator.py +++ b/src/aosm/azext_aosm/generate_nsd/nsd_generator.py @@ -4,7 +4,6 @@ # -------------------------------------------------------------------------------------- """Contains a class for generating NSDs and associated resources.""" import json -import copy import os import shutil import tempfile @@ -62,10 +61,6 @@ def __init__(self, api_clients: ApiClients, config: NSConfiguration): self.nsd_bicep_template_name = NSD_DEFINITION_JINJA2_SOURCE_TEMPLATE self.nf_bicep_template_name = NF_TEMPLATE_JINJA2_SOURCE_TEMPLATE self.nsd_bicep_output_name = NSD_BICEP_FILENAME - self.nfdv_parameter_name = ( - f"{self.config.network_function_definition_group_name.replace('-', '_')}" - "_nfd_version" - ) nfdv = self._get_nfdv(config, api_clients) print("Finding the deploy parameters of the NFDV resource") if not nfdv.deploy_parameters: @@ -75,6 +70,10 @@ def __init__(self, api_clients: ApiClients, config: NSConfiguration): self.deploy_parameters: Optional[Dict[str, Any]] = json.loads( nfdv.deploy_parameters ) + self.nf_type = self.config.network_function_definition_group_name.replace( + "-", "_" + ) + self.nfdv_parameter_name = f"{self.nf_type}_nfd_version" # pylint: disable=no-self-use def _get_nfdv( @@ -100,7 +99,9 @@ def generate_nsd(self) -> None: # Create temporary folder. with tempfile.TemporaryDirectory() as tmpdirname: - self.tmp_folder_name = tmpdirname # pylint: disable=attribute-defined-outside-init + self.tmp_folder_name = ( + tmpdirname # pylint: disable=attribute-defined-outside-init + ) self.create_config_group_schema_files() self.write_nsd_manifest() @@ -126,24 +127,12 @@ def config_group_schema_dict(self) -> Dict[str, Any]: """ assert self.deploy_parameters - # Take a copy of the deploy parameters. - cgs_dict = copy.deepcopy(self.deploy_parameters) - - # Re-title it. - cgs_dict["title"] = self.config.cg_schema_name - - # Add in the NFDV version as a parameter. - description_string = ( + nfdv_version_description_string = ( f"The version of the {self.config.network_function_definition_group_name} " "NFD to use. This version must be compatible with (have the same " "parameters exposed as) " f"{self.config.network_function_definition_version_name}." ) - cgs_dict["properties"][self.nfdv_parameter_name] = { - "type": "string", - "description": description_string, - } - cgs_dict.setdefault("required", []).append(self.nfdv_parameter_name) managed_identity_description_string = ( "The managed identity to use to deploy NFs within this SNS. This should " @@ -152,13 +141,52 @@ def config_group_schema_dict(self) -> Dict[str, Any]: "userAssignedIdentities/{identityName}. " "If you wish to use a system assigned identity, set this to a blank string." ) - cgs_dict["properties"]["managedIdentity"] = { - "type": "string", - "description": managed_identity_description_string, + + if self.config.multiple_instances: + deploy_parameters = { + "type": "array", + "items": { + "type": "object", + "properties": self.deploy_parameters["properties"], + }, + } + else: + deploy_parameters = { + "type": "object", + "properties": self.deploy_parameters["properties"], + } + + cgs_dict = { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": self.config.cg_schema_name, + "type": "object", + "properties": { + self.config.network_function_definition_group_name: { + "type": "object", + "properties": { + "deploymentParameters": deploy_parameters, + self.nfdv_parameter_name: { + "type": "string", + "description": nfdv_version_description_string, + }, + }, + "required": ["deploymentParameters", self.nfdv_parameter_name], + }, + "managedIdentity": { + "type": "string", + "description": managed_identity_description_string, + }, + }, + "required": [ + self.config.network_function_definition_group_name, + "managedIdentity", + ], } - cgs_dict["required"].append("managedIdentity") if self.config.network_function_type == CNF: + nf_schema = cgs_dict["properties"][ + self.config.network_function_definition_group_name + ] custom_location_description_string = ( "The custom location ID of the ARC-Enabled AKS Cluster to deploy the CNF " "to. Should be of the form " @@ -166,11 +194,12 @@ def config_group_schema_dict(self) -> Dict[str, Any]: "/{resourceGroupName}/providers/microsoft.extendedlocation/" "customlocations/{customLocationName}'" ) - cgs_dict["properties"]["customLocationId"] = { + + nf_schema["properties"]["customLocationId"] = { "type": "string", "description": custom_location_description_string, } - cgs_dict["required"].append("customLocationId") + nf_schema["required"].append("customLocationId") return cgs_dict @@ -207,12 +236,19 @@ def write_config_mappings(self, folder_path: str) -> None: :param folder_path: The folder to put this file in. """ - deploy_properties = self.config_group_schema_dict["properties"] + nf = self.config.network_function_definition_group_name + + logger.debug("Create %s", NSD_CONFIG_MAPPING_FILENAME) + + deployment_parameters = f"{{configurationparameters('{self.config.cg_schema_name}').{nf}.deploymentParameters}}" + + if not self.config.multiple_instances: + deployment_parameters = f"[{deployment_parameters}]" - logger.debug("Create configMappings.json") config_mappings = { - key: f"{{configurationparameters('{self.config.cg_schema_name}').{key}}}" - for key in deploy_properties + "deploymentParameters": deployment_parameters, + self.nfdv_parameter_name: f"{{configurationparameters('{self.config.cg_schema_name}').{nf}.{self.nfdv_parameter_name}}}", + "managedIdentity": f"{{configurationparameters('{self.config.cg_schema_name}').managedIdentity}}", } config_mappings_path = os.path.join(folder_path, NSD_CONFIG_MAPPING_FILENAME) @@ -224,38 +260,10 @@ def write_config_mappings(self, folder_path: str) -> None: def write_nf_bicep(self) -> None: """Write out the Network Function bicep file.""" - bicep_params = "" - - bicep_deploymentValues = "" - - if not self.deploy_parameters or not self.deploy_parameters.get("properties"): - raise ValueError( - f"NFDV in {self.config.network_function_definition_group_name} has " - "no properties within deployParameters" - ) - deploy_properties = self.deploy_parameters["properties"] - logger.debug("Deploy properties: %s", deploy_properties) - - for key, value in deploy_properties.items(): - # location is sometimes part of deploy_properties. - # We want to avoid having duplicate params in the bicep template - logger.debug( - "Adding deploy parameter key: %s, value: %s to nf template", key, value - ) - if key != "location": - bicep_type = ( - NFV_TO_BICEP_PARAM_TYPES.get(value["type"]) or value["type"] - ) - bicep_params += f"param {key} {bicep_type}\n" - bicep_deploymentValues += f"{key}: {key}\n " - - # pylint: disable=no-member self.generate_bicep( self.nf_bicep_template_name, NF_DEFINITION_BICEP_FILENAME, { - "bicep_params": bicep_params, - "deploymentValues": bicep_deploymentValues, "network_function_name": self.config.network_function_name, "publisher_name": self.config.publisher_name, "network_function_definition_group_name": ( diff --git a/src/aosm/azext_aosm/generate_nsd/templates/nf_template.bicep.j2 b/src/aosm/azext_aosm/generate_nsd/templates/nf_template.bicep.j2 index 8cf4a207a23..43d3ea8b429 100644 --- a/src/aosm/azext_aosm/generate_nsd/templates/nf_template.bicep.j2 +++ b/src/aosm/azext_aosm/generate_nsd/templates/nf_template.bicep.j2 @@ -29,11 +29,7 @@ param nfviType string = '{{nfvi_type}}' param resourceGroupId string = resourceGroup().id -{{bicep_params}} - -var deploymentValues = { - {{deploymentValues}} -} +param deploymentParameters array var identityObject = (managedIdentity == '') ? { type: 'SystemAssigned' @@ -44,8 +40,8 @@ var identityObject = (managedIdentity == '') ? { } } -resource nf_resource 'Microsoft.HybridNetwork/networkFunctions@2023-04-01-preview' = { - name: '{{network_function_name}}' +resource nf_resource 'Microsoft.HybridNetwork/networkFunctions@2023-04-01-preview' = [for (values, i) in deploymentParameters: { + name: '{{network_function_name}}${i}' location: location identity: identityObject properties: { @@ -61,6 +57,6 @@ resource nf_resource 'Microsoft.HybridNetwork/networkFunctions@2023-04-01-previe nfviId: resourceGroupId {%- endif %} allowSoftwareUpdate: true - deploymentValues: string(deploymentValues) + deploymentValues: string(values) } -} +}] diff --git a/src/aosm/azext_aosm/tests/latest/mock_nsd/input_multiple_instances.json b/src/aosm/azext_aosm/tests/latest/mock_nsd/input_multiple_instances.json new file mode 100644 index 00000000000..90566c3bbdd --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/mock_nsd/input_multiple_instances.json @@ -0,0 +1,14 @@ +{ + "location": "eastus", + "publisher_name": "jamie-mobile-publisher", + "publisher_resource_group_name": "Jamie-publisher", + "acr_artifact_store_name": "ubuntu-acr", + "network_function_definition_group_name": "ubuntu-vm-nfdg", + "network_function_definition_version_name": "1.0.0", + "network_function_definition_offering_location": "eastus", + "network_function_type": "vnf", + "nsdg_name": "ubuntu", + "nsd_version": "1.0.0", + "nsdv_description": "Plain ubuntu VM", + "multiple_instances": true +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/test_nsd.py b/src/aosm/azext_aosm/tests/latest/test_nsd.py index 0046b2aaf95..3026ffd5a09 100644 --- a/src/aosm/azext_aosm/tests/latest/test_nsd.py +++ b/src/aosm/azext_aosm/tests/latest/test_nsd.py @@ -6,6 +6,9 @@ import os from dataclasses import dataclass import json +import jsonschema +import shutil +import subprocess from pathlib import Path from unittest.mock import patch from tempfile import TemporaryDirectory @@ -16,6 +19,114 @@ mock_nsd_folder = ((Path(__file__).parent) / "mock_nsd").resolve() +CGV_DATA = { + "ubuntu-vm-nfdg": { + "deploymentParameters": { + "location": "eastus", + "subnetName": "subnet", + "virtualNetworkId": "bar", + "sshPublicKeyAdmin": "foo", + }, + "ubuntu_vm_nfdg_nfd_version": "1.0.0", + }, + "managedIdentity": "blah", +} + + +MULTIPLE_INSTANCES_CGV_DATA = { + "ubuntu-vm-nfdg": { + "deploymentParameters": [ + { + "location": "eastus", + "subnetName": "subnet", + "virtualNetworkId": "bar", + "sshPublicKeyAdmin": "foo", + }, + { + "location": "eastus", + "subnetName": "subnet2", + "virtualNetworkId": "bar2", + "sshPublicKeyAdmin": "foo2", + }, + ], + "ubuntu_vm_nfdg_nfd_version": "1.0.0", + }, + "managedIdentity": "blah", +} + + +deploy_parameters = { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "DeployParametersSchema", + "type": "object", + "properties": { + "location": {"type": "string"}, + "subnetName": {"type": "string"}, + "virtualNetworkId": {"type": "string"}, + "sshPublicKeyAdmin": {"type": "string"}, + }, +} + +deploy_parameters_string = json.dumps(deploy_parameters) + + +# We don't want to get details from a real NFD (calling out to Azure) in a UT. +# Therefore we pass in a fake client to supply the deployment parameters from the "NFD". +@dataclass +class NFDV: + deploy_parameters: Dict[str, Any] + + +nfdv = NFDV(deploy_parameters_string) + + +class NFDVs: + def get(self, **_): + return nfdv + + +class AOSMClient: + def __init__(self) -> None: + self.network_function_definition_versions = NFDVs() + + +mock_client = AOSMClient() + + +class FakeCmd: + def __init__(self) -> None: + self.cli_ctx = None + + +mock_cmd = FakeCmd() + + +def validate_json_against_schema(json_data, schema_file): + with open(schema_file, "r") as f: + schema = json.load(f) + + jsonschema.validate(instance=json_data, schema=schema) + + +def build_bicep(bicep_template_path): + bicep_output = subprocess.run( # noqa + [ + str(shutil.which("az")), + "bicep", + "build", + "--file", + bicep_template_path, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + if bicep_output.returncode != 0: + print(f"Invalid bicep: {bicep_template_path}") + print(str(bicep_output.stderr).replace("\\n", "\n").replace("\\t", "\t")) + raise RuntimeError("Invalid Bicep") + + class TestNSDGenerator: def test_generate_config(self): """ @@ -36,55 +147,52 @@ def test_build(self, cf_resources): """ Test building the NSD bicep templates. """ - # We don't want to get details from a real NFD (calling out to Azure) in a UT. - # Therefore we pass in a fake client to supply the deployment parameters from - # the "NFD". - deploy_parameters = { - "$schema": "https://json-schema.org/draft-07/schema#", - "title": "DeployParametersSchema", - "type": "object", - "properties": { - "location": {"type": "string"}, - "subnetName": {"type": "string"}, - "virtualNetworkId": {"type": "string"}, - "sshPublicKeyAdmin": {"type": "string"}, - }, - } - - deploy_parameters_string = json.dumps(deploy_parameters) - - @dataclass - class NFDV: - deploy_parameters: Dict[str, Any] - - nfdv = NFDV(deploy_parameters_string) - - class NFDVs: - def get(self, **_): - return nfdv - - class AOSMClient: - def __init__(self) -> None: - self.network_function_definition_versions = NFDVs() - - mock_client = AOSMClient() + starting_directory = os.getcwd() + with TemporaryDirectory() as test_dir: + os.chdir(test_dir) - class FakeCmd: - def __init__(self) -> None: - self.cli_ctx = None + try: + build_design( + mock_cmd, + client=mock_client, + config_file=str(mock_nsd_folder / "input.json"), + ) - cmd = FakeCmd() + assert os.path.exists("nsd-bicep-templates") + validate_json_against_schema( + CGV_DATA, + "nsd-bicep-templates/schemas/ubuntu_ConfigGroupSchema.json", + ) + build_bicep("nsd-bicep-templates/nf_definition.bicep") + build_bicep("nsd-bicep-templates/nsd_definition.bicep") + build_bicep("nsd-bicep-templates/artifact_manifest.bicep") + finally: + os.chdir(starting_directory) + @patch("azext_aosm.custom.cf_resources") + def test_build_multiple_instances(self, cf_resources): + """ + Test building the NSD bicep templates with multiple NFs allowed. + """ starting_directory = os.getcwd() with TemporaryDirectory() as test_dir: os.chdir(test_dir) try: build_design( - cmd, + mock_cmd, client=mock_client, - config_file=str(mock_nsd_folder / "input.json"), + config_file=str(mock_nsd_folder / "input_multiple_instances.json"), ) + assert os.path.exists("nsd-bicep-templates") + validate_json_against_schema( + MULTIPLE_INSTANCES_CGV_DATA, + "nsd-bicep-templates/schemas/ubuntu_ConfigGroupSchema.json", + ) + + # Don't bother validating the bicep here. It takes ages and there + # nothing different about the bicep in the multiple instances case. + finally: os.chdir(starting_directory) diff --git a/src/aosm/azext_aosm/tests/latest/test_vnf.py b/src/aosm/azext_aosm/tests/latest/test_vnf.py index 03dd4eb3d66..946255b8ccd 100644 --- a/src/aosm/azext_aosm/tests/latest/test_vnf.py +++ b/src/aosm/azext_aosm/tests/latest/test_vnf.py @@ -41,7 +41,7 @@ def test_build(self): assert os.path.exists("nfd-bicep-ubuntu-template") finally: os.chdir(starting_directory) - + with TemporaryDirectory() as test_dir: os.chdir(test_dir) From ebb5b46b0b7eaf06d3af3c6c644a790e6e75c66f Mon Sep 17 00:00:00 2001 From: Jamie Parsons Date: Sat, 8 Jul 2023 09:02:19 +0100 Subject: [PATCH 2/5] Tidy up --- src/aosm/azext_aosm/_configuration.py | 21 +- .../azext_aosm/generate_nsd/nsd_generator.py | 5 + .../azext_aosm/tests/latest/metaschema.json | 264 ++++++++++++++++++ .../tests/latest/metaschema_modified.json | 260 +++++++++++++++++ src/aosm/azext_aosm/tests/latest/test_nsd.py | 30 +- 5 files changed, 567 insertions(+), 13 deletions(-) create mode 100644 src/aosm/azext_aosm/tests/latest/metaschema.json create mode 100644 src/aosm/azext_aosm/tests/latest/metaschema_modified.json diff --git a/src/aosm/azext_aosm/_configuration.py b/src/aosm/azext_aosm/_configuration.py index 10585c2e761..a4616248c40 100644 --- a/src/aosm/azext_aosm/_configuration.py +++ b/src/aosm/azext_aosm/_configuration.py @@ -85,8 +85,9 @@ "Type of nf in the definition. Valid values are 'cnf' or 'vnf'" ), "multiple_instances": ( - "Whether the NSD should allow arbitrary numbers of this type of NF. If set to " - "false only a single instance will be allowed. Defaults to false." + "Set to true or false. Whether the NSD should allow arbitrary numbers of this " + "type of NF. If set to false only a single instance will be allowed. Only " + "supported on VNFs, must be set to false on CNFs." ), "helm_package_name": "Name of the Helm package", "path_to_chart": ( @@ -227,13 +228,6 @@ class NSConfiguration(Configuration): nsdv_description: str = DESCRIPTION_MAP["nsdv_description"] multiple_instances: bool = DESCRIPTION_MAP["multiple_instances"] - def __post_init__(self): - """ - Finish setting up the instance. - """ - if self.multiple_instances == DESCRIPTION_MAP["multiple_instances"]: - self.multiple_instances = False - def validate(self): """Validate that all of the configuration parameters are set.""" @@ -275,15 +269,24 @@ def validate(self): raise ValueError( "Network Function Definition Offering Location must be set" ) + if self.network_function_type not in [CNF, VNF]: raise ValueError("Network Function Type must be cnf or vnf") + if self.nsdg_name == DESCRIPTION_MAP["nsdg_name"] or "": raise ValueError("NSDG name must be set") + if self.nsd_version == DESCRIPTION_MAP["nsd_version"] or "": raise ValueError("NSD Version must be set") + if not isinstance(self.multiple_instances, bool): raise ValueError("multiple_instances must be a boolean") + # There is currently a NFM bug that means that multiple copies of the same NF + # cannot be deployed to the same custom location. + if self.network_function_type == CNF and self.multiple_instances: + raise ValueError("Multiple instances is not supported on CNFs.") + @property def output_directory_for_build(self) -> Path: """Return the local folder for generating the bicep template to.""" diff --git a/src/aosm/azext_aosm/generate_nsd/nsd_generator.py b/src/aosm/azext_aosm/generate_nsd/nsd_generator.py index 4111377d9c2..4e4ab68f3a3 100644 --- a/src/aosm/azext_aosm/generate_nsd/nsd_generator.py +++ b/src/aosm/azext_aosm/generate_nsd/nsd_generator.py @@ -251,6 +251,11 @@ def write_config_mappings(self, folder_path: str) -> None: "managedIdentity": f"{{configurationparameters('{self.config.cg_schema_name}').managedIdentity}}", } + if self.config.network_function_type == CNF: + config_mappings[ + "customLocationId" + ] = f"{{configurationparameters('{self.config.cg_schema_name}').{nf}.customLocationId}}" + config_mappings_path = os.path.join(folder_path, NSD_CONFIG_MAPPING_FILENAME) with open(config_mappings_path, "w", encoding="utf-8") as _file: diff --git a/src/aosm/azext_aosm/tests/latest/metaschema.json b/src/aosm/azext_aosm/tests/latest/metaschema.json new file mode 100644 index 00000000000..2281fcccfe9 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/metaschema.json @@ -0,0 +1,264 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "version": "2023-04-01-preview", + "$ref": "#/definitions/schemaObjectRoot", + "definitions": { + "schemaArray": { + "type": "object", + "properties": { + "type": { + "const": "array" + }, + "items": { + "$ref": "#/definitions/schemaAnyOf" + }, + "minItems": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxItems": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "schemaArrayMultiType": { + "type": "object", + "properties": { + "type": { + "const": "array" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/schemaAnyOf" + } + }, + "minItems": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "maxItems": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "additionalItems": { + "const": false + }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "type", + "additionalItems" + ], + "additionalProperties": false + }, + "schemaObjectRoot": { + "type": "object", + "properties": { + "format": { + "type": "string" + }, + "type": { + "const": "object" + }, + "$schema": { + "type": "string", + "format": "uri", + "not": { + "const": "https://json-schema.org/draft/2020-12/schema" + } + }, + //"$id": { + // "type": "string", + // "format": "uri-reference" + //}, + "version": { + "type": "string" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "required": { + "type": "array" + }, + "additionalProperties": { + "type": "boolean" + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "maxProperties": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minProperties": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/schemaAnyOf" + } + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "schemaObject": { + "type": "object", + "properties": { + "format": { + "type": "string" + }, + "type": { + "const": "object" + }, + "version": { + "type": "string" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "required": { + "type": "array" + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "additionalProperties": { + "type": "boolean" + }, + "maxProperties": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minProperties": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/schemaAnyOf" + } + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "schemaAnyOf": { + "anyOf": [ + { + "$ref": "#/definitions/schemaObject" + }, + { + "$ref": "#/definitions/schemaArray" + }, + { + "$ref": "#/definitions/schemaArrayMultiType" + }, + { + "$ref": "#/definitions/schemaPrimitive" + } + ] + }, + "schemaPrimitive": { + "type": "object", + "properties": { + "type": { + "enum": [ + "number", + "integer", + "string", + "boolean", + "null" + ] + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "default": { + "not": {} + } + }, + "required": [ + "type" + ], + "additionalProperties": true + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { + "$ref": "#/definitions/nonNegativeInteger" + }, + { + "default": 0 + } + ] + } + } +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/metaschema_modified.json b/src/aosm/azext_aosm/tests/latest/metaschema_modified.json new file mode 100644 index 00000000000..f0dfef8b530 --- /dev/null +++ b/src/aosm/azext_aosm/tests/latest/metaschema_modified.json @@ -0,0 +1,260 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "version": "2023-04-01-preview", + "$ref": "#/definitions/schemaObjectRoot", + "definitions": { + "schemaArray": { + "type": "object", + "properties": { + "type": { + "const": "array" + }, + "items": { + "$ref": "#/definitions/schemaAnyOf" + }, + "minItems": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxItems": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "schemaArrayMultiType": { + "type": "object", + "properties": { + "type": { + "const": "array" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/schemaAnyOf" + } + }, + "minItems": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "maxItems": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "additionalItems": { + "const": false + }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "type", + "additionalItems" + ], + "additionalProperties": false + }, + "schemaObjectRoot": { + "type": "object", + "properties": { + "format": { + "type": "string" + }, + "type": { + "const": "object" + }, + "$schema": { + "type": "string", + "format": "uri", + "not": { + "const": "https://json-schema.org/draft/2020-12/schema" + } + }, + "version": { + "type": "string" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "required": { + "type": "array" + }, + "additionalProperties": { + "type": "boolean" + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "maxProperties": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minProperties": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/schemaAnyOf" + } + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "schemaObject": { + "type": "object", + "properties": { + "format": { + "type": "string" + }, + "type": { + "const": "object" + }, + "version": { + "type": "string" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "required": { + "type": "array" + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "additionalProperties": { + "type": "boolean" + }, + "maxProperties": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minProperties": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/schemaAnyOf" + } + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "schemaAnyOf": { + "anyOf": [ + { + "$ref": "#/definitions/schemaObject" + }, + { + "$ref": "#/definitions/schemaArray" + }, + { + "$ref": "#/definitions/schemaArrayMultiType" + }, + { + "$ref": "#/definitions/schemaPrimitive" + } + ] + }, + "schemaPrimitive": { + "type": "object", + "properties": { + "type": { + "enum": [ + "number", + "integer", + "string", + "boolean", + "null" + ] + }, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "default": { + "not": {} + } + }, + "required": [ + "type" + ], + "additionalProperties": true + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { + "$ref": "#/definitions/nonNegativeInteger" + }, + { + "default": 0 + } + ] + } + } +} \ No newline at end of file diff --git a/src/aosm/azext_aosm/tests/latest/test_nsd.py b/src/aosm/azext_aosm/tests/latest/test_nsd.py index 3026ffd5a09..e229d58d721 100644 --- a/src/aosm/azext_aosm/tests/latest/test_nsd.py +++ b/src/aosm/azext_aosm/tests/latest/test_nsd.py @@ -101,10 +101,32 @@ def __init__(self) -> None: mock_cmd = FakeCmd() +def validate_schema_against_metaschema(schema_data): + """ + Validate that the schema produced by the CLI matches the AOSM metaschema. + """ + + # There is a bug in the jsonschema module that means that it hits an error in with + # the "$id" bit of the metaschema. Here we use a modified version of the metaschema + # with that small section removed. + metaschema_file_path = ( + (Path(__file__).parent) / "metaschema_modified.json" + ).resolve() + with open(metaschema_file_path, "r", encoding="utf8") as f: + metaschema = json.load(f) + + jsonschema.validate(instance=schema_data, schema=metaschema) + + def validate_json_against_schema(json_data, schema_file): - with open(schema_file, "r") as f: + """ + Validate some test data against the schema produced by the CLI. + """ + with open(schema_file, "r", encoding="utf8") as f: schema = json.load(f) + validate_schema_against_metaschema(schema) + jsonschema.validate(instance=json_data, schema=schema) @@ -163,9 +185,9 @@ def test_build(self, cf_resources): CGV_DATA, "nsd-bicep-templates/schemas/ubuntu_ConfigGroupSchema.json", ) - build_bicep("nsd-bicep-templates/nf_definition.bicep") - build_bicep("nsd-bicep-templates/nsd_definition.bicep") - build_bicep("nsd-bicep-templates/artifact_manifest.bicep") + # build_bicep("nsd-bicep-templates/nf_definition.bicep") + # build_bicep("nsd-bicep-templates/nsd_definition.bicep") + # build_bicep("nsd-bicep-templates/artifact_manifest.bicep") finally: os.chdir(starting_directory) From 1f18b8d25f8db57f229d9a3c851e4ddd446ab3b2 Mon Sep 17 00:00:00 2001 From: Jamie Parsons Date: Sat, 8 Jul 2023 09:16:04 +0100 Subject: [PATCH 3/5] Mypy and docs --- src/aosm/README.md | 9 +++++++++ src/aosm/azext_aosm/_configuration.py | 4 ++-- src/aosm/azext_aosm/generate_nsd/nsd_generator.py | 2 +- src/aosm/azext_aosm/tests/latest/test_nsd.py | 6 +++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/aosm/README.md b/src/aosm/README.md index d28dd58d2bd..aae28507f51 100644 --- a/src/aosm/README.md +++ b/src/aosm/README.md @@ -22,6 +22,8 @@ https://github.com/jddarby/azure-cli-extensions/releases/download/aosm-extension To install, download this wheel and run: `az extension add --source path/to/aosm-0.2.0-py2.py3-none-any.whl` +You must also have helm installed, instructions can be found here: https://helm.sh/docs/intro/install/#through-package-managers + ## Updating We are currently not bumping versions, so if you would like the most up to date version of the CLI. You should run: @@ -190,6 +192,13 @@ coverage run -m pytest . coverage report --include="*/src/aosm/*" --omit="*/src/aosm/azext_aosm/vendored_sdks/*","*/src/aosm/azext_aosm/tests/*" -m ``` +## Linting +Please run mypy on your changes and fix up any issues before merging. +```bash +cd src/aosm +mypy . --ignore-missing-imports --no-namespace-packages --exclude "azext_aosm/vendored_sdks/*" +``` + ## Pipelines The pipelines for the Azure CLI run in ADO, not in github. To trigger a pipeline you need to create a PR against main. diff --git a/src/aosm/azext_aosm/_configuration.py b/src/aosm/azext_aosm/_configuration.py index a4616248c40..6cc90a83744 100644 --- a/src/aosm/azext_aosm/_configuration.py +++ b/src/aosm/azext_aosm/_configuration.py @@ -10,7 +10,7 @@ import re from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from azure.cli.core.azclierror import InvalidArgumentValueError, ValidationError from azext_aosm.util.constants import ( @@ -226,7 +226,7 @@ class NSConfiguration(Configuration): nsdg_name: str = DESCRIPTION_MAP["nsdg_name"] nsd_version: str = DESCRIPTION_MAP["nsd_version"] nsdv_description: str = DESCRIPTION_MAP["nsdv_description"] - multiple_instances: bool = DESCRIPTION_MAP["multiple_instances"] + multiple_instances: Union[str, bool] = DESCRIPTION_MAP["multiple_instances"] def validate(self): """Validate that all of the configuration parameters are set.""" diff --git a/src/aosm/azext_aosm/generate_nsd/nsd_generator.py b/src/aosm/azext_aosm/generate_nsd/nsd_generator.py index 4e4ab68f3a3..1f6029917a2 100644 --- a/src/aosm/azext_aosm/generate_nsd/nsd_generator.py +++ b/src/aosm/azext_aosm/generate_nsd/nsd_generator.py @@ -156,7 +156,7 @@ def config_group_schema_dict(self) -> Dict[str, Any]: "properties": self.deploy_parameters["properties"], } - cgs_dict = { + cgs_dict: Dict[str, Any] = { "$schema": "https://json-schema.org/draft-07/schema#", "title": self.config.cg_schema_name, "type": "object", diff --git a/src/aosm/azext_aosm/tests/latest/test_nsd.py b/src/aosm/azext_aosm/tests/latest/test_nsd.py index e229d58d721..8bfd2186a1a 100644 --- a/src/aosm/azext_aosm/tests/latest/test_nsd.py +++ b/src/aosm/azext_aosm/tests/latest/test_nsd.py @@ -6,13 +6,13 @@ import os from dataclasses import dataclass import json -import jsonschema import shutil import subprocess from pathlib import Path from unittest.mock import patch from tempfile import TemporaryDirectory -from typing import Any, Dict + +import jsonschema from azext_aosm.custom import generate_design_config, build_design @@ -74,7 +74,7 @@ # Therefore we pass in a fake client to supply the deployment parameters from the "NFD". @dataclass class NFDV: - deploy_parameters: Dict[str, Any] + deploy_parameters: str nfdv = NFDV(deploy_parameters_string) From 64e9a44ca71764c1143c9738ab3842768960ec92 Mon Sep 17 00:00:00 2001 From: Jamie Parsons Date: Sat, 8 Jul 2023 09:17:24 +0100 Subject: [PATCH 4/5] history.rst --- src/aosm/HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aosm/HISTORY.rst b/src/aosm/HISTORY.rst index bb9d6dc4b25..f1b77fa61f6 100644 --- a/src/aosm/HISTORY.rst +++ b/src/aosm/HISTORY.rst @@ -18,6 +18,7 @@ unreleased * Add validation of source_registry_id format for CNF configuration * Workaround Oras client bug (#90) on Windows for Artifact upload to ACR * Take Oras 0.1.18 so above Workaround could be removed +* Support deploying multiple instances of the same NF in an SNS 0.2.0 ++++++ From 71ae242d8de3d648dc0cfe86ed45d4c0f8a05088 Mon Sep 17 00:00:00 2001 From: Jamie Parsons Date: Tue, 11 Jul 2023 15:52:10 +0100 Subject: [PATCH 5/5] self markups --- src/aosm/azext_aosm/_configuration.py | 3 ++- src/aosm/azext_aosm/tests/latest/metaschema.json | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/aosm/azext_aosm/_configuration.py b/src/aosm/azext_aosm/_configuration.py index 6cc90a83744..645e7897dab 100644 --- a/src/aosm/azext_aosm/_configuration.py +++ b/src/aosm/azext_aosm/_configuration.py @@ -283,7 +283,8 @@ def validate(self): raise ValueError("multiple_instances must be a boolean") # There is currently a NFM bug that means that multiple copies of the same NF - # cannot be deployed to the same custom location. + # cannot be deployed to the same custom location: + # https://portal.microsofticm.com/imp/v3/incidents/details/405078667/home if self.network_function_type == CNF and self.multiple_instances: raise ValueError("Multiple instances is not supported on CNFs.") diff --git a/src/aosm/azext_aosm/tests/latest/metaschema.json b/src/aosm/azext_aosm/tests/latest/metaschema.json index 2281fcccfe9..81f2b211f0c 100644 --- a/src/aosm/azext_aosm/tests/latest/metaschema.json +++ b/src/aosm/azext_aosm/tests/latest/metaschema.json @@ -106,10 +106,10 @@ "const": "https://json-schema.org/draft/2020-12/schema" } }, - //"$id": { - // "type": "string", - // "format": "uri-reference" - //}, + "$id": { + "type": "string", + "format": "uri-reference" + }, "version": { "type": "string" },